diff --git a/addon/YaCy.app/Contents/Info.plist b/addon/YaCy.app/Contents/Info.plist index 6ebb43566..c6f81e821 100644 --- a/addon/YaCy.app/Contents/Info.plist +++ b/addon/YaCy.app/Contents/Info.plist @@ -39,7 +39,7 @@ $JAVAROOT/htroot $JAVAROOT/lib/activation.jar $JAVAROOT/lib/apache-mime4j-0.6.jar - $JAVAROOT/lib/apache-solr-solrj-3.1.0.jar + $JAVAROOT/lib/apache-solr-solrj-3.3.0.jar $JAVAROOT/lib/bcmail-jdk15-145.jar $JAVAROOT/lib/bcprov-jdk15-145.jar $JAVAROOT/lib/bzip2.jar @@ -53,11 +53,12 @@ $JAVAROOT/lib/httpcore-4.1.1.jar $JAVAROOT/lib/httpmime-4.1.1.jar $JAVAROOT/lib/icu4j-core.jar - $JAVAROOT/lib/jempbox-1.6.0.jar $JAVAROOT/lib/J7Zip-modified.jar $JAVAROOT/lib/jakarta-oro-2.0.8.jar $JAVAROOT/lib/jcifs-1.3.15.jar + $JAVAROOT/lib/jempbox-1.6.0.jar $JAVAROOT/lib/jsch-0.1.42.jar + $JAVAROOT/lib/json-simple-1.1.jar $JAVAROOT/lib/log4j-1.2.16.jar $JAVAROOT/lib/metadata-extractor-2.4.0-beta-1.jar $JAVAROOT/lib/mysql-connector-java-5.1.12-bin.jar @@ -65,9 +66,8 @@ $JAVAROOT/lib/poi-3.6-20091214.jar $JAVAROOT/lib/poi-scratchpad-3.6-20091214.jar $JAVAROOT/lib/servlet-api.jar - $JAVAROOT/lib/json-simple-1.1.jar - $JAVAROOT/lib/slf4j-api-1.5.5.jar - $JAVAROOT/lib/slf4j-jdk14-1.5.5.jar + $JAVAROOT/lib/slf4j-api-1.6.1.jar + $JAVAROOT/lib/slf4j-jdk14-1.6.1.jar $JAVAROOT/lib/webcat-0.1-swf.jar $JAVAROOT/lib/xercesImpl.jar $JAVAROOT/lib/xml-apis.jar diff --git a/build.xml b/build.xml index d1c9dcb71..90d41572e 100644 --- a/build.xml +++ b/build.xml @@ -181,7 +181,7 @@ - + @@ -195,10 +195,10 @@ - + @@ -208,8 +208,8 @@ - - + + diff --git a/defaults/yacy.init b/defaults/yacy.init index 05cbb04c7..d168359f9 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -275,7 +275,7 @@ minimumGlobalDelta = 500 # the following mime-types are a blacklist for indexing: # parser.mime.deny: specifies mime-types that shall not be indexed parser.mime.deny= -parser.extensions.deny= +parser.extensions.deny=pdf # Promotion Strings # These strings appear in the Web Mask of the YACY search client diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 713211b1f..dd798dcd0 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -22,10 +22,14 @@ * If not, see . */ +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; import java.util.Iterator; +import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.services.federated.solr.SolrChardingConnector; import net.yacy.cora.services.federated.solr.SolrChardingSelection; @@ -51,12 +55,25 @@ public class IndexFederated_p { final boolean solrWasOn = env.getConfigBool("federated.service.solr.indexing.enabled", true); final boolean solrIsOnAfterwards = post.getBoolean("solr.indexing.enabled", false); env.setConfig("federated.service.solr.indexing.enabled", solrIsOnAfterwards); - env.setConfig("federated.service.solr.indexing.url", post.get("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr"))); + String solrurls = post.get("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr")); + final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls)))); + final StringBuilder s = new StringBuilder(); + String s0; + try { + while ((s0 = r.readLine()) != null) { + s0 = s0.trim(); + if (s0.length() > 0) s.append(s0).append(','); + } + } catch (final IOException e1) { + } + if (s.length() > 0) s.setLength(s.length() - 1); + solrurls = s.toString().trim(); + env.setConfig("federated.service.solr.indexing.url", solrurls); env.setConfig("federated.service.solr.indexing.charding", post.get("solr.indexing.charding", env.getConfig("federated.service.solr.indexing.charding", "modulo-host-md5"))); final String schemename = post.get("solr.indexing.schemefile", env.getConfig("federated.service.solr.indexing.schemefile", "solr.keys.default.list")); env.setConfig("federated.service.solr.indexing.schemefile", schemename); - if (solrWasOn && !solrIsOnAfterwards) { + if (solrWasOn) { // switch off sb.solrConnector.close(); sb.solrConnector = null; @@ -64,9 +81,8 @@ public class IndexFederated_p { final SolrScheme scheme = new SolrScheme(new File(env.getDataPath(), "DATA/SETTINGS/" + schemename)); - if (!solrWasOn && solrIsOnAfterwards) { + if (solrIsOnAfterwards) { // switch on - final String solrurls = sb.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr"); final boolean usesolr = sb.getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0; try { sb.solrConnector = (usesolr) ? new SolrChardingConnector(solrurls, scheme, SolrChardingSelection.Method.MODULO_HOST_MD5) : null; @@ -98,21 +114,15 @@ public class IndexFederated_p { prop.put("table", 0); } else { prop.put("table", 1); - try { - final long[] size = sb.solrConnector.getSizeList(); - final String[] urls = sb.solrConnector.getAdminInterfaceList(); - boolean dark = false; - for (int i = 0; i < size.length; i++) { - prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark; - prop.put("table_list_" + i + "_url", urls[i]); - prop.put("table_list_" + i + "_size", size[i]); - } - prop.put("table_list", size.length); - - } catch (final IOException e) { - Log.logException(e); - prop.put("table", 0); + final long[] size = sb.solrConnector.getSizeList(); + final String[] urls = sb.solrConnector.getAdminInterfaceList(); + boolean dark = false; + for (int i = 0; i < size.length; i++) { + prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark; + prop.put("table_list_" + i + "_url", urls[i]); + prop.put("table_list_" + i + "_size", size[i]); } + prop.put("table_list", size.length); } // write scheme @@ -138,7 +148,7 @@ public class IndexFederated_p { // fill attribute fields prop.put("yacy.indexing.enabled.checked", env.getConfigBool("federated.service.yacy.indexing.enabled", true) ? 1 : 0); prop.put("solr.indexing.enabled.checked", env.getConfigBool("federated.service.solr.indexing.enabled", false) ? 1 : 0); - prop.put("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr")); + prop.put("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr").replace(",", "\n")); prop.put("solr.indexing.charding", env.getConfig("federated.service.solr.indexing.charding", "modulo-host-md5")); prop.put("solr.indexing.schemefile", schemename); diff --git a/lib/apache-solr-solrj-3.1.0.txt b/lib/apache-solr-solrj-3.3.0.License similarity index 100% rename from lib/apache-solr-solrj-3.1.0.txt rename to lib/apache-solr-solrj-3.3.0.License diff --git a/lib/apache-solr-solrj-3.1.0.jar b/lib/apache-solr-solrj-3.3.0.jar similarity index 73% rename from lib/apache-solr-solrj-3.1.0.jar rename to lib/apache-solr-solrj-3.3.0.jar index 058d27350..06b81f3fb 100644 Binary files a/lib/apache-solr-solrj-3.1.0.jar and b/lib/apache-solr-solrj-3.3.0.jar differ diff --git a/lib/slf4j-api-1.5.5.jar b/lib/slf4j-api-1.5.5.jar deleted file mode 100644 index 4bb4abbb0..000000000 Binary files a/lib/slf4j-api-1.5.5.jar and /dev/null differ diff --git a/lib/slf4j-api-1.6.1.jar b/lib/slf4j-api-1.6.1.jar new file mode 100644 index 000000000..42e0ad0de Binary files /dev/null and b/lib/slf4j-api-1.6.1.jar differ diff --git a/lib/slf4j-jdk14-1.5.5.jar b/lib/slf4j-jdk14-1.5.5.jar deleted file mode 100644 index d58ef5a16..000000000 Binary files a/lib/slf4j-jdk14-1.5.5.jar and /dev/null differ diff --git a/lib/slf4j-jdk14-1.6.1.jar b/lib/slf4j-jdk14-1.6.1.jar new file mode 100644 index 000000000..f4eb2f8af Binary files /dev/null and b/lib/slf4j-jdk14-1.6.1.jar differ diff --git a/source/de/anomic/search/SwitchboardConstants.java b/source/de/anomic/search/SwitchboardConstants.java index 1497dcabc..00a72ec50 100644 --- a/source/de/anomic/search/SwitchboardConstants.java +++ b/source/de/anomic/search/SwitchboardConstants.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -42,7 +42,7 @@ public final class SwitchboardConstants { * by first encoding username:password as Base64 and hashing it using {@link MapTools#encodeMD5Hex(String)}.

*/ public static final String ADMIN_ACCOUNT_B64MD5 = "adminAccountBase64MD5"; - + public static final int CRAWLJOB_SYNC = 0; public static final int CRAWLJOB_STATUS = 1; // 20_dhtdistribution @@ -86,7 +86,7 @@ public final class SwitchboardConstants { *

public static final String CRAWLJOB_LOCAL_CRAWL = "50_localcrawl"

*

Name of the local crawler thread, popping one entry off the Local Crawl Queue, and passing it to the * proxy cache enqueue thread to download and further process it

- * + * * @see Switchboard#PROXY_CACHE_ENQUEUE */ public static final String CRAWLJOB_LOCAL_CRAWL = "50_localcrawl"; @@ -99,7 +99,7 @@ public final class SwitchboardConstants { /** *

public static final String CRAWLJOB_REMOTE_CRAWL_LOADER = "60_remotecrawlloader"

*

Name of the remote crawl list loading thread

- * + * * @see Switchboard#CRAWLJOB_REMOTE_CRAWL_LOADER */ public static final String CRAWLJOB_REMOTE_CRAWL_LOADER = "60_remotecrawlloader"; @@ -134,7 +134,7 @@ public final class SwitchboardConstants { // 90_cleanup /** *

public static final String CLEANUP = "90_cleanup"

- *

The cleanup thread which is responsible for pendant cleanup-jobs, news/ranking distribution, etc.

+ *

The cleanup thread which is responsible for pendant cleanup-jobs, news/ranking distribution, etc.

*/ public static final String CLEANUP = "90_cleanup"; public static final String CLEANUP_METHOD_START = "cleanupJob"; @@ -226,7 +226,7 @@ public final class SwitchboardConstants { /** *

public static final String INDEX_DIST_ALLOW = "allowDistributeIndex"

*

Name of the setting whether Index Distribution shall be allowed (and the DHT-thread therefore started) or not

- * + * * @see Switchboard#INDEX_DIST_ALLOW_WHILE_CRAWLING */ public static final String INDEX_DIST_ALLOW = "allowDistributeIndex"; @@ -236,7 +236,7 @@ public final class SwitchboardConstants { *

Name of the setting whether Index Distribution shall be allowed while crawling is in progress, i.e. * the Local Crawler Queue is filled.

*

This setting only has effect if {@link #INDEX_DIST_ALLOW} is enabled

- * + * * @see Switchboard#INDEX_DIST_ALLOW */ public static final String INDEX_DIST_ALLOW_WHILE_CRAWLING = "allowDistributeIndexWhileCrawling"; @@ -244,9 +244,10 @@ public final class SwitchboardConstants { public static final String INDEX_TRANSFER_TIMEOUT = "indexTransfer.timeout"; public static final String INDEX_TRANSFER_GZIP_BODY = "indexTransfer.gzipBody"; public static final String PARSER_MIME_DENY = "parser.mime.deny"; + public static final String PARSER_EXTENSIONS_DENY = "parser.extensions.deny"; /** *

public static final String PROXY_ONLINE_CAUTION_DELAY = "onlineCautionDelay"

- *

Name of the setting how long indexing should pause after the last time the proxy was used in milliseconds

+ *

Name of the setting how long indexing should pause after the last time the proxy was used in milliseconds

*/ public static final String PROXY_ONLINE_CAUTION_DELAY = "crawlPause.proxy"; public static final String LOCALSEACH_ONLINE_CAUTION_DELAY = "crawlPause.localsearch"; @@ -265,16 +266,16 @@ public final class SwitchboardConstants { *

public static final String PROXY_CACHE_LAYOUT = "proxyCacheLayout"

*

Name of the setting which file-/folder-layout the proxy cache shall use. Possible values are {@link #PROXY_CACHE_LAYOUT_TREE} * and {@link #PROXY_CACHE_LAYOUT_HASH}

- * + * * @see Switchboard#PROXY_CACHE_LAYOUT_TREE * @see Switchboard#PROXY_CACHE_LAYOUT_HASH */ public static final String PROXY_YACY_ONLY = "proxyYacyOnly"; - + ////////////////////////////////////////////////////////////////////////////////////////////// // Cluster settings ////////////////////////////////////////////////////////////////////////////////////////////// - + public static final String CLUSTER_MODE = "cluster.mode"; public static final String CLUSTER_MODE_PUBLIC_CLUSTER = "publiccluster"; public static final String CLUSTER_MODE_PRIVATE_CLUSTER = "privatecluster"; @@ -289,14 +290,14 @@ public final class SwitchboardConstants { public static final String REMOTESEARCH_MAXCOUNT_USER = "remotesearch.maxcount"; public static final String REMOTESEARCH_MAXTIME_USER = "remotesearch.maxtime"; - + /** *

public static final String CRAWLER_THREADS_ACTIVE_MAX = "crawler.MaxActiveThreads"

*

Name of the setting how many active crawler-threads may maximal be running on the same time

*/ public static final String CRAWLER_THREADS_ACTIVE_MAX = "crawler.MaxActiveThreads"; public static final String YACY_MODE_DEBUG = "yacyDebugMode"; - + /** *

public static final String WORDCACHE_MAX_COUNT = "wordCacheMaxCount"

*

Name of the setting how many words the word-cache (or DHT-Out cache) shall contain maximal. Indexing pages if the @@ -306,39 +307,39 @@ public final class SwitchboardConstants { public static final String HTTPC_NAME_CACHE_CACHING_PATTERNS_NO = "httpc.nameCacheNoCachingPatterns"; public static final String ROBOTS_TXT = "httpd.robots.txt"; public static final String ROBOTS_TXT_DEFAULT = RobotsTxtConfig.LOCKED + "," + RobotsTxtConfig.DIRS; - + /** *

public static final String BLACKLIST_CLASS_DEFAULT = "de.anomic.plasma.urlPattern.defaultURLPattern"

*

Package and name of YaCy's {@link DefaultBlacklist default} blacklist implementation

- * + * * @see DefaultBlacklist for a detailed overview about the syntax of the default implementation */ public static final String LIST_BLUE = "plasmaBlueList"; public static final String LIST_BLUE_DEFAULT = null; public static final String LIST_BADWORDS_DEFAULT = "yacy.badwords"; public static final String LIST_STOPWORDS_DEFAULT = "yacy.stopwords"; - + /** *

public static final String HTCACHE_PATH = "proxyCache"

*

Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all * downloaded webpages and their respective ressources and HTTP-headers are stored. It is the location containing * the proxy-cache

- * + * * @see Switchboard#PROXY_CACHE_LAYOUT for details on the file-layout in this path */ public static final String HTCACHE_PATH = "proxyCache"; public static final String HTCACHE_PATH_DEFAULT = "DATA/HTCACHE"; public static final String RELEASE_PATH = "releases"; public static final String RELEASE_PATH_DEFAULT = "DATA/RELEASE"; - + public static final String SURROGATES_IN_PATH = "surrogates.in"; public static final String SURROGATES_IN_PATH_DEFAULT = "DATA/SURROGATES/in"; public static final String SURROGATES_OUT_PATH = "surrogates.out"; public static final String SURROGATES_OUT_PATH_DEFAULT = "DATA/SURROGATES/out"; - + public static final String DICTIONARY_SOURCE_PATH = "dictionaries"; public static final String DICTIONARY_SOURCE_PATH_DEFAULT = "DATA/DICTIONARIES"; - + /** *

public static final String HTDOCS_PATH = "htDocsPath"

*

Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all @@ -350,7 +351,7 @@ public final class SwitchboardConstants { /** *

public static final String HTROOT_PATH = "htRootPath"

*

Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all - * original servlets, their stylesheets, scripts, etc. lie. It is also home of the XML-interface to YaCy

+ * original servlets, their stylesheets, scripts, etc. lie. It is also home of the XML-interface to YaCy

*/ public static final String HTROOT_PATH = "htRootPath"; public static final String HTROOT_PATH_DEFAULT = "htroot"; @@ -372,7 +373,7 @@ public final class SwitchboardConstants { *

public static final String WORK_PATH = "wordPath"

*

Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all * DBs containing "work" of the user are saved. Such include bookmarks, messages, wiki, blog

- * + * * @see Switchboard#DBFILE_BLOG * @see Switchboard#DBFILE_BOOKMARKS * @see Switchboard#DBFILE_BOOKMARKS_DATES @@ -384,16 +385,16 @@ public final class SwitchboardConstants { public static final String WORK_PATH = "workPath"; public static final String WORK_PATH_DEFAULT = "DATA/WORK"; - + /** * ResourceObserver */ public static final String DISK_FREE = "disk.free"; public static final String DISK_FREE_HARDLIMIT = "disk.free.hardlimit"; - + public static final String MEMORY_ACCEPTDHT = "memory.acceptDHTabove"; public static final String INDEX_RECEIVE_AUTODISABLED = "memory.disabledDHT"; - + /* * Some constants */ @@ -401,16 +402,16 @@ public final class SwitchboardConstants { /** * network properties - * + * */ public static final String NETWORK_NAME = "network.unit.name"; public static final String NETWORK_DOMAIN = "network.unit.domain"; public static final String NETWORK_DOMAIN_NOCHECK = "network.unit.domain.nocheck"; public static final String NETWORK_WHITELIST = "network.unit.access.whitelist"; public static final String NETWORK_BLACKLIST = "network.unit.access.blacklist"; - + public static final String NETWORK_SEARCHVERIFY = "network.unit.inspection.searchverify"; - + /** * appearance */ @@ -419,24 +420,24 @@ public final class SwitchboardConstants { public static final String GREETING_HOMEPAGE = "promoteSearchPageGreeting.homepage"; public static final String GREETING_LARGE_IMAGE = "promoteSearchPageGreeting.largeImage"; public static final String GREETING_SMALL_IMAGE = "promoteSearchPageGreeting.smallImage"; - + /** * browser pop up */ public static final String BROWSER_POP_UP_TRIGGER = "browserPopUpTrigger"; public static final String BROWSER_POP_UP_PAGE = "browserPopUpPage"; - + /** * forwarder of the index page */ public static final String INDEX_FORWARD = "indexForward"; - + public static final String UPNP_ENABLED = "upnp.enabled"; public static final String UPNP_REMOTEHOST = "upnp.remoteHost"; public static final String SEARCH_ITEMS = "search.items"; public static final String SEARCH_TARGET = "search.target"; - + /** * system tray */ @@ -444,7 +445,7 @@ public final class SwitchboardConstants { public static final String TRAY_ICON_FORCED = "trayIcon.force"; public static final String TRAY_LABEL = "tray.label"; public static final String BROWSERINTEGRATION = "browserintegration"; - + /** * Segments */ diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index fdcacde9b..8ca6572ac 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -2,7 +2,7 @@ * HTTPClient * Copyright 2010 by Sebastian Gaebel * First released 01.07.2010 at http://yacy.net - * + * * $LastChangedDate$ * $LastChangedRevision$ * $LastChangedBy$ @@ -11,12 +11,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -35,8 +35,8 @@ import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Set; import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; @@ -55,12 +55,16 @@ import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpVersion; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.params.HttpClientParams; +import org.apache.http.client.protocol.ClientContext; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.params.ConnRouteParams; import org.apache.http.conn.routing.HttpRoute; @@ -73,6 +77,7 @@ import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; import org.apache.http.impl.client.AbstractHttpClient; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.http.params.BasicHttpParams; @@ -87,7 +92,7 @@ import org.apache.http.util.EntityUtils; /** * HttpClient implementation which uses HttpComponents Client {@link http://hc.apache.org/} - * + * * @author sixcooler * */ @@ -96,6 +101,7 @@ public class HTTPClient { private final static int maxcon = 200; private static IdledConnectionEvictor idledConnectionEvictor = null; private static HttpClient httpClient = initConnectionManager(); + private static final CredentialsProvider credsProvider = new BasicCredentialsProvider(); private Set> headers = null; private HttpResponse httpResponse = null; private HttpUriRequest currentRequest = null; @@ -105,33 +111,33 @@ public class HTTPClient { private String host = null; private boolean redirecting = true; private String realm = null; - + public HTTPClient() { super(); } - + public HTTPClient(final String userAgent) { super(); this.userAgent = userAgent; } - + public HTTPClient(final String userAgent, final int timeout) { super(); this.userAgent = userAgent; this.timeout = timeout; } - + public static void setDefaultUserAgent(final String defaultAgent) { HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent); } - + public static HttpClient initConnectionManager() { // Create and initialize scheme registry final SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory())); - ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry); + final ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry); // Create and initialize HTTP parameters final HttpParams httpParams = new BasicHttpParams(); @@ -145,7 +151,7 @@ public class HTTPClient { // connections per host (2 default) clientConnectionManager.setDefaultMaxPerRoute(2); // Increase max connections for localhost - HttpHost localhost = new HttpHost("localhost"); + final HttpHost localhost = new HttpHost("localhost"); clientConnectionManager.setMaxForRoute(new HttpRoute(localhost), maxcon); /** * HTTP protocol settings @@ -170,7 +176,7 @@ public class HTTPClient { HttpConnectionParams.setTcpNoDelay(httpParams, false); // Defines whether the socket can be bound even though a previous connection is still in a timeout state. HttpConnectionParams.setSoReuseaddr(httpParams, true); - + httpClient = new DefaultHttpClient(clientConnectionManager, httpParams); // ask for gzip ((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor()); @@ -183,12 +189,12 @@ public class HTTPClient { } return httpClient; } - + /** * This method should be called just before shutdown * to stop the ConnectionManager and idledConnectionEvictor - * - * @throws InterruptedException + * + * @throws InterruptedException */ public static void closeConnectionManager() throws InterruptedException { if (idledConnectionEvictor != null) { @@ -200,79 +206,86 @@ public class HTTPClient { // Shut down the connection manager httpClient.getConnectionManager().shutdown(); } - + } - + + public static void setAuth(final String host, final int port, final String user, final String pw) { + final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw); + final AuthScope scope = new AuthScope(host, port); + credsProvider.setCredentials(scope, creds); + httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider); + } + /** * this method sets a host on which more than the default of 2 router per host are allowed - * + * * @param the host to be raised in 'route per host' */ public static void setMaxRouteHost(final String host) { - HttpHost mHost = new HttpHost(host); + final HttpHost mHost = new HttpHost(host); ((ThreadSafeClientConnManager) httpClient.getConnectionManager()).setMaxForRoute(new HttpRoute(mHost), 50); } - + /** * This method sets the Header used for the request - * + * * @param entrys to be set as request header */ public void setHeader(final Set> entrys) { this.headers = entrys; } - + /** * This method sets the timeout of the Connection and Socket - * + * * @param timeout in milliseconds */ public void setTimout(final int timeout) { this.timeout = timeout; } - + /** * This method sets the UserAgent to be used for the request - * + * * @param userAgent */ public void setUserAgent(final String userAgent) { this.userAgent = userAgent; } - + /** * This method sets the host to be called at the request - * + * * @param host */ public void setHost(final String host) { this.host = host; } - + /** * This method sets if requests should follow redirects - * + * * @param redirecting */ public void setRedirecting(final boolean redirecting) { this.redirecting = redirecting; } - + /** * This method sets the authorization realm for the request - * + * * @param realm */ public void setRealm(final String realm) { this.realm = realm; } - + /** * This method GETs a page from the server. - * + * * @param uri the url to get * @return content bytes - * @throws IOException + * @throws IOException */ public byte[] GETbytes(final String uri) throws IOException { return GETbytes(uri, Long.MAX_VALUE); @@ -280,64 +293,64 @@ public class HTTPClient { /** * This method GETs a page from the server. - * + * * @param uri the url to get * @return content bytes - * @throws IOException + * @throws IOException */ public byte[] GETbytes(final MultiProtocolURI url) throws IOException { return GETbytes(url, Long.MAX_VALUE); } - + /** * This method GETs a page from the server. - * + * * @param uri the url to get * @param maxBytes to get * @return content bytes - * @throws IOException + * @throws IOException */ - public byte[] GETbytes(final String uri, long maxBytes) throws IOException { + public byte[] GETbytes(final String uri, final long maxBytes) throws IOException { return GETbytes(new MultiProtocolURI(uri), maxBytes); } - + /** * This method GETs a page from the server. - * + * * @param uri the url to get * @param maxBytes to get * @return content bytes - * @throws IOException + * @throws IOException */ - public byte[] GETbytes(final MultiProtocolURI url, long maxBytes) throws IOException { - boolean localhost = url.getHost().equals("localhost"); - String urix = url.toNormalform(true, false, !localhost, false); + public byte[] GETbytes(final MultiProtocolURI url, final long maxBytes) throws IOException { + final boolean localhost = url.getHost().equals("localhost"); + final String urix = url.toNormalform(true, false, !localhost, false); final HttpGet httpGet = new HttpGet(urix); if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service return getContentBytes(httpGet, maxBytes); } - + /** * This method GETs a page from the server. * to be used for streaming out * Please take care to call finish()! - * + * * @param uri the url to get * @throws IOException */ public void GET(final String uri) throws IOException { - if (currentRequest != null) throw new IOException("Client is in use!"); + if (this.currentRequest != null) throw new IOException("Client is in use!"); final MultiProtocolURI url = new MultiProtocolURI(uri); final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false)); setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service - currentRequest = httpGet; + this.currentRequest = httpGet; execute(httpGet); } /** * This method gets HEAD response - * + * * @param uri the url to Response from * @return the HttpResponse * @throws IOException @@ -349,21 +362,21 @@ public class HTTPClient { execute(httpHead); finish(); ConnectionInfo.removeConnection(httpHead.hashCode()); - return httpResponse; + return this.httpResponse; } - + /** * This method POSTs a page from the server. * to be used for streaming out * Please take care to call finish()! - * + * * @param uri the url to post * @param instream the input to post * @param length the contentlength - * @throws IOException + * @throws IOException */ - public void POST(final String uri, final InputStream instream, long length) throws IOException { - if (currentRequest != null) throw new IOException("Client is in use!"); + public void POST(final String uri, final InputStream instream, final long length) throws IOException { + if (this.currentRequest != null) throw new IOException("Client is in use!"); final MultiProtocolURI url = new MultiProtocolURI(uri); final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false)); String host = url.getHost(); @@ -371,19 +384,19 @@ public class HTTPClient { setHost(host); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service final NonClosingInputStreamEntity inputStreamEntity = new NonClosingInputStreamEntity(instream, length); // statistics - upbytes = length; + this.upbytes = length; httpPost.setEntity(inputStreamEntity); - currentRequest = httpPost; + this.currentRequest = httpPost; execute(httpPost); } - + /** * send data to the server named by uri - * + * * @param uri the url to post * @param parts to post * @return content bytes - * @throws IOException + * @throws IOException */ public byte[] POSTbytes(final String uri, final Map parts, final boolean usegzip) throws IOException { final MultiProtocolURI url = new MultiProtocolURI(uri); @@ -392,7 +405,7 @@ public class HTTPClient { /** * send data to the server named by vhost - * + * * @param url address of the server * @param vhost name of the server at address which should respond * @param post data to send (name-value-pairs) @@ -402,7 +415,7 @@ public class HTTPClient { */ public byte[] POSTbytes(final MultiProtocolURI url, final String vhost, final Map post, final boolean usegzip) throws IOException { final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false)); - + setHost(vhost); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service if (vhost == null) setHost("127.0.0.1"); @@ -410,27 +423,27 @@ public class HTTPClient { for (final Entry part : post.entrySet()) multipartEntity.addPart(part.getKey(), part.getValue()); // statistics - upbytes = multipartEntity.getContentLength(); + this.upbytes = multipartEntity.getContentLength(); if (usegzip) { httpPost.setEntity(new GzipCompressingEntity(multipartEntity)); } else { httpPost.setEntity(multipartEntity); } - + return getContentBytes(httpPost, Long.MAX_VALUE); } - + /** * send stream-data to the server named by uri - * + * * @param uri the url to post * @param instream the stream to send * @param length the length of the stream * @return content bytes * @throws IOException */ - public byte[] POSTbytes(final String uri, final InputStream instream, long length) throws IOException { + public byte[] POSTbytes(final String uri, final InputStream instream, final long length) throws IOException { final MultiProtocolURI url = new MultiProtocolURI(uri); final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false)); String host = url.getHost(); @@ -439,27 +452,27 @@ public class HTTPClient { final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length); // statistics - upbytes = length; + this.upbytes = length; httpPost.setEntity(inputStreamEntity); return getContentBytes(httpPost, Long.MAX_VALUE); } - + /** - * + * * @return HttpResponse from call */ public HttpResponse getHttpResponse() { - return httpResponse; + return this.httpResponse; } - + /** - * + * * @return status code from http request */ public int getStatusCode() { - return httpResponse.getStatusLine().getStatusCode(); + return this.httpResponse.getStatusLine().getStatusCode(); } - + /** * This method gets direct access to the content-stream * Since this way is uncontrolled by the Client think of using 'writeTo' instead! @@ -469,20 +482,20 @@ public class HTTPClient { * @throws IOException */ public InputStream getContentstream() throws IOException { - if (httpResponse != null && currentRequest != null) { - final HttpEntity httpEntity = httpResponse.getEntity(); + if (this.httpResponse != null && this.currentRequest != null) { + final HttpEntity httpEntity = this.httpResponse.getEntity(); if (httpEntity != null) try { return httpEntity.getContent(); } catch (final IOException e) { - ConnectionInfo.removeConnection(currentRequest.hashCode()); - currentRequest.abort(); - currentRequest = null; + ConnectionInfo.removeConnection(this.currentRequest.hashCode()); + this.currentRequest.abort(); + this.currentRequest = null; throw e; } } return null; } - + /** * This method streams the content to the outputStream * Please take care to call finish()! @@ -491,24 +504,24 @@ public class HTTPClient { * @throws IOException */ public void writeTo(final OutputStream outputStream) throws IOException { - if (httpResponse != null && currentRequest != null) { - final HttpEntity httpEntity = httpResponse.getEntity(); + if (this.httpResponse != null && this.currentRequest != null) { + final HttpEntity httpEntity = this.httpResponse.getEntity(); if (httpEntity != null) try { httpEntity.writeTo(outputStream); outputStream.flush(); // Ensures that the entity content is fully consumed and the content stream, if exists, is closed. EntityUtils.consume(httpEntity); - ConnectionInfo.removeConnection(currentRequest.hashCode()); - currentRequest = null; + ConnectionInfo.removeConnection(this.currentRequest.hashCode()); + this.currentRequest = null; } catch (final IOException e) { - ConnectionInfo.removeConnection(currentRequest.hashCode()); - currentRequest.abort(); - currentRequest = null; + ConnectionInfo.removeConnection(this.currentRequest.hashCode()); + this.currentRequest.abort(); + this.currentRequest = null; throw e; } } } - + /** * This method ensures correct finish of client-connections * This method should be used after every use of GET or POST and writeTo or getContentstream! @@ -516,35 +529,35 @@ public class HTTPClient { * @throws IOException */ public void finish() throws IOException { - if (httpResponse != null) { - final HttpEntity httpEntity = httpResponse.getEntity(); + if (this.httpResponse != null) { + final HttpEntity httpEntity = this.httpResponse.getEntity(); if (httpEntity != null && httpEntity.isStreaming()) { // Ensures that the entity content is fully consumed and the content stream, if exists, is closed. EntityUtils.consume(httpEntity); } } - if (currentRequest != null) { - ConnectionInfo.removeConnection(currentRequest.hashCode()); - currentRequest.abort(); - currentRequest = null; + if (this.currentRequest != null) { + ConnectionInfo.removeConnection(this.currentRequest.hashCode()); + this.currentRequest.abort(); + this.currentRequest = null; } } - + private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final long maxBytes) throws IOException { byte[] content = null; try { execute(httpUriRequest); - if (httpResponse == null) return null; + if (this.httpResponse == null) return null; // get the response body - final HttpEntity httpEntity = httpResponse.getEntity(); + final HttpEntity httpEntity = this.httpResponse.getEntity(); if (httpEntity != null) { if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) { try { content = EntityUtils.toByteArray(httpEntity); - } catch (OutOfMemoryError e) { + } catch (final OutOfMemoryError e) { throw new IOException(e.toString()); } - } + } // Ensures that the entity content is fully consumed and the content stream, if exists, is closed. EntityUtils.consume(httpEntity); } @@ -556,7 +569,7 @@ public class HTTPClient { ConnectionInfo.removeConnection(httpUriRequest.hashCode()); return content; } - + private void execute(final HttpUriRequest httpUriRequest) throws IOException { final HttpContext httpContext = new BasicHttpContext(); setHeaders(httpUriRequest); @@ -566,8 +579,8 @@ public class HTTPClient { storeConnectionInfo(httpUriRequest); // execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF if (httpUriRequest instanceof HttpEntityEnclosingRequest) { - HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) httpUriRequest; - HttpEntity entity = hrequest.getEntity(); + final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) httpUriRequest; + final HttpEntity entity = hrequest.getEntity(); assert entity != null; //assert !entity.isChunked(); //assert entity.getContentLength() >= 0; @@ -575,45 +588,45 @@ public class HTTPClient { } try { - long time = System.currentTimeMillis(); - httpResponse = httpClient.execute(httpUriRequest, httpContext); - httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time)); - } catch (IOException e) { + final long time = System.currentTimeMillis(); + this.httpResponse = httpClient.execute(httpUriRequest, httpContext); + this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time)); + } catch (final IOException e) { ConnectionInfo.removeConnection(httpUriRequest.hashCode()); httpUriRequest.abort(); throw new IOException("Client can't execute: " + e.getMessage()); } } - + private void setHeaders(final HttpUriRequest httpUriRequest) { - if (headers != null) { - for (final Entry entry : headers) { + if (this.headers != null) { + for (final Entry entry : this.headers) { httpUriRequest.setHeader(entry.getKey(),entry.getValue()); } } - if (host != null) - httpUriRequest.setHeader(HTTP.TARGET_HOST, host); - if (realm != null) - httpUriRequest.setHeader("Authorization", "realm=" + realm); + if (this.host != null) + httpUriRequest.setHeader(HTTP.TARGET_HOST, this.host); + if (this.realm != null) + httpUriRequest.setHeader("Authorization", "realm=" + this.realm); } - + private void setParams(final HttpParams httpParams) { - HttpClientParams.setRedirecting(httpParams, redirecting); - HttpConnectionParams.setConnectionTimeout(httpParams, timeout); - HttpConnectionParams.setSoTimeout(httpParams, timeout); - if (userAgent != null) - HttpProtocolParams.setUserAgent(httpParams, userAgent); - if (host != null) - httpParams.setParameter(HTTP.TARGET_HOST, host); - } - + HttpClientParams.setRedirecting(httpParams, this.redirecting); + HttpConnectionParams.setConnectionTimeout(httpParams, this.timeout); + HttpConnectionParams.setSoTimeout(httpParams, this.timeout); + if (this.userAgent != null) + HttpProtocolParams.setUserAgent(httpParams, this.userAgent); + if (this.host != null) + httpParams.setParameter(HTTP.TARGET_HOST, this.host); + } + private void setProxy(final HttpParams httpParams) { if (ProxySettings.use) ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost()); // TODO find a better way for this ProxySettings.setProxyCreds((AbstractHttpClient) httpClient); } - + private void storeConnectionInfo(final HttpUriRequest httpUriRequest) { final int port = httpUriRequest.getURI().getPort(); final String thost = httpUriRequest.getURI().getHost(); @@ -624,16 +637,16 @@ public class HTTPClient { httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(), httpUriRequest.hashCode(), System.currentTimeMillis(), - upbytes)); + this.upbytes)); } - + private static SSLSocketFactory getSSLSocketFactory() { final TrustManager trustManager = new X509TrustManager() { - public void checkClientTrusted(X509Certificate[] chain, String authType) + public void checkClientTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { } - public void checkServerTrusted(X509Certificate[] chain, String authType) + public void checkServerTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { } @@ -645,10 +658,10 @@ public class HTTPClient { try { sslContext = SSLContext.getInstance("TLS"); sslContext.init(null, new TrustManager[] { trustManager }, null); - } catch (NoSuchAlgorithmException e) { + } catch (final NoSuchAlgorithmException e) { // should not happen // e.printStackTrace(); - } catch (KeyManagementException e) { + } catch (final KeyManagementException e) { // should not happen // e.printStackTrace(); } @@ -669,10 +682,10 @@ public class HTTPClient { try { newparts.put("foo", new StringBody("FooBar")); newparts.put("bar", new StringBody("BarFoo")); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { System.out.println(e.getStackTrace()); } - HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(); client.setUserAgent("foobar"); client.setRedirecting(false); // Get some @@ -683,7 +696,7 @@ public class HTTPClient { } try { System.out.println(UTF8.String(client.GETbytes(url))); - } catch (IOException e) { + } catch (final IOException e) { e.printStackTrace(); } } @@ -710,14 +723,14 @@ public class HTTPClient { // Close out connection manager try { HTTPClient.closeConnectionManager(); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { e.printStackTrace(); } } - - + + /** - * + * * @see: http://hc.apache.org/httpcomponents-client-4.0.1/tutorial/html/connmgmt.html#d4e638 * */ @@ -727,7 +740,7 @@ public class HTTPClient { private volatile boolean shutdown; - public IdledConnectionEvictor(ClientConnectionManager clientConnectionManager) { + public IdledConnectionEvictor(final ClientConnectionManager clientConnectionManager) { super(); this.clientConnectionManager = clientConnectionManager; } @@ -735,29 +748,29 @@ public class HTTPClient { @Override public void run() { try { - while (!shutdown) { + while (!this.shutdown) { synchronized (this) { wait(5000); // Close expired connections - clientConnectionManager.closeExpiredConnections(); + this.clientConnectionManager.closeExpiredConnections(); // Optionally, close connections // that have been idle longer than 5 sec // (some SOHO router act strange on >5sec idled connections) - clientConnectionManager.closeIdleConnections(5, TimeUnit.SECONDS); + this.clientConnectionManager.closeIdleConnections(5, TimeUnit.SECONDS); } } - } catch (InterruptedException ex) { + } catch (final InterruptedException ex) { // terminate } } public void shutdown() { - shutdown = true; + this.shutdown = true; synchronized (this) { notifyAll(); } } } - + } diff --git a/source/net/yacy/cora/services/federated/solr/SolrChardingConnector.java b/source/net/yacy/cora/services/federated/solr/SolrChardingConnector.java index d4287a2a0..3f4b31f8e 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrChardingConnector.java +++ b/source/net/yacy/cora/services/federated/solr/SolrChardingConnector.java @@ -32,6 +32,7 @@ import java.util.List; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.document.Document; import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.logging.Log; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -46,6 +47,7 @@ public class SolrChardingConnector { private final String[] urls; public SolrChardingConnector(final String urlList, final SolrScheme scheme, final SolrChardingSelection.Method method) throws IOException { + urlList.replace(' ', ','); this.urls = urlList.split(","); this.connectors = new ArrayList(); for (final String u: this.urls) { @@ -156,17 +158,22 @@ public class SolrChardingConnector { return list; } - public long[] getSizeList() throws IOException { + public long[] getSizeList() { final long[] size = new long[this.connectors.size()]; int i = 0; for (final SolrSingleConnector connector: this.connectors) { - final SolrDocumentList list = connector.get("*:*", 0, 1); - size[i++] = list.getNumFound(); + try { + final SolrDocumentList list = connector.get("*:*", 0, 1); + size[i++] = list.getNumFound(); + } catch (final IOException e) { + Log.logException(e); + size[i++] = 0; + } } return size; } - public long getSize() throws IOException { + public long getSize() { final long[] size = getSizeList(); long s = 0; for (final long l: size) s += l; diff --git a/source/net/yacy/cora/services/federated/solr/SolrHTTPClient.java b/source/net/yacy/cora/services/federated/solr/SolrHTTPClient.java index 5f61ecf1d..e3b78039e 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrHTTPClient.java +++ b/source/net/yacy/cora/services/federated/solr/SolrHTTPClient.java @@ -34,6 +34,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.http.HTTPClient; import org.apache.http.entity.mime.content.ContentBody; @@ -44,9 +45,9 @@ import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.BinaryResponseParser; -import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CommonParams; @@ -57,9 +58,9 @@ import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.NamedList; /** - * The {@link SolrHTTPClient} uses the Apache Commons HTTP Client to connect to solr. + * The {@link SolrHTTPClient} uses the Apache Commons HTTP Client to connect to solr. * 
SolrServer server = new CommonsHttpSolrServer( url );
- * + * * @version $Id: CommonsHttpSolrServer.java 1067552 2011-02-05 23:52:42Z koji $ * @since solr 1.3 */ @@ -71,7 +72,7 @@ public class SolrHTTPClient extends SolrServer { * org.apache.commons.httpclient.HttpClient HttpClient} to the Solr * server from the client. */ - public static final String AGENT = "Solr["+SolrHTTPClient.class.getName()+"] 1.0"; + public static final String AGENT = "Solr["+SolrHTTPClient.class.getName()+"] 1.0"; public final static Charset utf8; static { @@ -81,15 +82,16 @@ public class SolrHTTPClient extends SolrServer { /** * The URL of the Solr server. */ - protected String _baseURL; - + protected String _baseURL, host, solraccount, solrpw; + protected int port; + /** * Default value: null / empty.

- * Parameters that are added to every request regardless. This may be a place to add + * Parameters that are added to every request regardless. This may be a place to add * something like an authentication token. */ protected ModifiableSolrParams _invariantParams; - + /** * Default response parser is BinaryResponseParser

* This parser represents the default Response Parser chosen to @@ -104,14 +106,14 @@ public class SolrHTTPClient extends SolrServer { * @see org.apache.solr.client.solrj.request.RequestWriter */ protected RequestWriter requestWriter = new RequestWriter(); - - /** - * @param solrServerUrl The URL of the Solr server. For + + /** + * @param solrServerUrl The URL of the Solr server. For * example, "http://localhost:8983/solr/" - * if you are using the standard distribution Solr webapp + * if you are using the standard distribution Solr webapp * on your local machine. */ - public SolrHTTPClient(String solrServerUrl) throws MalformedURLException { + public SolrHTTPClient(final String solrServerUrl) throws MalformedURLException { this(new URL(solrServerUrl)); } @@ -120,7 +122,7 @@ public class SolrHTTPClient extends SolrServer { * "http://localhost:8983/solr/" if you are using the * standard distribution Solr webapp on your local machine. */ - public SolrHTTPClient(URL baseURL) + public SolrHTTPClient(final URL baseURL) { this(baseURL, new BinaryResponseParser()); } @@ -129,17 +131,39 @@ public class SolrHTTPClient extends SolrServer { * @see #useMultiPartPost * @see #_parser */ - public SolrHTTPClient(URL baseURL, ResponseParser parser) { - _baseURL = baseURL.toExternalForm(); - if( _baseURL.endsWith( "/" ) ) { - _baseURL = _baseURL.substring( 0, _baseURL.length()-1 ); + public SolrHTTPClient(final URL baseURL, final ResponseParser parser) { + this._baseURL = baseURL.toExternalForm(); + if( this._baseURL.endsWith( "/" ) ) { + this._baseURL = this._baseURL.substring( 0, this._baseURL.length()-1 ); } - if( _baseURL.indexOf( '?' ) >=0 ) { - throw new RuntimeException( "Invalid base url for solrj. The base URL must not contain parameters: "+_baseURL ); + if( this._baseURL.indexOf( '?' ) >=0 ) { + throw new RuntimeException( "Invalid base url for solrj. The base URL must not contain parameters: "+this._baseURL ); } - _parser = parser; + + MultiProtocolURI u; + try { + u = new MultiProtocolURI(this._baseURL.toString()); + this.host = u.getHost(); + this.port = u.getPort(); + final String userinfo = u.getUserInfo(); + if (userinfo == null || userinfo.length() == 0) { + this.solraccount = ""; this.solrpw = ""; + } else { + final int p = userinfo.indexOf(':'); + if (p < 0) { + this.solraccount = userinfo; this.solrpw = ""; + } else { + this.solraccount = userinfo.substring(0, p); this.solrpw = userinfo.substring(p + 1); + } + } + } catch (final MalformedURLException e) { + this.solraccount = ""; this.solrpw = ""; + this.host = ""; this.port = -1; + } + + this._parser = parser; } - + //------------------------------------------------------------------------ //------------------------------------------------------------------------ @@ -158,26 +182,26 @@ public class SolrHTTPClient extends SolrServer { { ResponseParser responseParser = request.getResponseParser(); if (responseParser == null) { - responseParser = _parser; + responseParser = this._parser; } return request(request, responseParser); } - - public NamedList request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException { + + public NamedList request(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException { SolrParams params = request.getParams(); - Collection streams = requestWriter.getContentStreams(request); - String path = requestWriter.getPath(request); + final Collection streams = this.requestWriter.getContentStreams(request); + String path = this.requestWriter.getPath(request); if( path == null || !path.startsWith( "/" ) ) { path = "/select"; } - + // The parser 'wt=' and 'version=' params are used instead of the original params ResponseParser parser = request.getResponseParser(); if( parser == null ) { - parser = _parser; + parser = this._parser; } - ModifiableSolrParams wparams = new ModifiableSolrParams(); + final ModifiableSolrParams wparams = new ModifiableSolrParams(); wparams.set( CommonParams.WT, parser.getWriterType() ); wparams.set( CommonParams.VERSION, parser.getVersion()); if( params == null ) { @@ -186,26 +210,30 @@ public class SolrHTTPClient extends SolrServer { else { params = new DefaultSolrParams( wparams, params ); } - - if( _invariantParams != null ) { - params = new DefaultSolrParams( _invariantParams, params ); + + if( this._invariantParams != null ) { + params = new DefaultSolrParams( this._invariantParams, params ); } - - + + byte[] result = null; - HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(); + if (this.solraccount.length() > 0 && this.solrpw.length() > 0 && this.host.length() > 0) { + HTTPClient.setAuth(this.host, this.port, this.solraccount, this.solrpw); + } + if (SolrRequest.METHOD.POST == request.getMethod()) { - boolean isMultipart = ( streams != null && streams.size() > 1 ); + final boolean isMultipart = ( streams != null && streams.size() > 1 ); if (streams == null || isMultipart) { - String url = _baseURL + path; - - HashMap parts = new HashMap(); - Iterator iter = params.getParameterNamesIterator(); + String url = this._baseURL + path; + + final HashMap parts = new HashMap(); + final Iterator iter = params.getParameterNamesIterator(); while (iter.hasNext()) { - String p = iter.next(); - String[] vals = params.getParams(p); + final String p = iter.next(); + final String[] vals = params.getParams(p); if (vals != null) { - for (String v : vals) { + for (final String v : vals) { if (isMultipart) { parts.put(p, new StringBody(v, utf8)); } else { @@ -214,13 +242,13 @@ public class SolrHTTPClient extends SolrServer { } } } - + if (isMultipart) { - for (ContentStream content : streams) { + for (final ContentStream content : streams) { parts.put(content.getName(), new InputStreamBody(content.getStream(), content.getContentType(), null)); } } - + try { result = client.POSTbytes(url, parts, true); } finally { @@ -228,32 +256,32 @@ public class SolrHTTPClient extends SolrServer { } } else { // It has one stream, this is the post body, put the params in the URL - String pstr = ClientUtils.toQueryString(params, false); - String url = _baseURL + path + pstr; - + final String pstr = ClientUtils.toQueryString(params, false); + final String url = this._baseURL + path + pstr; + // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; - for (ContentStream content : streams) { + for (final ContentStream content : streams) { contentStream[0] = content; break; } result = client.POSTbytes(url, contentStream[0].getStream(), contentStream[0].getStream().available()); } } else if (SolrRequest.METHOD.GET == request.getMethod()) { - result = client.GETbytes( _baseURL + path + ClientUtils.toQueryString( params, false )); + result = client.GETbytes( this._baseURL + path + ClientUtils.toQueryString( params, false )); } else { throw new SolrServerException("Unsupported method: "+request.getMethod() ); } - int statusCode = client.getStatusCode(); + final int statusCode = client.getStatusCode(); if (statusCode != 200) { - throw new IOException("bad status code: " + statusCode + ", " + client.getHttpResponse().getStatusLine()); + throw new IOException("bad status code: " + statusCode + ", " + client.getHttpResponse().getStatusLine() + ", url = " + this._baseURL + path); } // Read the contents //System.out.println("SOLR RESPONSE: " + UTF8.String(result)); - InputStream respBody = new ByteArrayInputStream(result); + final InputStream respBody = new ByteArrayInputStream(result); return processor.processResponse(respBody, "UTF-8"); } @@ -268,12 +296,12 @@ public class SolrHTTPClient extends SolrServer { if( path == null || !path.startsWith( "/" ) ) { path = "/select"; } - + ResponseParser parser = request.getResponseParser(); if( parser == null ) { parser = _parser; } - + // The parser 'wt=' and 'version=' params are used instead of the original params ModifiableSolrParams wparams = new ModifiableSolrParams(); wparams.set( CommonParams.WT, parser.getWriterType() ); @@ -284,7 +312,7 @@ public class SolrHTTPClient extends SolrServer { else { params = new DefaultSolrParams( wparams, params ); } - + if( _invariantParams != null ) { params = new DefaultSolrParams( _invariantParams, params ); } @@ -505,30 +533,30 @@ public class SolrHTTPClient extends SolrServer { } */ - + //------------------------------------------------------------------- //------------------------------------------------------------------- - + /** * Retrieve the default list of parameters are added to every request regardless. - * + * * @see #_invariantParams */ public ModifiableSolrParams getInvariantParams() { - return _invariantParams; + return this._invariantParams; } public String getBaseURL() { - return _baseURL; + return this._baseURL; } - public void setBaseURL(String baseURL) { + public void setBaseURL(final String baseURL) { this._baseURL = baseURL; } public ResponseParser getParser() { - return _parser; + return this._parser; } /** @@ -536,12 +564,12 @@ public class SolrHTTPClient extends SolrServer { * @param processor Default Response Parser chosen to parse the response if the parser were not specified as part of the request. * @see org.apache.solr.client.solrj.SolrRequest#getResponseParser() */ - public void setParser(ResponseParser processor) { - _parser = processor; + public void setParser(final ResponseParser processor) { + this._parser = processor; } - public void setRequestWriter(RequestWriter requestWriter) { + public void setRequestWriter(final RequestWriter requestWriter) { this.requestWriter = requestWriter; } @@ -552,10 +580,10 @@ public class SolrHTTPClient extends SolrServer { * * @return the response from the SolrServer */ - public UpdateResponse add(Iterator docIterator) + public UpdateResponse add(final Iterator docIterator) throws SolrServerException, IOException { - UpdateRequest req = new UpdateRequest(); - req.setDocIterator(docIterator); + final UpdateRequest req = new UpdateRequest(); + req.setDocIterator(docIterator); return req.process(this); } @@ -568,7 +596,7 @@ public class SolrHTTPClient extends SolrServer { */ public UpdateResponse addBeans(final Iterator beanIterator) throws SolrServerException, IOException { - UpdateRequest req = new UpdateRequest(); + final UpdateRequest req = new UpdateRequest(); req.setDocIterator(new Iterator() { public boolean hasNext() { @@ -576,7 +604,7 @@ public class SolrHTTPClient extends SolrServer { } public SolrInputDocument next() { - Object o = beanIterator.next(); + final Object o = beanIterator.next(); if (o == null) return null; return getBinder().toSolrInputDocument(o); }