- replaced solrj 3.1 with solrj 3.3

- updated also slf4j
- added authentication for solrj


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7829 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent e7c7598923
commit 768c59740c

@ -39,7 +39,7 @@
<string>$JAVAROOT/htroot</string>
<string>$JAVAROOT/lib/activation.jar</string>
<string>$JAVAROOT/lib/apache-mime4j-0.6.jar</string>
<string>$JAVAROOT/lib/apache-solr-solrj-3.1.0.jar</string>
<string>$JAVAROOT/lib/apache-solr-solrj-3.3.0.jar</string>
<string>$JAVAROOT/lib/bcmail-jdk15-145.jar</string>
<string>$JAVAROOT/lib/bcprov-jdk15-145.jar</string>
<string>$JAVAROOT/lib/bzip2.jar</string>
@ -53,11 +53,12 @@
<string>$JAVAROOT/lib/httpcore-4.1.1.jar</string>
<string>$JAVAROOT/lib/httpmime-4.1.1.jar</string>
<string>$JAVAROOT/lib/icu4j-core.jar</string>
<string>$JAVAROOT/lib/jempbox-1.6.0.jar</string>
<string>$JAVAROOT/lib/J7Zip-modified.jar</string>
<string>$JAVAROOT/lib/jakarta-oro-2.0.8.jar</string>
<string>$JAVAROOT/lib/jcifs-1.3.15.jar</string>
<string>$JAVAROOT/lib/jempbox-1.6.0.jar</string>
<string>$JAVAROOT/lib/jsch-0.1.42.jar</string>
<string>$JAVAROOT/lib/json-simple-1.1.jar</string>
<string>$JAVAROOT/lib/log4j-1.2.16.jar</string>
<string>$JAVAROOT/lib/metadata-extractor-2.4.0-beta-1.jar</string>
<string>$JAVAROOT/lib/mysql-connector-java-5.1.12-bin.jar</string>
@ -65,9 +66,8 @@
<string>$JAVAROOT/lib/poi-3.6-20091214.jar</string>
<string>$JAVAROOT/lib/poi-scratchpad-3.6-20091214.jar</string>
<string>$JAVAROOT/lib/servlet-api.jar</string>
<string>$JAVAROOT/lib/json-simple-1.1.jar</string>
<string>$JAVAROOT/lib/slf4j-api-1.5.5.jar</string>
<string>$JAVAROOT/lib/slf4j-jdk14-1.5.5.jar</string>
<string>$JAVAROOT/lib/slf4j-api-1.6.1.jar</string>
<string>$JAVAROOT/lib/slf4j-jdk14-1.6.1.jar</string>
<string>$JAVAROOT/lib/webcat-0.1-swf.jar</string>
<string>$JAVAROOT/lib/xercesImpl.jar</string>
<string>$JAVAROOT/lib/xml-apis.jar</string>

@ -181,7 +181,7 @@
<pathelement location="${htroot}" />
<pathelement location="${lib}/activation.jar" />
<pathelement location="${lib}/apache-mime4j-0.6.jar" />
<pathelement location="${lib}/apache-solr-solrj-3.1.0.jar" />
<pathelement location="${lib}/apache-solr-solrj-3.3.0.jar" />
<pathelement location="${lib}/bcmail-jdk15-145.jar" />
<pathelement location="${lib}/bcprov-jdk15-145.jar" />
<pathelement location="${lib}/bzip2.jar" />
@ -195,10 +195,10 @@
<pathelement location="${lib}/httpcore-4.1.1.jar" />
<pathelement location="${lib}/httpmime-4.1.1.jar" />
<pathelement location="${lib}/icu4j-core.jar" />
<pathelement location="${lib}/jempbox-1.6.0.jar" />
<pathelement location="${lib}/J7Zip-modified.jar" />
<pathelement location="${lib}/jakarta-oro-2.0.8.jar" />
<pathelement location="${lib}/jcifs-1.3.15.jar" />
<pathelement location="${lib}/jempbox-1.6.0" />
<pathelement location="${lib}/jsch-0.1.42.jar" />
<pathelement location="${lib}/json-simple-1.1.jar" />
<pathelement location="${lib}/log4j-1.2.16.jar" />
@ -208,8 +208,8 @@
<pathelement location="${lib}/poi-3.6-20091214.jar" />
<pathelement location="${lib}/poi-scratchpad-3.6-20091214.jar" />
<pathelement location="${lib}/servlet-api.jar" />
<pathelement location="${lib}/slf4j-api-1.5.5.jar" />
<pathelement location="${lib}/slf4j-jdk14-1.5.5.jar" />
<pathelement location="${lib}/slf4j-api-1.6.1.jar" />
<pathelement location="${lib}/slf4j-jdk14-1.6.1.jar" />
<pathelement location="${lib}/webcat-0.1-swf.jar" />
<pathelement location="${lib}/xercesImpl.jar" />
<pathelement location="${lib}/xml-apis.jar" />

@ -275,7 +275,7 @@ minimumGlobalDelta = 500
# the following mime-types are a blacklist for indexing:
# parser.mime.deny: specifies mime-types that shall not be indexed
parser.mime.deny=
parser.extensions.deny=
parser.extensions.deny=pdf
# Promotion Strings
# These strings appear in the Web Mask of the YACY search client

@ -22,10 +22,14 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Iterator;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.solr.SolrChardingConnector;
import net.yacy.cora.services.federated.solr.SolrChardingSelection;
@ -51,12 +55,25 @@ public class IndexFederated_p {
final boolean solrWasOn = env.getConfigBool("federated.service.solr.indexing.enabled", true);
final boolean solrIsOnAfterwards = post.getBoolean("solr.indexing.enabled", false);
env.setConfig("federated.service.solr.indexing.enabled", solrIsOnAfterwards);
env.setConfig("federated.service.solr.indexing.url", post.get("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr")));
String solrurls = post.get("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr"));
final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));
final StringBuilder s = new StringBuilder();
String s0;
try {
while ((s0 = r.readLine()) != null) {
s0 = s0.trim();
if (s0.length() > 0) s.append(s0).append(',');
}
} catch (final IOException e1) {
}
if (s.length() > 0) s.setLength(s.length() - 1);
solrurls = s.toString().trim();
env.setConfig("federated.service.solr.indexing.url", solrurls);
env.setConfig("federated.service.solr.indexing.charding", post.get("solr.indexing.charding", env.getConfig("federated.service.solr.indexing.charding", "modulo-host-md5")));
final String schemename = post.get("solr.indexing.schemefile", env.getConfig("federated.service.solr.indexing.schemefile", "solr.keys.default.list"));
env.setConfig("federated.service.solr.indexing.schemefile", schemename);
if (solrWasOn && !solrIsOnAfterwards) {
if (solrWasOn) {
// switch off
sb.solrConnector.close();
sb.solrConnector = null;
@ -64,9 +81,8 @@ public class IndexFederated_p {
final SolrScheme scheme = new SolrScheme(new File(env.getDataPath(), "DATA/SETTINGS/" + schemename));
if (!solrWasOn && solrIsOnAfterwards) {
if (solrIsOnAfterwards) {
// switch on
final String solrurls = sb.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr");
final boolean usesolr = sb.getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0;
try {
sb.solrConnector = (usesolr) ? new SolrChardingConnector(solrurls, scheme, SolrChardingSelection.Method.MODULO_HOST_MD5) : null;
@ -98,21 +114,15 @@ public class IndexFederated_p {
prop.put("table", 0);
} else {
prop.put("table", 1);
try {
final long[] size = sb.solrConnector.getSizeList();
final String[] urls = sb.solrConnector.getAdminInterfaceList();
boolean dark = false;
for (int i = 0; i < size.length; i++) {
prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark;
prop.put("table_list_" + i + "_url", urls[i]);
prop.put("table_list_" + i + "_size", size[i]);
}
prop.put("table_list", size.length);
} catch (final IOException e) {
Log.logException(e);
prop.put("table", 0);
final long[] size = sb.solrConnector.getSizeList();
final String[] urls = sb.solrConnector.getAdminInterfaceList();
boolean dark = false;
for (int i = 0; i < size.length; i++) {
prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark;
prop.put("table_list_" + i + "_url", urls[i]);
prop.put("table_list_" + i + "_size", size[i]);
}
prop.put("table_list", size.length);
}
// write scheme
@ -138,7 +148,7 @@ public class IndexFederated_p {
// fill attribute fields
prop.put("yacy.indexing.enabled.checked", env.getConfigBool("federated.service.yacy.indexing.enabled", true) ? 1 : 0);
prop.put("solr.indexing.enabled.checked", env.getConfigBool("federated.service.solr.indexing.enabled", false) ? 1 : 0);
prop.put("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr"));
prop.put("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr").replace(",", "\n"));
prop.put("solr.indexing.charding", env.getConfig("federated.service.solr.indexing.charding", "modulo-host-md5"));
prop.put("solr.indexing.schemefile", schemename);

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

@ -9,7 +9,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -42,7 +42,7 @@ public final class SwitchboardConstants {
* by first encoding <code>username:password</code> as Base64 and hashing it using {@link MapTools#encodeMD5Hex(String)}.</p>
*/
public static final String ADMIN_ACCOUNT_B64MD5 = "adminAccountBase64MD5";
public static final int CRAWLJOB_SYNC = 0;
public static final int CRAWLJOB_STATUS = 1;
// 20_dhtdistribution
@ -86,7 +86,7 @@ public final class SwitchboardConstants {
* <p><code>public static final String <strong>CRAWLJOB_LOCAL_CRAWL</strong> = "50_localcrawl"</code></p>
* <p>Name of the local crawler thread, popping one entry off the Local Crawl Queue, and passing it to the
* proxy cache enqueue thread to download and further process it</p>
*
*
* @see Switchboard#PROXY_CACHE_ENQUEUE
*/
public static final String CRAWLJOB_LOCAL_CRAWL = "50_localcrawl";
@ -99,7 +99,7 @@ public final class SwitchboardConstants {
/**
* <p><code>public static final String <strong>CRAWLJOB_REMOTE_CRAWL_LOADER</strong> = "60_remotecrawlloader"</code></p>
* <p>Name of the remote crawl list loading thread</p>
*
*
* @see Switchboard#CRAWLJOB_REMOTE_CRAWL_LOADER
*/
public static final String CRAWLJOB_REMOTE_CRAWL_LOADER = "60_remotecrawlloader";
@ -134,7 +134,7 @@ public final class SwitchboardConstants {
// 90_cleanup
/**
* <p><code>public static final String <strong>CLEANUP</strong> = "90_cleanup"</code></p>
* <p>The cleanup thread which is responsible for pendant cleanup-jobs, news/ranking distribution, etc.</p>
* <p>The cleanup thread which is responsible for pendant cleanup-jobs, news/ranking distribution, etc.</p>
*/
public static final String CLEANUP = "90_cleanup";
public static final String CLEANUP_METHOD_START = "cleanupJob";
@ -226,7 +226,7 @@ public final class SwitchboardConstants {
/**
* <p><code>public static final String <strong>INDEX_DIST_ALLOW</strong> = "allowDistributeIndex"</code></p>
* <p>Name of the setting whether Index Distribution shall be allowed (and the DHT-thread therefore started) or not</p>
*
*
* @see Switchboard#INDEX_DIST_ALLOW_WHILE_CRAWLING
*/
public static final String INDEX_DIST_ALLOW = "allowDistributeIndex";
@ -236,7 +236,7 @@ public final class SwitchboardConstants {
* <p>Name of the setting whether Index Distribution shall be allowed while crawling is in progress, i.e.
* the Local Crawler Queue is filled.</p>
* <p>This setting only has effect if {@link #INDEX_DIST_ALLOW} is enabled</p>
*
*
* @see Switchboard#INDEX_DIST_ALLOW
*/
public static final String INDEX_DIST_ALLOW_WHILE_CRAWLING = "allowDistributeIndexWhileCrawling";
@ -244,9 +244,10 @@ public final class SwitchboardConstants {
public static final String INDEX_TRANSFER_TIMEOUT = "indexTransfer.timeout";
public static final String INDEX_TRANSFER_GZIP_BODY = "indexTransfer.gzipBody";
public static final String PARSER_MIME_DENY = "parser.mime.deny";
public static final String PARSER_EXTENSIONS_DENY = "parser.extensions.deny";
/**
* <p><code>public static final String <strong>PROXY_ONLINE_CAUTION_DELAY</strong> = "onlineCautionDelay"</code></p>
* <p>Name of the setting how long indexing should pause after the last time the proxy was used in milliseconds</p>
* <p>Name of the setting how long indexing should pause after the last time the proxy was used in milliseconds</p>
*/
public static final String PROXY_ONLINE_CAUTION_DELAY = "crawlPause.proxy";
public static final String LOCALSEACH_ONLINE_CAUTION_DELAY = "crawlPause.localsearch";
@ -265,16 +266,16 @@ public final class SwitchboardConstants {
* <p><code>public static final String <strong>PROXY_CACHE_LAYOUT</strong> = "proxyCacheLayout"</code></p>
* <p>Name of the setting which file-/folder-layout the proxy cache shall use. Possible values are {@link #PROXY_CACHE_LAYOUT_TREE}
* and {@link #PROXY_CACHE_LAYOUT_HASH}</p>
*
*
* @see Switchboard#PROXY_CACHE_LAYOUT_TREE
* @see Switchboard#PROXY_CACHE_LAYOUT_HASH
*/
public static final String PROXY_YACY_ONLY = "proxyYacyOnly";
//////////////////////////////////////////////////////////////////////////////////////////////
// Cluster settings
//////////////////////////////////////////////////////////////////////////////////////////////
public static final String CLUSTER_MODE = "cluster.mode";
public static final String CLUSTER_MODE_PUBLIC_CLUSTER = "publiccluster";
public static final String CLUSTER_MODE_PRIVATE_CLUSTER = "privatecluster";
@ -289,14 +290,14 @@ public final class SwitchboardConstants {
public static final String REMOTESEARCH_MAXCOUNT_USER = "remotesearch.maxcount";
public static final String REMOTESEARCH_MAXTIME_USER = "remotesearch.maxtime";
/**
* <p><code>public static final String <strong>CRAWLER_THREADS_ACTIVE_MAX</strong> = "crawler.MaxActiveThreads"</code></p>
* <p>Name of the setting how many active crawler-threads may maximal be running on the same time</p>
*/
public static final String CRAWLER_THREADS_ACTIVE_MAX = "crawler.MaxActiveThreads";
public static final String YACY_MODE_DEBUG = "yacyDebugMode";
/**
* <p><code>public static final String <strong>WORDCACHE_MAX_COUNT</strong> = "wordCacheMaxCount"</code></p>
* <p>Name of the setting how many words the word-cache (or DHT-Out cache) shall contain maximal. Indexing pages if the
@ -306,39 +307,39 @@ public final class SwitchboardConstants {
public static final String HTTPC_NAME_CACHE_CACHING_PATTERNS_NO = "httpc.nameCacheNoCachingPatterns";
public static final String ROBOTS_TXT = "httpd.robots.txt";
public static final String ROBOTS_TXT_DEFAULT = RobotsTxtConfig.LOCKED + "," + RobotsTxtConfig.DIRS;
/**
* <p><code>public static final String <strong>BLACKLIST_CLASS_DEFAULT</strong> = "de.anomic.plasma.urlPattern.defaultURLPattern"</code></p>
* <p>Package and name of YaCy's {@link DefaultBlacklist default} blacklist implementation</p>
*
*
* @see DefaultBlacklist for a detailed overview about the syntax of the default implementation
*/
public static final String LIST_BLUE = "plasmaBlueList";
public static final String LIST_BLUE_DEFAULT = null;
public static final String LIST_BADWORDS_DEFAULT = "yacy.badwords";
public static final String LIST_STOPWORDS_DEFAULT = "yacy.stopwords";
/**
* <p><code>public static final String <strong>HTCACHE_PATH</strong> = "proxyCache"</code></p>
* <p>Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all
* downloaded webpages and their respective ressources and HTTP-headers are stored. It is the location containing
* the proxy-cache</p>
*
*
* @see Switchboard#PROXY_CACHE_LAYOUT for details on the file-layout in this path
*/
public static final String HTCACHE_PATH = "proxyCache";
public static final String HTCACHE_PATH_DEFAULT = "DATA/HTCACHE";
public static final String RELEASE_PATH = "releases";
public static final String RELEASE_PATH_DEFAULT = "DATA/RELEASE";
public static final String SURROGATES_IN_PATH = "surrogates.in";
public static final String SURROGATES_IN_PATH_DEFAULT = "DATA/SURROGATES/in";
public static final String SURROGATES_OUT_PATH = "surrogates.out";
public static final String SURROGATES_OUT_PATH_DEFAULT = "DATA/SURROGATES/out";
public static final String DICTIONARY_SOURCE_PATH = "dictionaries";
public static final String DICTIONARY_SOURCE_PATH_DEFAULT = "DATA/DICTIONARIES";
/**
* <p><code>public static final String <strong>HTDOCS_PATH</strong> = "htDocsPath"</code></p>
* <p>Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all
@ -350,7 +351,7 @@ public final class SwitchboardConstants {
/**
* <p><code>public static final String <strong>HTROOT_PATH</strong> = "htRootPath"</code></p>
* <p>Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all
* original servlets, their stylesheets, scripts, etc. lie. It is also home of the XML-interface to YaCy</p>
* original servlets, their stylesheets, scripts, etc. lie. It is also home of the XML-interface to YaCy</p>
*/
public static final String HTROOT_PATH = "htRootPath";
public static final String HTROOT_PATH_DEFAULT = "htroot";
@ -372,7 +373,7 @@ public final class SwitchboardConstants {
* <p><code>public static final String <strong>WORK_PATH</strong> = "wordPath"</code></p>
* <p>Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all
* DBs containing "work" of the user are saved. Such include bookmarks, messages, wiki, blog</p>
*
*
* @see Switchboard#DBFILE_BLOG
* @see Switchboard#DBFILE_BOOKMARKS
* @see Switchboard#DBFILE_BOOKMARKS_DATES
@ -384,16 +385,16 @@ public final class SwitchboardConstants {
public static final String WORK_PATH = "workPath";
public static final String WORK_PATH_DEFAULT = "DATA/WORK";
/**
* ResourceObserver
*/
public static final String DISK_FREE = "disk.free";
public static final String DISK_FREE_HARDLIMIT = "disk.free.hardlimit";
public static final String MEMORY_ACCEPTDHT = "memory.acceptDHTabove";
public static final String INDEX_RECEIVE_AUTODISABLED = "memory.disabledDHT";
/*
* Some constants
*/
@ -401,16 +402,16 @@ public final class SwitchboardConstants {
/**
* network properties
*
*
*/
public static final String NETWORK_NAME = "network.unit.name";
public static final String NETWORK_DOMAIN = "network.unit.domain";
public static final String NETWORK_DOMAIN_NOCHECK = "network.unit.domain.nocheck";
public static final String NETWORK_WHITELIST = "network.unit.access.whitelist";
public static final String NETWORK_BLACKLIST = "network.unit.access.blacklist";
public static final String NETWORK_SEARCHVERIFY = "network.unit.inspection.searchverify";
/**
* appearance
*/
@ -419,24 +420,24 @@ public final class SwitchboardConstants {
public static final String GREETING_HOMEPAGE = "promoteSearchPageGreeting.homepage";
public static final String GREETING_LARGE_IMAGE = "promoteSearchPageGreeting.largeImage";
public static final String GREETING_SMALL_IMAGE = "promoteSearchPageGreeting.smallImage";
/**
* browser pop up
*/
public static final String BROWSER_POP_UP_TRIGGER = "browserPopUpTrigger";
public static final String BROWSER_POP_UP_PAGE = "browserPopUpPage";
/**
* forwarder of the index page
*/
public static final String INDEX_FORWARD = "indexForward";
public static final String UPNP_ENABLED = "upnp.enabled";
public static final String UPNP_REMOTEHOST = "upnp.remoteHost";
public static final String SEARCH_ITEMS = "search.items";
public static final String SEARCH_TARGET = "search.target";
/**
* system tray
*/
@ -444,7 +445,7 @@ public final class SwitchboardConstants {
public static final String TRAY_ICON_FORCED = "trayIcon.force";
public static final String TRAY_LABEL = "tray.label";
public static final String BROWSERINTEGRATION = "browserintegration";
/**
* Segments
*/

@ -2,7 +2,7 @@
* HTTPClient
* Copyright 2010 by Sebastian Gaebel
* First released 01.07.2010 at http://yacy.net
*
*
* $LastChangedDate$
* $LastChangedRevision$
* $LastChangedBy$
@ -11,12 +11,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -35,8 +35,8 @@ import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLContext;
@ -55,12 +55,16 @@ import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.routing.HttpRoute;
@ -73,6 +77,7 @@ import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
@ -87,7 +92,7 @@ import org.apache.http.util.EntityUtils;
/**
* HttpClient implementation which uses HttpComponents Client {@link http://hc.apache.org/}
*
*
* @author sixcooler
*
*/
@ -96,6 +101,7 @@ public class HTTPClient {
private final static int maxcon = 200;
private static IdledConnectionEvictor idledConnectionEvictor = null;
private static HttpClient httpClient = initConnectionManager();
private static final CredentialsProvider credsProvider = new BasicCredentialsProvider();
private Set<Entry<String, String>> headers = null;
private HttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null;
@ -105,33 +111,33 @@ public class HTTPClient {
private String host = null;
private boolean redirecting = true;
private String realm = null;
public HTTPClient() {
super();
}
public HTTPClient(final String userAgent) {
super();
this.userAgent = userAgent;
}
public HTTPClient(final String userAgent, final int timeout) {
super();
this.userAgent = userAgent;
this.timeout = timeout;
}
public static void setDefaultUserAgent(final String defaultAgent) {
HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent);
}
public static HttpClient initConnectionManager() {
// Create and initialize scheme registry
final SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory()));
ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry);
final ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry);
// Create and initialize HTTP parameters
final HttpParams httpParams = new BasicHttpParams();
@ -145,7 +151,7 @@ public class HTTPClient {
// connections per host (2 default)
clientConnectionManager.setDefaultMaxPerRoute(2);
// Increase max connections for localhost
HttpHost localhost = new HttpHost("localhost");
final HttpHost localhost = new HttpHost("localhost");
clientConnectionManager.setMaxForRoute(new HttpRoute(localhost), maxcon);
/**
* HTTP protocol settings
@ -170,7 +176,7 @@ public class HTTPClient {
HttpConnectionParams.setTcpNoDelay(httpParams, false);
// Defines whether the socket can be bound even though a previous connection is still in a timeout state.
HttpConnectionParams.setSoReuseaddr(httpParams, true);
httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
// ask for gzip
((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
@ -183,12 +189,12 @@ public class HTTPClient {
}
return httpClient;
}
/**
* This method should be called just before shutdown
* to stop the ConnectionManager and idledConnectionEvictor
*
* @throws InterruptedException
*
* @throws InterruptedException
*/
public static void closeConnectionManager() throws InterruptedException {
if (idledConnectionEvictor != null) {
@ -200,79 +206,86 @@ public class HTTPClient {
// Shut down the connection manager
httpClient.getConnectionManager().shutdown();
}
}
public static void setAuth(final String host, final int port, final String user, final String pw) {
final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw);
final AuthScope scope = new AuthScope(host, port);
credsProvider.setCredentials(scope, creds);
httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider);
}
/**
* this method sets a host on which more than the default of 2 router per host are allowed
*
*
* @param the host to be raised in 'route per host'
*/
public static void setMaxRouteHost(final String host) {
HttpHost mHost = new HttpHost(host);
final HttpHost mHost = new HttpHost(host);
((ThreadSafeClientConnManager) httpClient.getConnectionManager()).setMaxForRoute(new HttpRoute(mHost), 50);
}
/**
* This method sets the Header used for the request
*
*
* @param entrys to be set as request header
*/
public void setHeader(final Set<Entry<String, String>> entrys) {
this.headers = entrys;
}
/**
* This method sets the timeout of the Connection and Socket
*
*
* @param timeout in milliseconds
*/
public void setTimout(final int timeout) {
this.timeout = timeout;
}
/**
* This method sets the UserAgent to be used for the request
*
*
* @param userAgent
*/
public void setUserAgent(final String userAgent) {
this.userAgent = userAgent;
}
/**
* This method sets the host to be called at the request
*
*
* @param host
*/
public void setHost(final String host) {
this.host = host;
}
/**
* This method sets if requests should follow redirects
*
*
* @param redirecting
*/
public void setRedirecting(final boolean redirecting) {
this.redirecting = redirecting;
}
/**
* This method sets the authorization realm for the request
*
*
* @param realm
*/
public void setRealm(final String realm) {
this.realm = realm;
}
/**
* This method GETs a page from the server.
*
*
* @param uri the url to get
* @return content bytes
* @throws IOException
* @throws IOException
*/
public byte[] GETbytes(final String uri) throws IOException {
return GETbytes(uri, Long.MAX_VALUE);
@ -280,64 +293,64 @@ public class HTTPClient {
/**
* This method GETs a page from the server.
*
*
* @param uri the url to get
* @return content bytes
* @throws IOException
* @throws IOException
*/
public byte[] GETbytes(final MultiProtocolURI url) throws IOException {
return GETbytes(url, Long.MAX_VALUE);
}
/**
* This method GETs a page from the server.
*
*
* @param uri the url to get
* @param maxBytes to get
* @return content bytes
* @throws IOException
* @throws IOException
*/
public byte[] GETbytes(final String uri, long maxBytes) throws IOException {
public byte[] GETbytes(final String uri, final long maxBytes) throws IOException {
return GETbytes(new MultiProtocolURI(uri), maxBytes);
}
/**
* This method GETs a page from the server.
*
*
* @param uri the url to get
* @param maxBytes to get
* @return content bytes
* @throws IOException
* @throws IOException
*/
public byte[] GETbytes(final MultiProtocolURI url, long maxBytes) throws IOException {
boolean localhost = url.getHost().equals("localhost");
String urix = url.toNormalform(true, false, !localhost, false);
public byte[] GETbytes(final MultiProtocolURI url, final long maxBytes) throws IOException {
final boolean localhost = url.getHost().equals("localhost");
final String urix = url.toNormalform(true, false, !localhost, false);
final HttpGet httpGet = new HttpGet(urix);
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
return getContentBytes(httpGet, maxBytes);
}
/**
* This method GETs a page from the server.
* to be used for streaming out
* Please take care to call finish()!
*
*
* @param uri the url to get
* @throws IOException
*/
public void GET(final String uri) throws IOException {
if (currentRequest != null) throw new IOException("Client is in use!");
if (this.currentRequest != null) throw new IOException("Client is in use!");
final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false));
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
currentRequest = httpGet;
this.currentRequest = httpGet;
execute(httpGet);
}
/**
* This method gets HEAD response
*
*
* @param uri the url to Response from
* @return the HttpResponse
* @throws IOException
@ -349,21 +362,21 @@ public class HTTPClient {
execute(httpHead);
finish();
ConnectionInfo.removeConnection(httpHead.hashCode());
return httpResponse;
return this.httpResponse;
}
/**
* This method POSTs a page from the server.
* to be used for streaming out
* Please take care to call finish()!
*
*
* @param uri the url to post
* @param instream the input to post
* @param length the contentlength
* @throws IOException
* @throws IOException
*/
public void POST(final String uri, final InputStream instream, long length) throws IOException {
if (currentRequest != null) throw new IOException("Client is in use!");
public void POST(final String uri, final InputStream instream, final long length) throws IOException {
if (this.currentRequest != null) throw new IOException("Client is in use!");
final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
String host = url.getHost();
@ -371,19 +384,19 @@ public class HTTPClient {
setHost(host); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
final NonClosingInputStreamEntity inputStreamEntity = new NonClosingInputStreamEntity(instream, length);
// statistics
upbytes = length;
this.upbytes = length;
httpPost.setEntity(inputStreamEntity);
currentRequest = httpPost;
this.currentRequest = httpPost;
execute(httpPost);
}
/**
* send data to the server named by uri
*
*
* @param uri the url to post
* @param parts to post
* @return content bytes
* @throws IOException
* @throws IOException
*/
public byte[] POSTbytes(final String uri, final Map<String, ContentBody> parts, final boolean usegzip) throws IOException {
final MultiProtocolURI url = new MultiProtocolURI(uri);
@ -392,7 +405,7 @@ public class HTTPClient {
/**
* send data to the server named by vhost
*
*
* @param url address of the server
* @param vhost name of the server at address which should respond
* @param post data to send (name-value-pairs)
@ -402,7 +415,7 @@ public class HTTPClient {
*/
public byte[] POSTbytes(final MultiProtocolURI url, final String vhost, final Map<String, ContentBody> post, final boolean usegzip) throws IOException {
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
setHost(vhost); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (vhost == null) setHost("127.0.0.1");
@ -410,27 +423,27 @@ public class HTTPClient {
for (final Entry<String,ContentBody> part : post.entrySet())
multipartEntity.addPart(part.getKey(), part.getValue());
// statistics
upbytes = multipartEntity.getContentLength();
this.upbytes = multipartEntity.getContentLength();
if (usegzip) {
httpPost.setEntity(new GzipCompressingEntity(multipartEntity));
} else {
httpPost.setEntity(multipartEntity);
}
return getContentBytes(httpPost, Long.MAX_VALUE);
}
/**
* send stream-data to the server named by uri
*
*
* @param uri the url to post
* @param instream the stream to send
* @param length the length of the stream
* @return content bytes
* @throws IOException
*/
public byte[] POSTbytes(final String uri, final InputStream instream, long length) throws IOException {
public byte[] POSTbytes(final String uri, final InputStream instream, final long length) throws IOException {
final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
String host = url.getHost();
@ -439,27 +452,27 @@ public class HTTPClient {
final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length);
// statistics
upbytes = length;
this.upbytes = length;
httpPost.setEntity(inputStreamEntity);
return getContentBytes(httpPost, Long.MAX_VALUE);
}
/**
*
*
* @return HttpResponse from call
*/
public HttpResponse getHttpResponse() {
return httpResponse;
return this.httpResponse;
}
/**
*
*
* @return status code from http request
*/
public int getStatusCode() {
return httpResponse.getStatusLine().getStatusCode();
return this.httpResponse.getStatusLine().getStatusCode();
}
/**
* This method gets direct access to the content-stream
* Since this way is uncontrolled by the Client think of using 'writeTo' instead!
@ -469,20 +482,20 @@ public class HTTPClient {
* @throws IOException
*/
public InputStream getContentstream() throws IOException {
if (httpResponse != null && currentRequest != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (this.httpResponse != null && this.currentRequest != null) {
final HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) try {
return httpEntity.getContent();
} catch (final IOException e) {
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest.abort();
currentRequest = null;
ConnectionInfo.removeConnection(this.currentRequest.hashCode());
this.currentRequest.abort();
this.currentRequest = null;
throw e;
}
}
return null;
}
/**
* This method streams the content to the outputStream
* Please take care to call finish()!
@ -491,24 +504,24 @@ public class HTTPClient {
* @throws IOException
*/
public void writeTo(final OutputStream outputStream) throws IOException {
if (httpResponse != null && currentRequest != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (this.httpResponse != null && this.currentRequest != null) {
final HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) try {
httpEntity.writeTo(outputStream);
outputStream.flush();
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
EntityUtils.consume(httpEntity);
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
ConnectionInfo.removeConnection(this.currentRequest.hashCode());
this.currentRequest = null;
} catch (final IOException e) {
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest.abort();
currentRequest = null;
ConnectionInfo.removeConnection(this.currentRequest.hashCode());
this.currentRequest.abort();
this.currentRequest = null;
throw e;
}
}
}
/**
* This method ensures correct finish of client-connections
* This method should be used after every use of GET or POST and writeTo or getContentstream!
@ -516,35 +529,35 @@ public class HTTPClient {
* @throws IOException
*/
public void finish() throws IOException {
if (httpResponse != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (this.httpResponse != null) {
final HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null && httpEntity.isStreaming()) {
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
EntityUtils.consume(httpEntity);
}
}
if (currentRequest != null) {
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest.abort();
currentRequest = null;
if (this.currentRequest != null) {
ConnectionInfo.removeConnection(this.currentRequest.hashCode());
this.currentRequest.abort();
this.currentRequest = null;
}
}
private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final long maxBytes) throws IOException {
byte[] content = null;
try {
execute(httpUriRequest);
if (httpResponse == null) return null;
if (this.httpResponse == null) return null;
// get the response body
final HttpEntity httpEntity = httpResponse.getEntity();
final HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) {
if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) {
try {
content = EntityUtils.toByteArray(httpEntity);
} catch (OutOfMemoryError e) {
} catch (final OutOfMemoryError e) {
throw new IOException(e.toString());
}
}
}
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
EntityUtils.consume(httpEntity);
}
@ -556,7 +569,7 @@ public class HTTPClient {
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
return content;
}
private void execute(final HttpUriRequest httpUriRequest) throws IOException {
final HttpContext httpContext = new BasicHttpContext();
setHeaders(httpUriRequest);
@ -566,8 +579,8 @@ public class HTTPClient {
storeConnectionInfo(httpUriRequest);
// execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF
if (httpUriRequest instanceof HttpEntityEnclosingRequest) {
HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) httpUriRequest;
HttpEntity entity = hrequest.getEntity();
final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) httpUriRequest;
final HttpEntity entity = hrequest.getEntity();
assert entity != null;
//assert !entity.isChunked();
//assert entity.getContentLength() >= 0;
@ -575,45 +588,45 @@ public class HTTPClient {
}
try {
long time = System.currentTimeMillis();
httpResponse = httpClient.execute(httpUriRequest, httpContext);
httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time));
} catch (IOException e) {
final long time = System.currentTimeMillis();
this.httpResponse = httpClient.execute(httpUriRequest, httpContext);
this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time));
} catch (final IOException e) {
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
httpUriRequest.abort();
throw new IOException("Client can't execute: " + e.getMessage());
}
}
private void setHeaders(final HttpUriRequest httpUriRequest) {
if (headers != null) {
for (final Entry<String, String> entry : headers) {
if (this.headers != null) {
for (final Entry<String, String> entry : this.headers) {
httpUriRequest.setHeader(entry.getKey(),entry.getValue());
}
}
if (host != null)
httpUriRequest.setHeader(HTTP.TARGET_HOST, host);
if (realm != null)
httpUriRequest.setHeader("Authorization", "realm=" + realm);
if (this.host != null)
httpUriRequest.setHeader(HTTP.TARGET_HOST, this.host);
if (this.realm != null)
httpUriRequest.setHeader("Authorization", "realm=" + this.realm);
}
private void setParams(final HttpParams httpParams) {
HttpClientParams.setRedirecting(httpParams, redirecting);
HttpConnectionParams.setConnectionTimeout(httpParams, timeout);
HttpConnectionParams.setSoTimeout(httpParams, timeout);
if (userAgent != null)
HttpProtocolParams.setUserAgent(httpParams, userAgent);
if (host != null)
httpParams.setParameter(HTTP.TARGET_HOST, host);
}
HttpClientParams.setRedirecting(httpParams, this.redirecting);
HttpConnectionParams.setConnectionTimeout(httpParams, this.timeout);
HttpConnectionParams.setSoTimeout(httpParams, this.timeout);
if (this.userAgent != null)
HttpProtocolParams.setUserAgent(httpParams, this.userAgent);
if (this.host != null)
httpParams.setParameter(HTTP.TARGET_HOST, this.host);
}
private void setProxy(final HttpParams httpParams) {
if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
}
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
final int port = httpUriRequest.getURI().getPort();
final String thost = httpUriRequest.getURI().getHost();
@ -624,16 +637,16 @@ public class HTTPClient {
httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(),
httpUriRequest.hashCode(),
System.currentTimeMillis(),
upbytes));
this.upbytes));
}
private static SSLSocketFactory getSSLSocketFactory() {
final TrustManager trustManager = new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType)
public void checkClientTrusted(final X509Certificate[] chain, final String authType)
throws CertificateException {
}
public void checkServerTrusted(X509Certificate[] chain, String authType)
public void checkServerTrusted(final X509Certificate[] chain, final String authType)
throws CertificateException {
}
@ -645,10 +658,10 @@ public class HTTPClient {
try {
sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, new TrustManager[] { trustManager }, null);
} catch (NoSuchAlgorithmException e) {
} catch (final NoSuchAlgorithmException e) {
// should not happen
// e.printStackTrace();
} catch (KeyManagementException e) {
} catch (final KeyManagementException e) {
// should not happen
// e.printStackTrace();
}
@ -669,10 +682,10 @@ public class HTTPClient {
try {
newparts.put("foo", new StringBody("FooBar"));
newparts.put("bar", new StringBody("BarFoo"));
} catch (UnsupportedEncodingException e) {
} catch (final UnsupportedEncodingException e) {
System.out.println(e.getStackTrace());
}
HTTPClient client = new HTTPClient();
final HTTPClient client = new HTTPClient();
client.setUserAgent("foobar");
client.setRedirecting(false);
// Get some
@ -683,7 +696,7 @@ public class HTTPClient {
}
try {
System.out.println(UTF8.String(client.GETbytes(url)));
} catch (IOException e) {
} catch (final IOException e) {
e.printStackTrace();
}
}
@ -710,14 +723,14 @@ public class HTTPClient {
// Close out connection manager
try {
HTTPClient.closeConnectionManager();
} catch (InterruptedException e) {
} catch (final InterruptedException e) {
e.printStackTrace();
}
}
/**
*
*
* @see: http://hc.apache.org/httpcomponents-client-4.0.1/tutorial/html/connmgmt.html#d4e638
*
*/
@ -727,7 +740,7 @@ public class HTTPClient {
private volatile boolean shutdown;
public IdledConnectionEvictor(ClientConnectionManager clientConnectionManager) {
public IdledConnectionEvictor(final ClientConnectionManager clientConnectionManager) {
super();
this.clientConnectionManager = clientConnectionManager;
}
@ -735,29 +748,29 @@ public class HTTPClient {
@Override
public void run() {
try {
while (!shutdown) {
while (!this.shutdown) {
synchronized (this) {
wait(5000);
// Close expired connections
clientConnectionManager.closeExpiredConnections();
this.clientConnectionManager.closeExpiredConnections();
// Optionally, close connections
// that have been idle longer than 5 sec
// (some SOHO router act strange on >5sec idled connections)
clientConnectionManager.closeIdleConnections(5, TimeUnit.SECONDS);
this.clientConnectionManager.closeIdleConnections(5, TimeUnit.SECONDS);
}
}
} catch (InterruptedException ex) {
} catch (final InterruptedException ex) {
// terminate
}
}
public void shutdown() {
shutdown = true;
this.shutdown = true;
synchronized (this) {
notifyAll();
}
}
}
}

@ -32,6 +32,7 @@ import java.util.List;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.document.Document;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
@ -46,6 +47,7 @@ public class SolrChardingConnector {
private final String[] urls;
public SolrChardingConnector(final String urlList, final SolrScheme scheme, final SolrChardingSelection.Method method) throws IOException {
urlList.replace(' ', ',');
this.urls = urlList.split(",");
this.connectors = new ArrayList<SolrSingleConnector>();
for (final String u: this.urls) {
@ -156,17 +158,22 @@ public class SolrChardingConnector {
return list;
}
public long[] getSizeList() throws IOException {
public long[] getSizeList() {
final long[] size = new long[this.connectors.size()];
int i = 0;
for (final SolrSingleConnector connector: this.connectors) {
final SolrDocumentList list = connector.get("*:*", 0, 1);
size[i++] = list.getNumFound();
try {
final SolrDocumentList list = connector.get("*:*", 0, 1);
size[i++] = list.getNumFound();
} catch (final IOException e) {
Log.logException(e);
size[i++] = 0;
}
}
return size;
}
public long getSize() throws IOException {
public long getSize() {
final long[] size = getSizeList();
long s = 0;
for (final long l: size) s += l;

@ -34,6 +34,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.http.HTTPClient;
import org.apache.http.entity.mime.content.ContentBody;
@ -44,9 +45,9 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
@ -57,9 +58,9 @@ import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
/**
* The {@link SolrHTTPClient} uses the Apache Commons HTTP Client to connect to solr.
* The {@link SolrHTTPClient} uses the Apache Commons HTTP Client to connect to solr.
* <pre class="prettyprint" >SolrServer server = new CommonsHttpSolrServer( url );</pre>
*
*
* @version $Id: CommonsHttpSolrServer.java 1067552 2011-02-05 23:52:42Z koji $
* @since solr 1.3
*/
@ -71,7 +72,7 @@ public class SolrHTTPClient extends SolrServer {
* org.apache.commons.httpclient.HttpClient HttpClient} to the Solr
* server from the client.
*/
public static final String AGENT = "Solr["+SolrHTTPClient.class.getName()+"] 1.0";
public static final String AGENT = "Solr["+SolrHTTPClient.class.getName()+"] 1.0";
public final static Charset utf8;
static {
@ -81,15 +82,16 @@ public class SolrHTTPClient extends SolrServer {
/**
* The URL of the Solr server.
*/
protected String _baseURL;
protected String _baseURL, host, solraccount, solrpw;
protected int port;
/**
* Default value: null / empty. <p/>
* Parameters that are added to every request regardless. This may be a place to add
* Parameters that are added to every request regardless. This may be a place to add
* something like an authentication token.
*/
protected ModifiableSolrParams _invariantParams;
/**
* Default response parser is BinaryResponseParser <p/>
* This parser represents the default Response Parser chosen to
@ -104,14 +106,14 @@ public class SolrHTTPClient extends SolrServer {
* @see org.apache.solr.client.solrj.request.RequestWriter
*/
protected RequestWriter requestWriter = new RequestWriter();
/**
* @param solrServerUrl The URL of the Solr server. For
/**
* @param solrServerUrl The URL of the Solr server. For
* example, "<code>http://localhost:8983/solr/</code>"
* if you are using the standard distribution Solr webapp
* if you are using the standard distribution Solr webapp
* on your local machine.
*/
public SolrHTTPClient(String solrServerUrl) throws MalformedURLException {
public SolrHTTPClient(final String solrServerUrl) throws MalformedURLException {
this(new URL(solrServerUrl));
}
@ -120,7 +122,7 @@ public class SolrHTTPClient extends SolrServer {
* "<code>http://localhost:8983/solr/</code>" if you are using the
* standard distribution Solr webapp on your local machine.
*/
public SolrHTTPClient(URL baseURL)
public SolrHTTPClient(final URL baseURL)
{
this(baseURL, new BinaryResponseParser());
}
@ -129,17 +131,39 @@ public class SolrHTTPClient extends SolrServer {
* @see #useMultiPartPost
* @see #_parser
*/
public SolrHTTPClient(URL baseURL, ResponseParser parser) {
_baseURL = baseURL.toExternalForm();
if( _baseURL.endsWith( "/" ) ) {
_baseURL = _baseURL.substring( 0, _baseURL.length()-1 );
public SolrHTTPClient(final URL baseURL, final ResponseParser parser) {
this._baseURL = baseURL.toExternalForm();
if( this._baseURL.endsWith( "/" ) ) {
this._baseURL = this._baseURL.substring( 0, this._baseURL.length()-1 );
}
if( _baseURL.indexOf( '?' ) >=0 ) {
throw new RuntimeException( "Invalid base url for solrj. The base URL must not contain parameters: "+_baseURL );
if( this._baseURL.indexOf( '?' ) >=0 ) {
throw new RuntimeException( "Invalid base url for solrj. The base URL must not contain parameters: "+this._baseURL );
}
_parser = parser;
MultiProtocolURI u;
try {
u = new MultiProtocolURI(this._baseURL.toString());
this.host = u.getHost();
this.port = u.getPort();
final String userinfo = u.getUserInfo();
if (userinfo == null || userinfo.length() == 0) {
this.solraccount = ""; this.solrpw = "";
} else {
final int p = userinfo.indexOf(':');
if (p < 0) {
this.solraccount = userinfo; this.solrpw = "";
} else {
this.solraccount = userinfo.substring(0, p); this.solrpw = userinfo.substring(p + 1);
}
}
} catch (final MalformedURLException e) {
this.solraccount = ""; this.solrpw = "";
this.host = ""; this.port = -1;
}
this._parser = parser;
}
//------------------------------------------------------------------------
//------------------------------------------------------------------------
@ -158,26 +182,26 @@ public class SolrHTTPClient extends SolrServer {
{
ResponseParser responseParser = request.getResponseParser();
if (responseParser == null) {
responseParser = _parser;
responseParser = this._parser;
}
return request(request, responseParser);
}
public NamedList<Object> request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException {
public NamedList<Object> request(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException {
SolrParams params = request.getParams();
Collection<ContentStream> streams = requestWriter.getContentStreams(request);
String path = requestWriter.getPath(request);
final Collection<ContentStream> streams = this.requestWriter.getContentStreams(request);
String path = this.requestWriter.getPath(request);
if( path == null || !path.startsWith( "/" ) ) {
path = "/select";
}
// The parser 'wt=' and 'version=' params are used instead of the original params
ResponseParser parser = request.getResponseParser();
if( parser == null ) {
parser = _parser;
parser = this._parser;
}
ModifiableSolrParams wparams = new ModifiableSolrParams();
final ModifiableSolrParams wparams = new ModifiableSolrParams();
wparams.set( CommonParams.WT, parser.getWriterType() );
wparams.set( CommonParams.VERSION, parser.getVersion());
if( params == null ) {
@ -186,26 +210,30 @@ public class SolrHTTPClient extends SolrServer {
else {
params = new DefaultSolrParams( wparams, params );
}
if( _invariantParams != null ) {
params = new DefaultSolrParams( _invariantParams, params );
if( this._invariantParams != null ) {
params = new DefaultSolrParams( this._invariantParams, params );
}
byte[] result = null;
HTTPClient client = new HTTPClient();
final HTTPClient client = new HTTPClient();
if (this.solraccount.length() > 0 && this.solrpw.length() > 0 && this.host.length() > 0) {
HTTPClient.setAuth(this.host, this.port, this.solraccount, this.solrpw);
}
if (SolrRequest.METHOD.POST == request.getMethod()) {
boolean isMultipart = ( streams != null && streams.size() > 1 );
final boolean isMultipart = ( streams != null && streams.size() > 1 );
if (streams == null || isMultipart) {
String url = _baseURL + path;
HashMap<String, ContentBody> parts = new HashMap<String, ContentBody>();
Iterator<String> iter = params.getParameterNamesIterator();
String url = this._baseURL + path;
final HashMap<String, ContentBody> parts = new HashMap<String, ContentBody>();
final Iterator<String> iter = params.getParameterNamesIterator();
while (iter.hasNext()) {
String p = iter.next();
String[] vals = params.getParams(p);
final String p = iter.next();
final String[] vals = params.getParams(p);
if (vals != null) {
for (String v : vals) {
for (final String v : vals) {
if (isMultipart) {
parts.put(p, new StringBody(v, utf8));
} else {
@ -214,13 +242,13 @@ public class SolrHTTPClient extends SolrServer {
}
}
}
if (isMultipart) {
for (ContentStream content : streams) {
for (final ContentStream content : streams) {
parts.put(content.getName(), new InputStreamBody(content.getStream(), content.getContentType(), null));
}
}
try {
result = client.POSTbytes(url, parts, true);
} finally {
@ -228,32 +256,32 @@ public class SolrHTTPClient extends SolrServer {
}
} else {
// It has one stream, this is the post body, put the params in the URL
String pstr = ClientUtils.toQueryString(params, false);
String url = _baseURL + path + pstr;
final String pstr = ClientUtils.toQueryString(params, false);
final String url = this._baseURL + path + pstr;
// Single stream as body
// Using a loop just to get the first one
final ContentStream[] contentStream = new ContentStream[1];
for (ContentStream content : streams) {
for (final ContentStream content : streams) {
contentStream[0] = content;
break;
}
result = client.POSTbytes(url, contentStream[0].getStream(), contentStream[0].getStream().available());
}
} else if (SolrRequest.METHOD.GET == request.getMethod()) {
result = client.GETbytes( _baseURL + path + ClientUtils.toQueryString( params, false ));
result = client.GETbytes( this._baseURL + path + ClientUtils.toQueryString( params, false ));
} else {
throw new SolrServerException("Unsupported method: "+request.getMethod() );
}
int statusCode = client.getStatusCode();
final int statusCode = client.getStatusCode();
if (statusCode != 200) {
throw new IOException("bad status code: " + statusCode + ", " + client.getHttpResponse().getStatusLine());
throw new IOException("bad status code: " + statusCode + ", " + client.getHttpResponse().getStatusLine() + ", url = " + this._baseURL + path);
}
// Read the contents
//System.out.println("SOLR RESPONSE: " + UTF8.String(result));
InputStream respBody = new ByteArrayInputStream(result);
final InputStream respBody = new ByteArrayInputStream(result);
return processor.processResponse(respBody, "UTF-8");
}
@ -268,12 +296,12 @@ public class SolrHTTPClient extends SolrServer {
if( path == null || !path.startsWith( "/" ) ) {
path = "/select";
}
ResponseParser parser = request.getResponseParser();
if( parser == null ) {
parser = _parser;
}
// The parser 'wt=' and 'version=' params are used instead of the original params
ModifiableSolrParams wparams = new ModifiableSolrParams();
wparams.set( CommonParams.WT, parser.getWriterType() );
@ -284,7 +312,7 @@ public class SolrHTTPClient extends SolrServer {
else {
params = new DefaultSolrParams( wparams, params );
}
if( _invariantParams != null ) {
params = new DefaultSolrParams( _invariantParams, params );
}
@ -505,30 +533,30 @@ public class SolrHTTPClient extends SolrServer {
}
*/
//-------------------------------------------------------------------
//-------------------------------------------------------------------
/**
* Retrieve the default list of parameters are added to every request regardless.
*
*
* @see #_invariantParams
*/
public ModifiableSolrParams getInvariantParams()
{
return _invariantParams;
return this._invariantParams;
}
public String getBaseURL() {
return _baseURL;
return this._baseURL;
}
public void setBaseURL(String baseURL) {
public void setBaseURL(final String baseURL) {
this._baseURL = baseURL;
}
public ResponseParser getParser() {
return _parser;
return this._parser;
}
/**
@ -536,12 +564,12 @@ public class SolrHTTPClient extends SolrServer {
* @param processor Default Response Parser chosen to parse the response if the parser were not specified as part of the request.
* @see org.apache.solr.client.solrj.SolrRequest#getResponseParser()
*/
public void setParser(ResponseParser processor) {
_parser = processor;
public void setParser(final ResponseParser processor) {
this._parser = processor;
}
public void setRequestWriter(RequestWriter requestWriter) {
public void setRequestWriter(final RequestWriter requestWriter) {
this.requestWriter = requestWriter;
}
@ -552,10 +580,10 @@ public class SolrHTTPClient extends SolrServer {
*
* @return the response from the SolrServer
*/
public UpdateResponse add(Iterator<SolrInputDocument> docIterator)
public UpdateResponse add(final Iterator<SolrInputDocument> docIterator)
throws SolrServerException, IOException {
UpdateRequest req = new UpdateRequest();
req.setDocIterator(docIterator);
final UpdateRequest req = new UpdateRequest();
req.setDocIterator(docIterator);
return req.process(this);
}
@ -568,7 +596,7 @@ public class SolrHTTPClient extends SolrServer {
*/
public UpdateResponse addBeans(final Iterator<?> beanIterator)
throws SolrServerException, IOException {
UpdateRequest req = new UpdateRequest();
final UpdateRequest req = new UpdateRequest();
req.setDocIterator(new Iterator<SolrInputDocument>() {
public boolean hasNext() {
@ -576,7 +604,7 @@ public class SolrHTTPClient extends SolrServer {
}
public SolrInputDocument next() {
Object o = beanIterator.next();
final Object o = beanIterator.next();
if (o == null) return null;
return getBinder().toSolrInputDocument(o);
}

Loading…
Cancel
Save