From 5fa8038f10bd6b70c7445aba1c009f2ce36780e8 Mon Sep 17 00:00:00 2001 From: sixcooler Date: Tue, 20 Jul 2010 01:14:28 +0000 Subject: [PATCH] ... migrating to HttpComponents-Client-4.x ... monitoring and first try to use remoteProxy git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6979 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Connections_p.java | 7 +- source/de/anomic/search/Switchboard.java | 10 +- source/de/anomic/yacy/yacyNetwork.java | 70 ++--- source/net/yacy/cora/protocol/Client.java | 67 +++-- .../yacy/cora/protocol/ConnectionInfo.java | 255 ++++++++++++++++++ .../net/yacy/cora/protocol/ProxySettings.java | 20 ++ 6 files changed, 362 insertions(+), 67 deletions(-) create mode 100644 source/net/yacy/cora/protocol/ConnectionInfo.java diff --git a/htroot/Connections_p.java b/htroot/Connections_p.java index 2b68f8a93..60017922e 100644 --- a/htroot/Connections_p.java +++ b/htroot/Connections_p.java @@ -32,12 +32,13 @@ import java.net.InetAddress; import java.net.URLEncoder; import java.util.Set; +import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.workflow.WorkflowThread; -import de.anomic.http.client.ConnectionInfo; -import de.anomic.http.client.Client; +//import de.anomic.http.client.ConnectionInfo; +//import de.anomic.http.client.Client; import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverCore; @@ -167,7 +168,7 @@ public final class Connections_p { } } prop.put("clientList", c); - prop.put("clientActive", Client.connectionCount()); + prop.put("clientActive", ConnectionInfo.getCount()); // return rewrite values for templates return prop; diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index 6b3c5d7e7..9a6906050 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -72,6 +72,7 @@ import java.util.zip.ZipInputStream; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.ProxySettings; import net.yacy.document.Condenser; import net.yacy.document.Document; @@ -1485,6 +1486,7 @@ public final class Switchboard extends serverSwitch { // close unused connections Client.cleanup(); + ConnectionInfo.cleanUp(); // do transmission of CR-files checkInterruption(); @@ -2125,8 +2127,8 @@ public final class Switchboard extends serverSwitch { log.logInfo("dhtTransferJob: no selection, too many entries in transmission cloud: " + this.dhtDispatcher.cloudSize()); } else if (MemoryControl.available() < 1024*1024*25) { log.logInfo("dhtTransferJob: no selection, too less memory available : " + (MemoryControl.available() / 1024 / 1024) + " MB"); - } else if (net.yacy.cora.protocol.Client.connectionCount() > 5) { - log.logInfo("dhtTransferJob: too many connections in httpc pool : " + net.yacy.cora.protocol.Client.connectionCount()); + } else if (ConnectionInfo.getLoadPercent() > 50) { + log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount()); // close unused connections // Client.cleanup(); } else { @@ -2157,8 +2159,8 @@ public final class Switchboard extends serverSwitch { // check if we can deliver entries to other peers if (this.dhtDispatcher.transmissionSize() >= 10) { log.logInfo("dhtTransferJob: no dequeueing from cloud to transmission: too many concurrent sessions: " + this.dhtDispatcher.transmissionSize()); - } else if (net.yacy.cora.protocol.Client.connectionCount() > 5) { - log.logInfo("dhtTransferJob: too many connections in httpc pool : " + net.yacy.cora.protocol.Client.connectionCount()); + } else if (ConnectionInfo.getLoadPercent() > 75) { + log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount()); // close unused connections // Client.cleanup(); } else { diff --git a/source/de/anomic/yacy/yacyNetwork.java b/source/de/anomic/yacy/yacyNetwork.java index bb1f63dc3..721562777 100644 --- a/source/de/anomic/yacy/yacyNetwork.java +++ b/source/de/anomic/yacy/yacyNetwork.java @@ -28,19 +28,19 @@ package de.anomic.yacy; import java.io.UnsupportedEncodingException; -import java.util.ArrayList; +//import java.util.ArrayList; import java.util.Date; import java.util.LinkedHashMap; -import java.util.List; +//import java.util.List; import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.util.DateFormatter; -import org.apache.commons.httpclient.methods.multipart.Part; +//import org.apache.commons.httpclient.methods.multipart.Part; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; -import de.anomic.http.client.DefaultCharsetStringPart; +//import de.anomic.http.client.DefaultCharsetStringPart; import de.anomic.search.Switchboard; import de.anomic.search.SwitchboardConstants; import de.anomic.server.serverObjects; @@ -77,37 +77,37 @@ public class yacyNetwork { return false; } - public static final List basicRequestPost(final Switchboard sb, final String targetHash, final String salt) { - // put in all the essentials for routing and network authentication - // generate a session key - final ArrayList post = new ArrayList(); - post.add(new DefaultCharsetStringPart("key", salt)); - - // just standard identification essentials - post.add(new DefaultCharsetStringPart("iam", sb.peers.mySeed().hash)); - if (targetHash != null) post.add(new DefaultCharsetStringPart("youare", targetHash)); - - // time information for synchronization - post.add(new DefaultCharsetStringPart("mytime", DateFormatter.formatShortSecond(new Date()))); - post.add(new DefaultCharsetStringPart("myUTC", Long.toString(System.currentTimeMillis()))); - - // network identification - post.add(new DefaultCharsetStringPart(SwitchboardConstants.NETWORK_NAME, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT))); - - // authentication essentials - final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled"); - final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", ""); - if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0)) { - if (authenticationMethod.equals("salted-magic-sim")) { - // generate an authentication essential using the salt, the iam-hash and the network magic - final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", ""); - final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic); - post.add(new DefaultCharsetStringPart("magicmd5", md5)); - } - } - - return post; - } +// public static final List basicRequestPost(final Switchboard sb, final String targetHash, final String salt) { +// // put in all the essentials for routing and network authentication +// // generate a session key +// final ArrayList post = new ArrayList(); +// post.add(new DefaultCharsetStringPart("key", salt)); +// +// // just standard identification essentials +// post.add(new DefaultCharsetStringPart("iam", sb.peers.mySeed().hash)); +// if (targetHash != null) post.add(new DefaultCharsetStringPart("youare", targetHash)); +// +// // time information for synchronization +// post.add(new DefaultCharsetStringPart("mytime", DateFormatter.formatShortSecond(new Date()))); +// post.add(new DefaultCharsetStringPart("myUTC", Long.toString(System.currentTimeMillis()))); +// +// // network identification +// post.add(new DefaultCharsetStringPart(SwitchboardConstants.NETWORK_NAME, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT))); +// +// // authentication essentials +// final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled"); +// final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", ""); +// if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0)) { +// if (authenticationMethod.equals("salted-magic-sim")) { +// // generate an authentication essential using the salt, the iam-hash and the network magic +// final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", ""); +// final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic); +// post.add(new DefaultCharsetStringPart("magicmd5", md5)); +// } +// } +// +// return post; +// } public static final LinkedHashMap basicRequestParts(final Switchboard sb, final String targetHash, final String salt) throws UnsupportedEncodingException { // put in all the essentials for routing and network authentication diff --git a/source/net/yacy/cora/protocol/Client.java b/source/net/yacy/cora/protocol/Client.java index 5994f3224..02eceaf71 100644 --- a/source/net/yacy/cora/protocol/Client.java +++ b/source/net/yacy/cora/protocol/Client.java @@ -17,6 +17,7 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.params.ConnManagerParams; import org.apache.http.conn.params.ConnPerRouteBean; +import org.apache.http.conn.params.ConnRouteParams; import org.apache.http.conn.routing.HttpRoute; import org.apache.http.conn.scheme.PlainSocketFactory; import org.apache.http.conn.scheme.Scheme; @@ -25,11 +26,10 @@ import org.apache.http.conn.ssl.SSLSocketFactory; import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; +import org.apache.http.impl.client.AbstractHttpClient; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.http.params.CoreProtocolPNames; import org.apache.http.params.HttpConnectionParams; import org.apache.http.params.HttpParams; import org.apache.http.params.HttpProtocolParams; @@ -46,9 +46,9 @@ import org.apache.http.util.EntityUtils; */ public class Client { + private final static int maxcon = 20; private static IdledConnectionEvictor idledConnectionEvictor = null; private static HttpClient httpClient = null; - private static int count = 0; private int timeout = 10000; private String userAgent = null; private String host = null; @@ -67,12 +67,14 @@ public class Client { * ConnectionManager settings */ // TODO: how much connections do we need? - default: 20 - // ConnManagerParams.setMaxTotalConnections(httpParams, 100); + ConnManagerParams.setMaxTotalConnections(httpParams, maxcon); + // for statistics same value should also be set here + ConnectionInfo.setMaxcount(maxcon); // perhaps we need more than 2(default) connections per host? ConnPerRouteBean connPerRoute = new ConnPerRouteBean(2); // Increase max connections for localhost to 100 HttpHost localhost = new HttpHost("locahost"); - connPerRoute.setMaxForRoute(new HttpRoute(localhost), 100); + connPerRoute.setMaxForRoute(new HttpRoute(localhost), maxcon); ConnManagerParams.setMaxConnectionsPerRoute(httpParams, connPerRoute); // how long to wait for getting a connection from manager in milliseconds ConnManagerParams.setTimeout(httpParams, 3000L); @@ -88,7 +90,7 @@ public class Client { // timeout in milliseconds until a connection is established in milliseconds HttpConnectionParams.setConnectionTimeout(httpParams, 10000); // SO_LINGER affects the socket close operation in seconds - HttpConnectionParams.setLinger(httpParams, 6); + // HttpConnectionParams.setLinger(httpParams, 6); // TODO: is default ok? // HttpConnectionParams.setSocketBufferSize(httpParams, 8192); // SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds @@ -97,6 +99,7 @@ public class Client { HttpConnectionParams.setStaleCheckingEnabled(httpParams, true); // conserve bandwidth by minimizing the number of segments that are sent HttpConnectionParams.setTcpNoDelay(httpParams, false); + // TODO: testing noreuse - there will be HttpConnectionParams.setSoReuseaddr(HttpParams params, boolean reuseaddr) in core-4.1 // Create and initialize scheme registry SchemeRegistry schemeRegistry = new SchemeRegistry(); @@ -151,15 +154,6 @@ public class Client { this.host = host; } - /** - * number of active connections - * - * @return number of active connections - */ - public static int connectionCount() { - return count; - } - /** * This method GETs a page from the server. * @@ -204,16 +198,12 @@ public class Client { } private byte[] getContentBytes(HttpUriRequest httpUriRequest, long maxBytes) throws IOException { - count++; byte[] content = null; final HttpContext httpContext = new BasicHttpContext(); - httpUriRequest.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); - httpUriRequest.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, timeout); - if (userAgent != null) - httpUriRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent); - if (host != null) - httpUriRequest.getParams().setParameter(HTTP.TARGET_HOST, host); - + setParams(httpUriRequest.getParams()); + setProxy(httpUriRequest.getParams()); + // statistics + storeConnectionInfo(httpUriRequest); try { // execute the method HttpResponse httpResponse = httpClient.execute(httpUriRequest, httpContext); @@ -229,13 +219,40 @@ public class Client { } } catch (final IOException e) { httpUriRequest.abort(); - count--; + ConnectionInfo.removeConnection(httpUriRequest.hashCode()); throw e; } - count--; + ConnectionInfo.removeConnection(httpUriRequest.hashCode()); return content; } + private void setParams(HttpParams httpParams) { + HttpConnectionParams.setConnectionTimeout(httpParams, timeout); + HttpConnectionParams.setSoTimeout(httpParams, timeout); + if (userAgent != null) + HttpProtocolParams.setUserAgent(httpParams, userAgent); + if (host != null) + httpParams.setParameter(HTTP.TARGET_HOST, host); + } + + private void setProxy(HttpParams httpParams) { + if (ProxySettings.use) + ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost()); + // TODO find a better way for this + ProxySettings.setProxyCreds((AbstractHttpClient) httpClient); + } + + private void storeConnectionInfo(HttpUriRequest httpUriRequest) { + int port = httpUriRequest.getURI().getPort(); + String thost = httpUriRequest.getURI().getHost(); + ConnectionInfo.addConnection(new ConnectionInfo( + httpUriRequest.getURI().getScheme(), + port == 80 ? thost : thost + ":" + port, + httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(), + httpUriRequest.hashCode(), + System.currentTimeMillis())); + } + /** * provide system information for client identification */ diff --git a/source/net/yacy/cora/protocol/ConnectionInfo.java b/source/net/yacy/cora/protocol/ConnectionInfo.java new file mode 100644 index 000000000..70a2e4cda --- /dev/null +++ b/source/net/yacy/cora/protocol/ConnectionInfo.java @@ -0,0 +1,255 @@ +// HttpConnectionInfo.java +// (C) 2008 by Daniel Raap; danielr@users.berlios.de +// first published 07.04.2008 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $ +// $LastChangedRevision: 4558 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.cora.protocol; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + + +/** + * Information about a connection + * + * @author daniel + * @author sixcooler + */ +public class ConnectionInfo { + /** + * a list of all current connections to be shown in Connections_p + */ + private final static Set allConnections = Collections + .synchronizedSet(new HashSet()); + // this is only for statistics, so it can be bigger to see lost connectionInfos + private final static int staleAfterMillis = 30 * 60000; // 30 minutes + + private static int maxcount = 20; + + private final String protocol; + private final String targetHost; + private final String command; + private final int id; + private final long initTime; + + /** + * constructor setting all data + * + * @param protocol + * @param targetHost + * @param command + * @param id + * @param initTime + */ + public ConnectionInfo(final String protocol, final String targetHost, final String command, final int id, + final long initTime) { + this.protocol = protocol; + this.targetHost = targetHost; + this.command = command; + this.id = id; + this.initTime = initTime; + } + + /** + * @return + */ + public String getProtocol() { + return protocol; + } + + /** + * @return + */ + public long getLifetime() { + return System.currentTimeMillis() - initTime; + } + + /** + * @return dummy 0 + */ + public int getIdletime() { + return 0; + } + + /** + * @return + */ + public String getCommand() { + return command; + } + + /** + * @return + */ + public String getTargetHost() { + return targetHost; + } + + /** + * @return + */ + public int getID() { + return id; + } + + /** + * gets a {@link Set} of all collected ConnectionInfos + * + * Important: iterations must be synchronized! + * + * @return the allConnections + */ + public static Set getAllConnections() { + return allConnections; + } + + /** + * gets the number of active client connections + * + * @return count of active connections + */ + public static int getCount() { + return allConnections.size(); + } + + /** + * gets the usage of the Client connection manager by active connections + * + * @return load in percent + */ + public static int getLoadPercent() { + return getCount() * 100 / maxcount; + } + + /** + * gets the max connection count of the Client connection manager + * + * @return max connections + */ + public static int getMaxcount() { + return maxcount; + } + + /** + * gets the max connection count of the Client connection manager + * to be used in statistics + * + * @param max connections + */ + protected static void setMaxcount(final int max) { + if (max > 0) maxcount = max; + } + + /** + * add a connection to the list of all current connections + * + * @param conInfo + */ + protected static void addConnection(final ConnectionInfo conInfo) { + allConnections.add(conInfo); + } + + /** + * remove a connection from the list of all current connections + * + * @param conInfo + */ + protected static void removeConnection(final ConnectionInfo conInfo) { + allConnections.remove(conInfo); + } + + /** + * connections with same id {@link equals()} another + * + * @param id + */ + protected static void removeConnection(final int id) { + removeConnection(new ConnectionInfo(null, null, null, id, 0)); + } + + /** + * removes stale connections + */ + public static void cleanUp() { + try { + synchronized (allConnections) { + for(final ConnectionInfo con: allConnections) { + if(con.getLifetime() > staleAfterMillis) { + allConnections.remove(con); + } + } + } + } catch (final java.util.ConcurrentModificationException e) { + // there will be another try :-) + } + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + final StringBuilder string = new StringBuilder(50); + string.append("ID "); + string.append(getID()); + string.append(", "); + string.append(getProtocol()); + string.append("://"); + string.append(getTargetHost()); + string.append(" "); + string.append(getCommand()); + string.append(", since "); + string.append(getLifetime()); + string.append(" ms"); + return string.toString(); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + id; + return result; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(final Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + final ConnectionInfo other = (ConnectionInfo) obj; + return this.id == other.id; + } +} diff --git a/source/net/yacy/cora/protocol/ProxySettings.java b/source/net/yacy/cora/protocol/ProxySettings.java index 5b590fd93..87244b285 100644 --- a/source/net/yacy/cora/protocol/ProxySettings.java +++ b/source/net/yacy/cora/protocol/ProxySettings.java @@ -25,6 +25,10 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.httpclient.HostConfiguration; import org.apache.commons.httpclient.HttpClient; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.impl.client.AbstractHttpClient; /** * settings for a remote proxy @@ -55,6 +59,22 @@ public final class ProxySettings { return hostConfig; } + /** + * + * @return the HttpHost to be used as proxy + */ + public static HttpHost getProxyHost() { + if (!use) return null; + return new HttpHost(host, port); + } + + public static void setProxyCreds(AbstractHttpClient httpClient) { + if (!use) return; + httpClient.getCredentialsProvider().setCredentials( + new AuthScope(host, port), + new UsernamePasswordCredentials(user, password)); + } + /** * tell if a remote proxy will be used for the given host * @param host