... migrating to HttpComponents-Client-4.x ...

monitoring and first try to use remoteProxy

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6979 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 15 years ago
parent dec1419bc3
commit 5fa8038f10

@ -32,12 +32,13 @@ import java.net.InetAddress;
import java.net.URLEncoder;
import java.util.Set;
import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.workflow.WorkflowThread;
import de.anomic.http.client.ConnectionInfo;
import de.anomic.http.client.Client;
//import de.anomic.http.client.ConnectionInfo;
//import de.anomic.http.client.Client;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverCore;
@ -167,7 +168,7 @@ public final class Connections_p {
}
}
prop.put("clientList", c);
prop.put("clientActive", Client.connectionCount());
prop.put("clientActive", ConnectionInfo.getCount());
// return rewrite values for templates
return prop;

@ -72,6 +72,7 @@ import java.util.zip.ZipInputStream;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.ProxySettings;
import net.yacy.document.Condenser;
import net.yacy.document.Document;
@ -1485,6 +1486,7 @@ public final class Switchboard extends serverSwitch {
// close unused connections
Client.cleanup();
ConnectionInfo.cleanUp();
// do transmission of CR-files
checkInterruption();
@ -2125,8 +2127,8 @@ public final class Switchboard extends serverSwitch {
log.logInfo("dhtTransferJob: no selection, too many entries in transmission cloud: " + this.dhtDispatcher.cloudSize());
} else if (MemoryControl.available() < 1024*1024*25) {
log.logInfo("dhtTransferJob: no selection, too less memory available : " + (MemoryControl.available() / 1024 / 1024) + " MB");
} else if (net.yacy.cora.protocol.Client.connectionCount() > 5) {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + net.yacy.cora.protocol.Client.connectionCount());
} else if (ConnectionInfo.getLoadPercent() > 50) {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount());
// close unused connections
// Client.cleanup();
} else {
@ -2157,8 +2159,8 @@ public final class Switchboard extends serverSwitch {
// check if we can deliver entries to other peers
if (this.dhtDispatcher.transmissionSize() >= 10) {
log.logInfo("dhtTransferJob: no dequeueing from cloud to transmission: too many concurrent sessions: " + this.dhtDispatcher.transmissionSize());
} else if (net.yacy.cora.protocol.Client.connectionCount() > 5) {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + net.yacy.cora.protocol.Client.connectionCount());
} else if (ConnectionInfo.getLoadPercent() > 75) {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount());
// close unused connections
// Client.cleanup();
} else {

@ -28,19 +28,19 @@
package de.anomic.yacy;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
//import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
//import java.util.List;
import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.util.DateFormatter;
import org.apache.commons.httpclient.methods.multipart.Part;
//import org.apache.commons.httpclient.methods.multipart.Part;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
import de.anomic.http.client.DefaultCharsetStringPart;
//import de.anomic.http.client.DefaultCharsetStringPart;
import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
import de.anomic.server.serverObjects;
@ -77,37 +77,37 @@ public class yacyNetwork {
return false;
}
public static final List<Part> basicRequestPost(final Switchboard sb, final String targetHash, final String salt) {
// put in all the essentials for routing and network authentication
// generate a session key
final ArrayList<Part> post = new ArrayList<Part>();
post.add(new DefaultCharsetStringPart("key", salt));
// just standard identification essentials
post.add(new DefaultCharsetStringPart("iam", sb.peers.mySeed().hash));
if (targetHash != null) post.add(new DefaultCharsetStringPart("youare", targetHash));
// time information for synchronization
post.add(new DefaultCharsetStringPart("mytime", DateFormatter.formatShortSecond(new Date())));
post.add(new DefaultCharsetStringPart("myUTC", Long.toString(System.currentTimeMillis())));
// network identification
post.add(new DefaultCharsetStringPart(SwitchboardConstants.NETWORK_NAME, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT)));
// authentication essentials
final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", "");
if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0)) {
if (authenticationMethod.equals("salted-magic-sim")) {
// generate an authentication essential using the salt, the iam-hash and the network magic
final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
post.add(new DefaultCharsetStringPart("magicmd5", md5));
}
}
return post;
}
// public static final List<Part> basicRequestPost(final Switchboard sb, final String targetHash, final String salt) {
// // put in all the essentials for routing and network authentication
// // generate a session key
// final ArrayList<Part> post = new ArrayList<Part>();
// post.add(new DefaultCharsetStringPart("key", salt));
//
// // just standard identification essentials
// post.add(new DefaultCharsetStringPart("iam", sb.peers.mySeed().hash));
// if (targetHash != null) post.add(new DefaultCharsetStringPart("youare", targetHash));
//
// // time information for synchronization
// post.add(new DefaultCharsetStringPart("mytime", DateFormatter.formatShortSecond(new Date())));
// post.add(new DefaultCharsetStringPart("myUTC", Long.toString(System.currentTimeMillis())));
//
// // network identification
// post.add(new DefaultCharsetStringPart(SwitchboardConstants.NETWORK_NAME, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT)));
//
// // authentication essentials
// final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
// final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", "");
// if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0)) {
// if (authenticationMethod.equals("salted-magic-sim")) {
// // generate an authentication essential using the salt, the iam-hash and the network magic
// final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
// final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
// post.add(new DefaultCharsetStringPart("magicmd5", md5));
// }
// }
//
// return post;
// }
public static final LinkedHashMap<String,ContentBody> basicRequestParts(final Switchboard sb, final String targetHash, final String salt) throws UnsupportedEncodingException {
// put in all the essentials for routing and network authentication

@ -17,6 +17,7 @@ import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
@ -25,11 +26,10 @@ import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
@ -46,9 +46,9 @@ import org.apache.http.util.EntityUtils;
*/
public class Client {
private final static int maxcon = 20;
private static IdledConnectionEvictor idledConnectionEvictor = null;
private static HttpClient httpClient = null;
private static int count = 0;
private int timeout = 10000;
private String userAgent = null;
private String host = null;
@ -67,12 +67,14 @@ public class Client {
* ConnectionManager settings
*/
// TODO: how much connections do we need? - default: 20
// ConnManagerParams.setMaxTotalConnections(httpParams, 100);
ConnManagerParams.setMaxTotalConnections(httpParams, maxcon);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxcon);
// perhaps we need more than 2(default) connections per host?
ConnPerRouteBean connPerRoute = new ConnPerRouteBean(2);
// Increase max connections for localhost to 100
HttpHost localhost = new HttpHost("locahost");
connPerRoute.setMaxForRoute(new HttpRoute(localhost), 100);
connPerRoute.setMaxForRoute(new HttpRoute(localhost), maxcon);
ConnManagerParams.setMaxConnectionsPerRoute(httpParams, connPerRoute);
// how long to wait for getting a connection from manager in milliseconds
ConnManagerParams.setTimeout(httpParams, 3000L);
@ -88,7 +90,7 @@ public class Client {
// timeout in milliseconds until a connection is established in milliseconds
HttpConnectionParams.setConnectionTimeout(httpParams, 10000);
// SO_LINGER affects the socket close operation in seconds
HttpConnectionParams.setLinger(httpParams, 6);
// HttpConnectionParams.setLinger(httpParams, 6);
// TODO: is default ok?
// HttpConnectionParams.setSocketBufferSize(httpParams, 8192);
// SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds
@ -97,6 +99,7 @@ public class Client {
HttpConnectionParams.setStaleCheckingEnabled(httpParams, true);
// conserve bandwidth by minimizing the number of segments that are sent
HttpConnectionParams.setTcpNoDelay(httpParams, false);
// TODO: testing noreuse - there will be HttpConnectionParams.setSoReuseaddr(HttpParams params, boolean reuseaddr) in core-4.1
// Create and initialize scheme registry
SchemeRegistry schemeRegistry = new SchemeRegistry();
@ -151,15 +154,6 @@ public class Client {
this.host = host;
}
/**
* number of active connections
*
* @return number of active connections
*/
public static int connectionCount() {
return count;
}
/**
* This method GETs a page from the server.
*
@ -204,16 +198,12 @@ public class Client {
}
private byte[] getContentBytes(HttpUriRequest httpUriRequest, long maxBytes) throws IOException {
count++;
byte[] content = null;
final HttpContext httpContext = new BasicHttpContext();
httpUriRequest.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout);
httpUriRequest.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, timeout);
if (userAgent != null)
httpUriRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
if (host != null)
httpUriRequest.getParams().setParameter(HTTP.TARGET_HOST, host);
setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams());
// statistics
storeConnectionInfo(httpUriRequest);
try {
// execute the method
HttpResponse httpResponse = httpClient.execute(httpUriRequest, httpContext);
@ -229,13 +219,40 @@ public class Client {
}
} catch (final IOException e) {
httpUriRequest.abort();
count--;
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
throw e;
}
count--;
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
return content;
}
private void setParams(HttpParams httpParams) {
HttpConnectionParams.setConnectionTimeout(httpParams, timeout);
HttpConnectionParams.setSoTimeout(httpParams, timeout);
if (userAgent != null)
HttpProtocolParams.setUserAgent(httpParams, userAgent);
if (host != null)
httpParams.setParameter(HTTP.TARGET_HOST, host);
}
private void setProxy(HttpParams httpParams) {
if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
}
private void storeConnectionInfo(HttpUriRequest httpUriRequest) {
int port = httpUriRequest.getURI().getPort();
String thost = httpUriRequest.getURI().getHost();
ConnectionInfo.addConnection(new ConnectionInfo(
httpUriRequest.getURI().getScheme(),
port == 80 ? thost : thost + ":" + port,
httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(),
httpUriRequest.hashCode(),
System.currentTimeMillis()));
}
/**
* provide system information for client identification
*/

@ -0,0 +1,255 @@
// HttpConnectionInfo.java
// (C) 2008 by Daniel Raap; danielr@users.berlios.de
// first published 07.04.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.cora.protocol;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
/**
* Information about a connection
*
* @author daniel
* @author sixcooler
*/
public class ConnectionInfo {
/**
* a list of all current connections to be shown in Connections_p
*/
private final static Set<ConnectionInfo> allConnections = Collections
.synchronizedSet(new HashSet<ConnectionInfo>());
// this is only for statistics, so it can be bigger to see lost connectionInfos
private final static int staleAfterMillis = 30 * 60000; // 30 minutes
private static int maxcount = 20;
private final String protocol;
private final String targetHost;
private final String command;
private final int id;
private final long initTime;
/**
* constructor setting all data
*
* @param protocol
* @param targetHost
* @param command
* @param id
* @param initTime
*/
public ConnectionInfo(final String protocol, final String targetHost, final String command, final int id,
final long initTime) {
this.protocol = protocol;
this.targetHost = targetHost;
this.command = command;
this.id = id;
this.initTime = initTime;
}
/**
* @return
*/
public String getProtocol() {
return protocol;
}
/**
* @return
*/
public long getLifetime() {
return System.currentTimeMillis() - initTime;
}
/**
* @return dummy 0
*/
public int getIdletime() {
return 0;
}
/**
* @return
*/
public String getCommand() {
return command;
}
/**
* @return
*/
public String getTargetHost() {
return targetHost;
}
/**
* @return
*/
public int getID() {
return id;
}
/**
* gets a {@link Set} of all collected ConnectionInfos
*
* Important: iterations must be synchronized!
*
* @return the allConnections
*/
public static Set<ConnectionInfo> getAllConnections() {
return allConnections;
}
/**
* gets the number of active client connections
*
* @return count of active connections
*/
public static int getCount() {
return allConnections.size();
}
/**
* gets the usage of the Client connection manager by active connections
*
* @return load in percent
*/
public static int getLoadPercent() {
return getCount() * 100 / maxcount;
}
/**
* gets the max connection count of the Client connection manager
*
* @return max connections
*/
public static int getMaxcount() {
return maxcount;
}
/**
* gets the max connection count of the Client connection manager
* to be used in statistics
*
* @param max connections
*/
protected static void setMaxcount(final int max) {
if (max > 0) maxcount = max;
}
/**
* add a connection to the list of all current connections
*
* @param conInfo
*/
protected static void addConnection(final ConnectionInfo conInfo) {
allConnections.add(conInfo);
}
/**
* remove a connection from the list of all current connections
*
* @param conInfo
*/
protected static void removeConnection(final ConnectionInfo conInfo) {
allConnections.remove(conInfo);
}
/**
* connections with same id {@link equals()} another
*
* @param id
*/
protected static void removeConnection(final int id) {
removeConnection(new ConnectionInfo(null, null, null, id, 0));
}
/**
* removes stale connections
*/
public static void cleanUp() {
try {
synchronized (allConnections) {
for(final ConnectionInfo con: allConnections) {
if(con.getLifetime() > staleAfterMillis) {
allConnections.remove(con);
}
}
}
} catch (final java.util.ConcurrentModificationException e) {
// there will be another try :-)
}
}
/* (non-Javadoc)
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
final StringBuilder string = new StringBuilder(50);
string.append("ID ");
string.append(getID());
string.append(", ");
string.append(getProtocol());
string.append("://");
string.append(getTargetHost());
string.append(" ");
string.append(getCommand());
string.append(", since ");
string.append(getLifetime());
string.append(" ms");
return string.toString();
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + id;
return result;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(final Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
final ConnectionInfo other = (ConnectionInfo) obj;
return this.id == other.id;
}
}

@ -25,6 +25,10 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.AbstractHttpClient;
/**
* settings for a remote proxy
@ -55,6 +59,22 @@ public final class ProxySettings {
return hostConfig;
}
/**
*
* @return the HttpHost to be used as proxy
*/
public static HttpHost getProxyHost() {
if (!use) return null;
return new HttpHost(host, port);
}
public static void setProxyCreds(AbstractHttpClient httpClient) {
if (!use) return;
httpClient.getCredentialsProvider().setCredentials(
new AuthScope(host, port),
new UsernamePasswordCredentials(user, password));
}
/**
* tell if a remote proxy will be used for the given host
* @param host

Loading…
Cancel
Save