- increased default peer ping time from 2 minutes to 1 minute

- filtering out too old peers when reading seed lists (limit is now 240 minutes)
- added concurrent host names resolving in front of the http client because the http client uses the java built-in DNS resolve which is not multithreading-safe (i have seen deadlocks in thread dumps showing that this bug in jdk is still there)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7515 6c8d7289-2bf4-0310-a012-ef5d649a1542

defaults/yacy.init

@@ -550,8 +550,8 @@ filterOutStopwordsFromTopwords=true
 20_dhtdistribution_idlesleep=15000
 20_dhtdistribution_busysleep=10000
 20_dhtdistribution_memprereq=12582912
-30_peerping_idlesleep=120000
+30_peerping_idlesleep=60000
-30_peerping_busysleep=120000
+30_peerping_busysleep=60000
 30_peerping_memprereq=2097152
 40_peerseedcycle_idlesleep=1800000
 40_peerseedcycle_busysleep=1200000

source/de/anomic/search/Switchboard.java

@@ -2433,7 +2433,7 @@ public final class Switchboard extends serverSwitch {
         
         yacySeed           ys;
         String             seedListFileURL;
-        DigestURI            url;
+        DigestURI          url;
         Iterator<String>   enu;
         int                lc;
         final int          sc = peers.sizeConnected();
@@ -2487,8 +2487,11 @@ public final class Switchboard extends serverSwitch {
                             try {
                                 ys = yacySeed.genRemoteSeed(enu.next(), null, false);
                                 if ((ys != null) &&
-                                        ((!peers.mySeedIsDefined()) || !peers.mySeed().hash.equals(ys.hash))) {
+                                    (!peers.mySeedIsDefined() || !peers.mySeed().hash.equals(ys.hash))) {
-                                        if (peers.peerActions.connectPeer(ys, false)) lc++;
+                                        long lastseen = Math.abs((System.currentTimeMillis() - ys.getLastSeenUTC()) / 1000 / 60);
+                                        if (lastseen < 240) {
+                                            if (peers.peerActions.connectPeer(ys, false)) lc++;
+                                        }
                                     }
                             } catch (IOException e) {
                                 yacyCore.log.logInfo("BOOTSTRAP: bad seed: " + e.getMessage());

source/net/yacy/cora/protocol/http/HTTPClient.java

@@ -270,7 +270,9 @@ public class HTTPClient {
      * @throws IOException 
      */
     public byte[] GETbytes(final String uri, long maxBytes) throws IOException {
-    	final HttpGet httpGet = new HttpGet(uri);
+        MultiProtocolURI url = new MultiProtocolURI(uri);
+    	final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false));
+    	httpGet.setHeader("Host", url.getHost()); // overwrite resolved IP
     	return getContentBytes(httpGet, maxBytes);
     }
     
@@ -284,7 +286,9 @@ public class HTTPClient {
      */
     public void GET(final String uri) throws IOException {
         if (currentRequest != null) throw new IOException("Client is in use!");
-        final HttpGet httpGet = new HttpGet(uri);
+        MultiProtocolURI url = new MultiProtocolURI(uri);
+        final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false));
+        httpGet.setHeader("Host", url.getHost()); // overwrite resolved IP
         currentRequest = httpGet;
         execute(httpGet);
     }
@@ -297,7 +301,9 @@ public class HTTPClient {
      * @throws IOException
      */
     public HttpResponse HEADResponse(final String uri) throws IOException {
-    	final HttpHead httpHead = new HttpHead(uri);
+        MultiProtocolURI url = new MultiProtocolURI(uri);
+        final HttpHead httpHead = new HttpHead(url.toNormalform(true, false, true, false));
+        httpHead.setHeader("Host", url.getHost()); // overwrite resolved IP
     	execute(httpHead);
     	finish();
     	ConnectionInfo.removeConnection(httpHead.hashCode());
@@ -316,7 +322,9 @@ public class HTTPClient {
      */
     public void POST(final String uri, final InputStream instream, long length) throws IOException {
     	if (currentRequest != null) throw new IOException("Client is in use!");
-    	final HttpPost httpPost = new HttpPost(uri);
+        MultiProtocolURI url = new MultiProtocolURI(uri);
+        final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
+        httpPost.setHeader("Host", url.getHost()); // overwrite resolved IP
     	final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length);
     	// statistics
     	upbytes = length;
@@ -334,7 +342,9 @@ public class HTTPClient {
      * @throws IOException 
      */
     public byte[] POSTbytes(final String uri, final Map<String, ContentBody> parts, final boolean usegzip) throws IOException {
-    	final HttpPost httpPost = new HttpPost(uri);
+        MultiProtocolURI url = new MultiProtocolURI(uri);
+        final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
+        httpPost.setHeader("Host", url.getHost()); // overwrite resolved IP
 
     	final MultipartEntity multipartEntity = new MultipartEntity();
     	for (final Entry<String,ContentBody> part : parts.entrySet())
@@ -355,7 +365,7 @@ public class HTTPClient {
 	 * 
 	 * @return HttpResponse from call
 	 */
-	public HttpResponse getHttpResponse() {
+    public HttpResponse getHttpResponse() {
 		return httpResponse;
 	}
 	
@@ -363,7 +373,7 @@ public class HTTPClient {
 	 * 
 	 * @return status code from http request
 	 */
-	public int getStatusCode() {
+    public int getStatusCode() {
 	    return httpResponse.getStatusLine().getStatusCode();
 	}
 	
@@ -375,7 +385,7 @@ public class HTTPClient {
      * @return the content as InputStream
      * @throws IOException
      */
-    public InputStream getContentstream() throws IOException {
+	public InputStream getContentstream() throws IOException {
         if (httpResponse != null && currentRequest != null) {
             final HttpEntity httpEntity = httpResponse.getEntity();
             if (httpEntity != null) try {
@@ -397,7 +407,7 @@ public class HTTPClient {
      * @param outputStream
      * @throws IOException
      */
-    public void writeTo(final OutputStream outputStream) throws IOException {
+	public void writeTo(final OutputStream outputStream) throws IOException {
         if (httpResponse != null && currentRequest != null) {
             final HttpEntity httpEntity = httpResponse.getEntity();
             if (httpEntity != null) try {
@@ -625,7 +635,7 @@ public class HTTPClient {
 	 * @see: http://hc.apache.org/httpcomponents-client-4.0.1/tutorial/html/connmgmt.html#d4e638
 	 *
 	 */
-	public static class IdledConnectionEvictor extends Thread {
+	private static class IdledConnectionEvictor extends Thread {
 
 		private final ClientConnectionManager clientConnectionManager;
 

source/net/yacy/cora/protocol/http/HTTPConnector.java

@@ -80,7 +80,7 @@ public class HTTPConnector {
 		client.setUserAgent(this.userAgent);
 		client.setHost(vhost);
 		
-		return client.POSTbytes(url.toNormalform(false, false, true, false), post, usegzip);
+		return client.POSTbytes(url.toNormalform(true, false, true, false), post, usegzip);
 	}
 
 }