- reverted svn 4132; this did not solve the problem and removed the emergency mehtod which caused production failure for shure within some hours

- removed and added some debugging lines

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4133 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/httpc.java

@@ -227,10 +227,14 @@ public final class httpc {
             String outgoingByteCountAccounting            
     ) throws IOException {
         
-        // TODO method can be removed completely when no more CLOSE_WAIT connections appear.
-    	// remove old connections
-    	//checkIdleConnections();
+        // remove old connections
+        // do NOT remove this check; in case that everything works fine this call does nothing
+        // but if in any arror case connections stay open, this will ensure that the peer keeps running and the host server is not blocked from working
+    	checkIdleConnections();
     	
+        // register new connection
+        this.hashIndex = objCounter;
+        
     	// register new connection
     	this.hashIndex = objCounter;
     	objCounter++;
@@ -387,13 +391,15 @@ public final class httpc {
                 address = new InetSocketAddress(hostip, port);
             } else {
                 address = new InetSocketAddress(server,port);
-            }            
-
+            }
+            
             // trying to establish a connection to the address
             this.initTime = System.currentTimeMillis();
-            this.socket.setKeepAlive(false); //
+            this.socket.setKeepAlive(false);
             this.socket.connect(address, timeout);
-
+            // setting socket timeout and keep alive behaviour
+            this.socket.setSoTimeout(timeout); // waiting time for read
+            
             if (incomingByteCountAccounting != null) {
                 this.clientInputByteCount = new httpdByteCountInputStream(this.socket.getInputStream(),incomingByteCountAccounting);
             }
@@ -434,7 +440,6 @@ public final class httpc {
         return (this.clientOutputByteCount == null)?0:this.clientOutputByteCount.getCount();
     }
 
-    // TODO remove when it's sure that the CLOSE_WAIT problem was solved.
     public static int checkIdleConnections() {
         // try to find and close all connections that did not find a target server and are idle waiting for a server socket
         
@@ -514,7 +519,7 @@ public final class httpc {
     
     public void close() {
     	synchronized (activeConnections) {activeConnections.remove(this);}
-    	System.out.println("*** DEBUG close httpc: " + activeConnections.size() + " connections online");
+    	//System.out.println("*** DEBUG close httpc: " + activeConnections.size() + " connections online");
     	
         if (this.clientInput != null) {
             try {this.clientInput.close();} catch (Exception e) {}

source/de/anomic/http/httpdProxyHandler.java

@@ -861,7 +861,7 @@ public final class httpdProxyHandler {
         }
     }
     
-    public static void doHead(Properties conProp, httpHeader requestHeader, OutputStream respond) throws IOException {
+    public static void doHead(Properties conProp, httpHeader requestHeader, OutputStream respond) {
         
         httpc remote = null;
         httpc.response res = null;

source/de/anomic/plasma/plasmaSearchEvent.java

@@ -362,24 +362,24 @@ public final class plasmaSearchEvent {
             (!(comp.title().startsWith("Index of")))) {
             final Iterator wi = query.queryHashes.iterator();
             while (wi.hasNext()) wordIndex.removeEntry((String) wi.next(), page.hash());
-            registerFailure(page.hash(), "index-of constrained not fullfilled");
+            registerFailure(page.hash(), "index-of constraint not fullfilled");
             return null;
         }
         
         if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_AUDIO) && (page.laudio() == 0)) {
-            registerFailure(page.hash(), "contentdom-audio constrained not fullfilled");
+            registerFailure(page.hash(), "contentdom-audio constraint not fullfilled");
             return null;
         }
         if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_VIDEO) && (page.lvideo() == 0)) {
-            registerFailure(page.hash(), "contentdom-video constrained not fullfilled");
+            registerFailure(page.hash(), "contentdom-video constraint not fullfilled");
             return null;
         }
         if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_IMAGE) && (page.limage() == 0)) {
-            registerFailure(page.hash(), "contentdom-image constrained not fullfilled");
+            registerFailure(page.hash(), "contentdom-image constraint not fullfilled");
             return null;
         }
         if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_APP) && (page.lapp() == 0)) {
-            registerFailure(page.hash(), "contentdom-app constrained not fullfilled");
+            registerFailure(page.hash(), "contentdom-app constraint not fullfilled");
             return null;
         }
             
@@ -389,6 +389,7 @@ public final class plasmaSearchEvent {
             startTime = System.currentTimeMillis();
             plasmaSnippetCache.TextSnippet snippet = plasmaSnippetCache.retrieveTextSnippet(comp.url(), snippetFetchWordHashes, fetchSnippetOnline, query.constraint.get(plasmaCondenser.flag_cat_indexof), 180, 3000, (fetchSnippetOnline) ? Integer.MAX_VALUE : 100000);
             long snippetComputationTime = System.currentTimeMillis() - startTime;
+            serverLog.logInfo("SEARCH_EVENT", "text snippet load time for " + comp.url() + ": " + snippetComputationTime);
             
             if (snippet.getErrorCode() < 11) {
                 // we loaded the file and found the snippet
@@ -408,6 +409,7 @@ public final class plasmaSearchEvent {
             startTime = System.currentTimeMillis();
             ArrayList mediaSnippets = plasmaSnippetCache.retrieveMediaSnippets(comp.url(), snippetFetchWordHashes, query.contentdom, fetchSnippetOnline, 6000);
             long snippetComputationTime = System.currentTimeMillis() - startTime;
+            serverLog.logInfo("SEARCH_EVENT", "media snippet load time for " + comp.url() + ": " + snippetComputationTime);
             
             if ((mediaSnippets != null) && (mediaSnippets.size() > 0)) {
                 // found media snippets, return entry

source/de/anomic/server/serverFileUtils.java

@@ -122,7 +122,7 @@ public final class serverFileUtils {
             if (procOS != null) procOS.write(buffer, 0, l);
             if (bufferOS != null) bufferOS.write(buffer, 0, l);
         } catch (IOException e) {
-            System.out.println("** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes.");
+            //System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes.");
             break;
         }
         
@@ -143,7 +143,7 @@ public final class serverFileUtils {
             if (procOS != null) procOS.write(buffer, 0, l);
             if (bufferOS != null) bufferOS.write(buffer, 0, l);
         } catch (IOException e) {
-            System.out.println("** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes.");
+            //System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes.");
             break;
         }