*) httpc.java:

- Now it's possible to interrupt pending httpc-actions on server shutdown  
   - this is possible because of a newly introduced registration mechanism for
     open sockets
*) yacyCore.java
   - blocking peerPing threads can now be interrupted on server shutdown
*) serverCore.java
   - restructuring shutdown code 
*) error.html
   - port number is now set correctly if port forwarding was enabled


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@389 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/proxymsg/error.html

@@ -45,15 +45,15 @@
 			<tt>#[httpStatus]#</tt>
 			<p>
 				<b>
-					#(errorMessageType)#
+					#(errorMessageType)#<!-- 0 -->
 						unspecified error
-						::
+						::<!-- 1 -->
 						not-yet-assigned error
-						::
+						::<!-- 2 -->
 						You don't have an active internet connection. Please go online.
-						::
+						::<!-- 3 -->
 						Could not load resource. The file is not available.
-						::
+						::<!-- 4 -->
 						#[detailedErrorMsg]#
 					#(/errorMessageType)#
 				</b>

source/de/anomic/http/httpc.java

@@ -63,6 +63,7 @@ import java.net.SocketException;
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.Enumeration;
 import java.util.GregorianCalendar;
@@ -113,12 +114,13 @@ public final class httpc {
     
     // class variables
     private Socket socket = null; // client socket for commands
+    private Long socketOwnerID = null;
     private String host = null;
     private long timeout;
     private long handle;
     
     // output and input streams for client control connection
-    private PushbackInputStream clientInput = null;
+    PushbackInputStream clientInput = null;
     private OutputStream clientOutput = null;
     
     private boolean remoteProxyUse = false;
@@ -163,6 +165,8 @@ public final class httpc {
      * @see serverByteBuffer
      */
     final serverByteBuffer readLineBuffer = new serverByteBuffer(100);
+        
+    private static final Hashtable openSocketLookupTable = new Hashtable();
     
     public String toString() {
         return (this.savedRemoteHost == null) ? "Disconnected" : "Connected to " + this.savedRemoteHost +
@@ -316,19 +320,31 @@ public final class httpc {
                 hostip = dnsResolve(server);
                 if (hostip == null) throw new UnknownHostException(server);
             }
-            if (ssl)
-                socket = SSLSocketFactory.getDefault().createSocket(hostip, port);
-            else
-                socket = new Socket(hostip, port);
+            
+            // opening the socket
+            socket = (ssl) ? SSLSocketFactory.getDefault().createSocket(hostip, port)
+                    : new Socket(hostip, port);
+            
+            // registering the socket
+            this.socketOwnerID = this.registerOpenSocket(socket);
+            
+            // setting socket timeout and keep alive behaviour
             socket.setSoTimeout(timeout); // waiting time for write
             //socket.setSoLinger(true, timeout); // waiting time for read
             socket.setKeepAlive(true); //
+            
+            // getting input and output streams
             clientInput  = new PushbackInputStream(socket.getInputStream());
             clientOutput = socket.getOutputStream();
             // if we reached this point, we should have a connection
         } catch (UnknownHostException e) {
+            if (this.socket != null) {
+                this.unregisterOpenSocket(this.socket,this.socketOwnerID);
+            }
+            this.socket = null;
+            this.socketOwnerID = null;
             throw new IOException("unknown host: " + server);
-        }
+        } 
     }
     
     void reset() {
@@ -342,7 +358,9 @@ public final class httpc {
         }
         if (this.socket != null) {
             try {this.socket.close();} catch (Exception e) {}
+            this.unregisterOpenSocket(this.socket,this.socketOwnerID);
             this.socket = null;
+            this.socketOwnerID = null;
         }
         
         this.host = null;
@@ -1085,8 +1103,109 @@ do upload
         while (i.hasMoreElements()) System.out.println((String) i.nextElement());
     }
     
+    /**
+     * To register an open socket.
+     * This adds the socket to the list of open sockets where the current thread 
+     * is is the owner. 
+     * @param openedSocket the socket that should be registered
+     * @return the id of the current thread 
+     */
+    private Long registerOpenSocket(Socket openedSocket) {
+        Long currentThreadId = new Long(Thread.currentThread().getId());
+        synchronized (openSocketLookupTable) {
+            ArrayList openSockets = null;
+            if (openSocketLookupTable.containsKey(currentThreadId)) {
+                openSockets = (ArrayList) openSocketLookupTable.get(currentThreadId);
+            } else {
+                openSockets = new ArrayList(1);
+                openSocketLookupTable.put(currentThreadId,openSockets);
+            }
+            synchronized (openSockets) {
+                openSockets.add(openedSocket);
+            }
+            return currentThreadId;
+        }        
+    }
     
+    /**
+     * Closing all sockets that were opened in the context of the thread
+     * with the given thread id 
+     * @param threadId
+     */
+    public static int closeOpenSockets(Long threadId) {
+        
+        // getting all still opened sockets 
+        ArrayList openSockets = httpc.getRegisteredOpenSockets(threadId);
+        int closedSocketCount = 0;
+        
+        synchronized (openSockets) {        
+            // looping through the list of sockets and close each one
+            for (int socketCount = 0; socketCount < openSockets.size(); socketCount++) {
+                Socket openSocket = (Socket) openSockets.get(0);
+                try {
+                    // closing the socket
+                    if (!openSocket.isClosed()) {
+                        openSocket.close();
+                        closedSocketCount++;
+                    }
+                    // unregistering the socket
+                    httpc.unregisterOpenSocket(openSocket,threadId);
+                } catch (Exception ex) {}
+            }
+        }
+        
+        return closedSocketCount;
+    }
     
+    /**
+     * Unregistering the socket. 
+     * The socket will be removed from the list of sockets where the thread with the
+     * given thread id is the owner.
+     * @param closedSocket the socket that should be unregistered
+     * @param threadId the id of the owner thread
+     */
+    public static void unregisterOpenSocket(Socket closedSocket, Long threadId) {
+        synchronized (openSocketLookupTable) {
+            ArrayList openSockets = null;
+            if (openSocketLookupTable.containsKey(threadId)) {
+                openSockets = (ArrayList) openSocketLookupTable.get(threadId);
+                synchronized (openSockets) {
+                    openSockets.remove(closedSocket);
+                    if (openSockets.size() == 0) {
+                        openSocketLookupTable.remove(threadId);
+                    }
+                }
+            }
+        }          
+    }
+    
+    /**
+     * Getting a list of open sockets where the current thread is
+     * the owner 
+     * @return the list of open sockets
+     */
+    public static ArrayList getRegisteredOpenSockets() {
+        Long currentThreadId = new Long(Thread.currentThread().getId());
+        return getRegisteredOpenSockets(currentThreadId);      
+    }
+    
+    /**
+     * Getting a list of open sockets where the thread with the given
+     * thread id is the owner
+     * @param threadId the thread id of the owner thread
+     * @return the list of open sockets
+     */
+    public static ArrayList getRegisteredOpenSockets(Long threadId) {
+        synchronized (openSocketLookupTable) {
+            ArrayList openSockets = null;
+            if (openSocketLookupTable.containsKey(threadId)) {
+                openSockets = (ArrayList) openSocketLookupTable.get(threadId);
+            } else {
+                openSockets = new ArrayList(0);
+            }
+            return openSockets;
+        }        
+    }    
     
 }
 

source/de/anomic/http/httpd.java

@@ -1072,10 +1072,8 @@ public final class httpd implements serverHandler {
     ) throws IOException {
         
         FileInputStream fis = null;
+        ByteArrayOutputStream o = null;
         try {
-            
-            File htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot"));
-            
             // setting the proper http status message
             String httpVersion = conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER,"HTTP/1.1");
             if ((httpStatusText == null)||(httpStatusText.length()==0)) {
@@ -1107,8 +1105,12 @@ public final class httpd implements serverHandler {
             // set rewrite values
             serverObjects tp = new serverObjects();
             
+//            tp.put("host", serverCore.publicIP().getHostAddress());
+//            tp.put("port", switchboard.getConfig("port", "8080"));            
             tp.put("host", serverCore.publicIP().getHostAddress());
-            tp.put("port", switchboard.getConfig("port", "8080"));
+            tp.put("port", (serverCore.portForwardingEnabled && (serverCore.portForwarding != null)) 
+                           ? Integer.toString(serverCore.portForwarding.getPort()) 
+                           : switchboard.getConfig("port", "8080"));
             
             tp.put("errorMessageType", errorcase);            
             tp.put("httpStatus",       Integer.toString(httpStatusCode) + " " + httpStatusText);
@@ -1117,27 +1119,32 @@ public final class httpd implements serverHandler {
             tp.put("errorMessageType_detailedErrorMsg",(detailedErrorMsg != null) ? detailedErrorMsg : "");
             
             // building the stacktrace            
-            if (stackTrace != null) {    
+            if (stackTrace != null) {  
+                tp.put("printStackTrace",1);
+                
                 serverByteBuffer errorMsg = new serverByteBuffer(100);
-                errorMsg.append("Exception occurred:\r\n\r\n")
+                errorMsg.append("<i>Exception occurred:</i>&nbsp;<b>")
                         .append(stackTrace.toString())
-                        .append("\r\n")
-                        .append("TRACE: ");
+                        .append("</b>\r\n\r\n")
+                        .append("</i>TRACE:</i>\r\n");
                 stackTrace.printStackTrace(new PrintStream(errorMsg));
-                errorMsg.write(("\r\n").getBytes());
-                tp.put("printStackTrace",1);
+                errorMsg.append("\r\n");
+                
                 tp.put("printStackTrace_stacktrace",errorMsg.toString().replaceAll("\n","<br>"));
             } else {
                 tp.put("printStackTrace",0);
             }
             
             // rewrite the file
-            File file = new File(htRootPath, "/proxymsg/error.html");
-            byte[] result;
-            ByteArrayOutputStream o = new ByteArrayOutputStream();
-            fis = new FileInputStream(file);
-            httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
-            result = o.toByteArray();
+            File htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot"));
+            
+            httpTemplate.writeTemplate(
+                    fis = new FileInputStream(new File(htRootPath, "/proxymsg/error.html")), 
+                    o = new ByteArrayOutputStream(), 
+                    tp, 
+                    "-UNRESOLVED_PATTERN-".getBytes()
+            );
+            byte[] result = o.toByteArray();
             o.close(); o = null;
 
             httpHeader header = new httpHeader();            
@@ -1154,6 +1161,7 @@ public final class httpd implements serverHandler {
             throw new IOException(e.getMessage());
         } finally {
             if (fis != null) try { fis.close(); } catch (Exception e) {}
+            if (o != null)   try { o.close();   } catch (Exception e) {}
         }     
     }
     

source/de/anomic/http/httpdAbstractHandler.java

@@ -52,21 +52,28 @@
 package de.anomic.http;
 
 import java.text.SimpleDateFormat;
+import java.util.Properties;
 
-public abstract class httpdAbstractHandler {
+import de.anomic.server.logging.serverLog;
 
+public abstract class httpdAbstractHandler {
+    
     // static tools
-
+    
     private static int fileCounter = 0; // for unique file names
     
     private static SimpleDateFormat DateFileNameFormatter =
-	new SimpleDateFormat("yyyyMMddHHmmss");
-
+        new SimpleDateFormat("yyyyMMddHHmmss");
+    
     protected static String uniqueDateString() {
-	String c = "" + fileCounter;
-	fileCounter++; if (fileCounter>9999) fileCounter = 0;
-	while (c.length() < 4) { c = "0" + c; }
-	return "FILE" + DateFileNameFormatter.format(httpc.nowDate()) + c;
+        String c = "" + fileCounter;
+        fileCounter++; if (fileCounter>9999) fileCounter = 0;
+        while (c.length() < 4) { c = "0" + c; }
+        return "FILE" + DateFileNameFormatter.format(httpc.nowDate()) + c;
     } 
     
+    protected Properties connectionProperties = null;
+    
+    protected serverLog theLogger;
+    
 }

source/de/anomic/http/httpdFileHandler.java

@@ -117,14 +117,14 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
     private serverSwitch switchboard;
     private String adminAccountBase64MD5;
     
-    private Properties connectionProperties = null;    
     private MessageDigest md5Digest = null;
     
-    private final serverLog theLogger = new serverLog("FILEHANDLER");
-    
     public httpdFileHandler(serverSwitch switchboard) {
         this.switchboard = switchboard;
         
+        // creating a logger
+        this.theLogger = new serverLog("FILEHANDLER");
+        
         if (this.mimeTable == null) {
             // load the mime table
             this.mimeTable = new Properties();
@@ -476,37 +476,58 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
                 serverFileUtils.write(result, out);
             } else {
                 httpd.sendRespondError(conProp,out,3,404,"File not Found",null,null);
-                //textMessage(out, 404, "404 File not Found\r\n"); // would be a possible vuln to return original the original path
+                return;
             }
         } catch (Exception e) {     
-            if (e instanceof InterruptedException) {
-                this.theLogger.logInfo("Interruption detected while processing query: " + path + 
-                                       "\nClient: " + conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP,"unknown") + 
-                                       "\nReason: " + e.toString());
-                if (!conProp.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
-                    httpd.sendRespondError(conProp,out, 4, 503, null, "Exception with query: " + path + "; Service unavailable because of server shutdown.",e);
-                } else {
-                    conProp.put(httpd.CONNECTION_PROP_PERSISTENT,"close");
-                }
-            } else {
-                String errorMsg = e.getMessage();
-                if ((errorMsg != null) && (errorMsg.startsWith("Broken pipe") || errorMsg.startsWith("Connection reset"))) {
-                    // client closed the connection, so we just end silently
-                    this.theLogger.logInfo("Client unexpectedly closed connection while processing query " + path + 
-                                           "\nClient: " + conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP,"unknown")+ 
-                                           "\nReason: " + e.toString());
-                    conProp.put(httpd.CONNECTION_PROP_PERSISTENT,"close");
+            try {
+                // doing some errorhandling ...
+                int httpStatusCode = 400; 
+                String httpStatusText = null; 
+                StringBuffer errorMessage = new StringBuffer(); 
+                Exception errorExc = null;            
+                
+                if (e instanceof InterruptedException) {
+                    errorMessage.append("Interruption detected while processing query.");
+                    httpStatusCode = 503;
                 } else {
-                    this.theLogger.logError("ERROR: Exception with query: " + path + 
-                                            "\nClient: " + conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP,"unknown") + 
-                                            "\nReason: " + e.toString());
-                    if (!conProp.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
-                        httpd.sendRespondError(conProp,out, 4, 503, null, "Exception with query: " + path + "; '" + e.toString() + ":" + ((errorMsg ==null)?"":e.getMessage()) + "'",e);
+                    String errorMsg = e.getMessage();
+                    if ((errorMsg != null) && 
+                        (
+                           errorMsg.startsWith("Broken pipe") || 
+                           errorMsg.startsWith("Connection reset") ||
+                           errorMsg.startsWith("Software caused connection abort")
+                       )) {
+                        // client closed the connection, so we just end silently
+                        errorMessage.append("Client unexpectedly closed connection while processing query.");
                     } else {
-                        conProp.put(httpd.CONNECTION_PROP_PERSISTENT,"close");
+                        errorMessage.append("Unexpected error while processing query.");
+                        httpStatusCode = 500;
+                        errorExc = e;
                     }
                 }
-            }
+                
+                errorMessage.append("\nQuery:  ").append(path)
+                            .append("\nClient: ").append(conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP,"unknown")) 
+                            .append("\nReason: ").append(e.toString());    
+                
+                if (!conProp.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
+                    // sending back an error message to the client 
+                    // if we have not already send an http header
+                    httpd.sendRespondError(conProp,out, 4, httpStatusCode, httpStatusText, errorMessage.toString(),errorExc);
+                } else {
+                    // otherwise we close the connection
+                    this.forceConnectionClose();
+                }    
+                
+                // if it is an unexpected error we log it 
+                if (httpStatusCode == 500) {
+                    this.theLogger.logWarning(errorMessage.toString(),e);
+                }
+                
+            } catch (Exception ee) {
+                this.forceConnectionClose();
+            }            
+            
         } finally {
             try {out.flush();}catch (Exception e) {}
             if (!(requestHeader.get(httpHeader.CONNECTION, "close").equals("keep-alive"))) {
@@ -516,6 +537,12 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
         }
     }
     
+    private void forceConnectionClose() {
+        if (this.connectionProperties != null) {
+            this.connectionProperties.setProperty(httpd.CONNECTION_PROP_PERSISTENT,"close");            
+        }
+    }
+
     private static HashMap loadTemplates(File path) {
         // reads all templates from a path
         // we use only the folder from the given file path

source/de/anomic/http/httpdProxyHandler.java

@@ -126,9 +126,6 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
     public  static final String userAgent = "yacy (" + httpc.systemOST +") yacy.net";
     private File   htRootPath = null;
     
-    private serverLog theLogger;
-    private Properties currentConProp = null;
-    
     private static boolean doAccessLogging = false; 
     /**
      * Do logging configuration for special proxy access log file
@@ -348,12 +345,12 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
      */
     public void doGet(Properties conProp, httpHeader requestHeader, OutputStream respond) throws IOException {
         
-        this.currentConProp = conProp;
+        this.connectionProperties = conProp;
         
         try {
             // remembering the starting time of the request
             Date requestDate = new Date(); // remember the time...
-            this.currentConProp.put(httpd.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime()));
+            this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime()));
             if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
             
             // using an ByteCount OutputStream to count the send bytes (needed for the logfile)
@@ -476,8 +473,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
             try { respond.flush(); } catch (Exception e) {}
             if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish();
             
-            this.currentConProp.put(httpd.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis()));
-            this.currentConProp.put(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount()));
+            this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis()));
+            this.connectionProperties.put(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount()));
             this.logProxyAccess();
         }
     }
@@ -637,7 +634,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
                         cacheEntry.cacheArray = cacheArray;
                         cacheManager.push(cacheEntry); // necessary update, write response header to cache
                         conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS");
-                    }                        
+                    }          
                 } else {
                     // the file is too big to cache it in the ram, or the size is unknown
                     // write to file right here.
@@ -649,14 +646,17 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
                         // totally fresh file
                         cacheEntry.status = plasmaHTCache.CACHE_FILL; // it's an insert
                         cacheManager.push(cacheEntry);
+                        conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
                     } else if (sizeBeforeDelete == cacheFile.length()) {
                         // before we came here we deleted a cache entry
                         cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_BAD;
                         cacheManager.push(cacheEntry); // unnecessary update
+                        conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REF_FAIL_HIT");
                     } else {
                         // before we came here we deleted a cache entry
                         cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD;
                         cacheManager.push(cacheEntry); // necessary update, write response header to cache
+                        conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS");
                     }
                     // beware! all these writings will not fill the cacheEntry.cacheArray
                     // that means they are not available for the indexer (except they are scraped before)
@@ -675,6 +675,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
                     cacheEntry.status = plasmaHTCache.CACHE_STALE_NO_RELOAD;
                     cacheManager.push(cacheEntry);
                 }
+                conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
             }
             
             if (gzippedOut != null) {
@@ -685,9 +686,10 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
                 chunkedOut.flush();
             }
         } catch (Exception e) {
-            // this may happen if the targeted host does not exist or anything with the
-            // remote server was wrong.
-            // in any case, sending a 404 is appropriate
+            // this may happen if 
+            // - the targeted host does not exist 
+            // - anything with the remote server was wrong.
+            // - the client unexpectedly closed the connection ...
             try {
                 
                 // deleting cached content
@@ -713,7 +715,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
                         // just do nothing, we leave it this way
                         this.theLogger.logDebug("ignoring bad gzip trail for URL " + url + " (" + e.getMessage() + ")");
                         this.forceConnectionClose();
-                    } else if ((remote != null)&&(remote.isClosed())) { // TODO: query for broken pipe
+                    } else if ((remote != null)&&(remote.isClosed())) { 
+                        // TODO: query for broken pipe
                         errorMessage = "destination host unexpectedly closed connection";                 
                     } else {
                         errorMessage = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage();
@@ -849,13 +852,13 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
     }
         
     private void forceConnectionClose() {
-        if (this.currentConProp != null) {
-            this.currentConProp.setProperty(httpd.CONNECTION_PROP_PERSISTENT,"close");            
+        if (this.connectionProperties != null) {
+            this.connectionProperties.setProperty(httpd.CONNECTION_PROP_PERSISTENT,"close");            
         }
     }
     
     public void doHead(Properties conProp, httpHeader requestHeader, OutputStream respond) throws IOException {
-        this.currentConProp = conProp;
+        this.connectionProperties = conProp;
         
         String method = conProp.getProperty("METHOD");
         String host = conProp.getProperty("HOST");
@@ -937,12 +940,12 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
     
     public void doPost(Properties conProp, httpHeader requestHeader, OutputStream respond, PushbackInputStream body) throws IOException {
         
-        this.currentConProp = conProp;
+        this.connectionProperties = conProp;
         
         try {
             // remembering the starting time of the request
             Date requestDate = new Date(); // remember the time...
-            this.currentConProp.put(httpd.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime()));
+            this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime()));
             
             // using an ByteCount OutputStream to count the send bytes
             respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpd.CONNECTION_PROP_REQUESTLINE).length() + 2);
@@ -1022,14 +1025,14 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
             respond.flush();
             if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish();
             
-            this.currentConProp.put(httpd.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis()));
-            this.currentConProp.put(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount()));
+            this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis()));
+            this.connectionProperties.put(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount()));
             this.logProxyAccess();
         }
     }
     
     public void doConnect(Properties conProp, de.anomic.http.httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) throws IOException {
-        this.currentConProp = conProp;
+        this.connectionProperties = conProp;
         
         String host = conProp.getProperty("HOST");
         int    port = Integer.parseInt(conProp.getProperty("PORT"));
@@ -1222,8 +1225,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
         this.logMessage.append(' ');        
         
         // Elapsed time
-        Long requestStart = (Long) this.currentConProp.get(httpd.CONNECTION_PROP_REQUEST_START);
-        Long requestEnd =   (Long) this.currentConProp.get(httpd.CONNECTION_PROP_REQUEST_END);
+        Long requestStart = (Long) this.connectionProperties.get(httpd.CONNECTION_PROP_REQUEST_START);
+        Long requestEnd =   (Long) this.connectionProperties.get(httpd.CONNECTION_PROP_REQUEST_END);
         String elapsed = Long.toString(requestEnd.longValue()-requestStart.longValue());
         
         for (int i=0; i<6-elapsed.length(); i++) this.logMessage.append(' ');
@@ -1231,28 +1234,30 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
         this.logMessage.append(' ');
         
         // Remote Host
-        String clientIP = this.currentConProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP);
+        String clientIP = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_CLIENTIP);
         this.logMessage.append(clientIP);
         this.logMessage.append(' ');
         
         // Code/Status
-        String respondStatus = this.currentConProp.getProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_STATUS);        
-        this.logMessage.append("UNKNOWN/");
+        String respondStatus = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_STATUS);
+        String respondCode = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"UNKNOWN");        
+        this.logMessage.append(respondCode);
+        this.logMessage.append("/");
         this.logMessage.append(respondStatus);
         this.logMessage.append(' ');
         
         // Bytes
-        Long bytes = (Long) this.currentConProp.get(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE);
+        Long bytes = (Long) this.connectionProperties.get(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE);
         this.logMessage.append(bytes.toString());
         this.logMessage.append(' ');        
         
         // Method
-        String requestMethod = this.currentConProp.getProperty(httpd.CONNECTION_PROP_METHOD); 
+        String requestMethod = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_METHOD); 
         this.logMessage.append(requestMethod);
         this.logMessage.append(' ');  
         
         // URL
-        String requestURL = this.currentConProp.getProperty(httpd.CONNECTION_PROP_URL); 
+        String requestURL = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_URL); 
         this.logMessage.append(requestURL);
         this.logMessage.append(' ');          
         
@@ -1261,15 +1266,15 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
         this.logMessage.append(' ');
         
         //  Peerstatus/Peerhost
-        String host = this.currentConProp.getProperty(httpd.CONNECTION_PROP_HOST);
+        String host = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_HOST);
         this.logMessage.append("DIRECT/");
         this.logMessage.append(host);    
         this.logMessage.append(' ');
         
         // Type
         String mime = "-";
-        if (this.currentConProp.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
-            httpHeader proxyRespondHeader = (httpHeader) this.currentConProp.get(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER);
+        if (this.connectionProperties.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
+            httpHeader proxyRespondHeader = (httpHeader) this.connectionProperties.get(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER);
             mime = proxyRespondHeader.mime();
             if (mime.indexOf(";") != -1) {
                 mime = mime.substring(0,mime.indexOf(";"));

source/de/anomic/server/serverCore.java

@@ -76,6 +76,7 @@ import javax.net.ssl.SSLServerSocketFactory;
 import org.apache.commons.pool.impl.GenericObjectPool;
 import org.apache.commons.pool.impl.GenericObjectPool.Config;
 
+import de.anomic.http.httpc;
 import de.anomic.http.httpd;
 import de.anomic.server.logging.serverLog;
 import de.anomic.yacy.yacyCore;
@@ -405,7 +406,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
             this.log.logInfo("SLOWING DOWN ACCESS FOR BRUTE-FORCE PREVENTION FROM " + cIP);
             // add a delay to make brute-force harder
             announceThreadBlockApply();
-            try {Thread.currentThread().sleep(3000);} catch (InterruptedException e) {}
+            try {Thread.sleep(3000);} catch (InterruptedException e) {}
             announceThreadBlockRelease();
         }
         
@@ -434,13 +435,6 @@ public final class serverCore extends serverAbstractThread implements serverThre
             }
         }
         
-        // close the session pool
-        try {
-            this.theSessionPool.close();
-        } catch (Exception e) {
-            this.log.logWarning("Unable to close the session pool.");
-        }
-        
         // closing the serverchannel and socket
         try {
             this.socket.close();
@@ -448,6 +442,13 @@ public final class serverCore extends serverAbstractThread implements serverThre
             this.log.logWarning("Unable to close the server socket."); 
         }
 
+        // closing the session pool
+        try {
+            this.theSessionPool.close();
+        } catch (Exception e) {
+            this.log.logWarning("Unable to close the session pool.");
+        }        
+        
         this.log.logSystem("* terminated");
     }
     
@@ -501,8 +502,6 @@ public final class serverCore extends serverAbstractThread implements serverThre
             /*
              * shutdown all still running session threads ...
              */
-            // interrupting all still running or pooled threads ...
-            serverCore.this.theSessionThreadGroup.interrupt();
             
             /* waiting for all threads to finish */
             int threadCount  = serverCore.this.theSessionThreadGroup.activeCount();    
@@ -515,26 +514,28 @@ public final class serverCore extends serverAbstractThread implements serverThre
                 for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ )  {
                     ((Session)threadList[currentThreadIdx]).setStopped(true);
                 }          
+
+                // interrupting all still running or pooled threads ...
+                serverCore.this.log.logInfo("Sending interruption signal to " + threadCount + " remaining session threads ...");
+                serverCore.this.theSessionThreadGroup.interrupt();                
                 
                 // waiting a frew ms for the session objects to continue processing
                 Thread.sleep(500);
                 
                 // if there are some sessions that are blocking in IO, we simply close the socket
                 for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ )  {
-                    Session currentSession = (Session)threadList[currentThreadIdx];
-                    if (currentSession.isAlive()) {
-                        try {
-                            if ((currentSession.controlSocket != null)&&(currentSession.controlSocket.isConnected())) {
-                                currentSession.controlSocket.close();
-                                serverCore.this.log.logInfo("Closing socket of thread '" + currentSession.getName() + "'");
-                            }
-                        } catch (IOException e) {}
-                    }
+                    serverCore.this.log.logInfo("Trying to shutdown session thread '" + threadList[currentThreadIdx].getName() + "' [ID=" + threadList[currentThreadIdx].getId() + "].");
+                    ((Session)threadList[currentThreadIdx]).close();
                 }                
                 
                 // we need to use a timeout here because of missing interruptable session threads ...
                 for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ )  {
-                    if (threadList[currentThreadIdx].isAlive()) threadList[currentThreadIdx].join(500);
+                    if (threadList[currentThreadIdx].isAlive()) {
+                        serverCore.this.log.logDebug("Waiting for session thread '" + threadList[currentThreadIdx].getName() + "' [ID=" + threadList[currentThreadIdx].getId() + "] to finish shutdown.");
+                        try { 
+                            threadList[currentThreadIdx].join(500); 
+                        } catch (Exception ex) {}
+                    }
                 }
             } catch (InterruptedException e) {
                 serverCore.this.log.logWarning("Interruption while trying to shutdown all remaining session threads.");  
@@ -657,6 +658,27 @@ public final class serverCore extends serverAbstractThread implements serverThre
         public void setStopped(boolean stopped) {
             this.stopped = stopped;            
         }
+        
+        public void close() {
+            if (this.isAlive()) {
+                try {
+                    // trying to close all still open httpc-Sockets first                    
+                    int closedSockets = httpc.closeOpenSockets(new Long(this.getId()));
+                    if (closedSockets > 0) {
+                        serverCore.this.log.logInfo(closedSockets + " http-client sockets of thread '" + this.getName() + "' closed.");
+                    }
+                    
+                    // waiting some time 
+                    this.join(300);
+                    
+                    // closing the socket to the client
+                    if ((this.controlSocket != null)&&(this.controlSocket.isConnected())) {
+                        this.controlSocket.close();
+                        serverCore.this.log.logInfo("Closing main socket of thread '" + this.getName() + "'");
+                    }
+                } catch (Exception e) {}
+            }            
+        }
 
         public void execute(Socket controlSocket, int socketTimeout) {
             this.execute(controlSocket, socketTimeout, null);
@@ -765,7 +787,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
                     } finally  {
                         reset();
                         
-                        if (!this.stopped && !this.isInterrupted()) {
+                        if (!this.stopped && !this.isInterrupted() && !serverCore.this.theSessionPool.isClosed) {
                             try {
                                 this.setName("Session_inPool");
                                 serverCore.this.theSessionPool.returnObject(this);
@@ -857,7 +879,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
                     this.request = new String(requestBytes);
                     //log.logDebug("* session " + handle + " received command '" + request + "'. time = " + (System.currentTimeMillis() - handle));
                     log(false, this.request);
-                    try {
+                    try {                        
                         // if we can not determine the proper command string we try to call function emptyRequest
                         // of the commandObject
                         if (this.request.trim().length() == 0) this.request = "EMPTY";

source/de/anomic/server/serverPortForwardingSch.java

@@ -140,6 +140,7 @@ public class serverPortForwardingSch implements serverPortForwarding{
                 this.log.logDebug("Deploying port forwarding session watcher thread.");
                 this.switchboard.deployThread("portForwardingWatcher", "Remote Port Forwarding Watcher", "this thread is used to detect broken connections and to re-establish it if necessary.",
                         sessionWatcher = new serverInstantThread(this, "reconnect", null), 30000,30000,30000,1000);
+                sessionWatcher.setSyncObject(new Object());
             }
             
             this.log.logInfo("Remote port forwarding connection established: " + 

source/de/anomic/yacy/yacyClient.java

@@ -110,7 +110,11 @@ public class yacyClient {
             yacyCore.seedDB.sb.remoteProxyPort,
             obj));
         } catch (Exception e) {
-            yacyCore.log.logDebug("yacyClient.publishMySeed exception:" + e.getMessage());
+            if (Thread.currentThread().isInterrupted()) {
+                yacyCore.log.logDebug("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' interrupted.");
+            } else {
+                yacyCore.log.logDebug("yacyClient.publishMySeed exception:" + e.getMessage());
+            }
             return -1;
         }
         if ((result == null) || (result.size() < 3)) {

source/de/anomic/yacy/yacyCore.java

@@ -70,6 +70,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.TimeZone;
 
+import de.anomic.http.httpc;
 import de.anomic.net.natLib;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverSemaphore;
@@ -466,27 +467,39 @@ public class yacyCore {
             peerActions.saveMySeed();
             return 0;
         } catch (InterruptedException e) {
+            try {
             log.logInfo("publish: Interruption detected while publishing my seed.");
             
+                // consuming the theads interrupted signal
+                Thread.interrupted();
+                
             // interrupt all already started publishThreads
-            log.logInfo("publish: Trying to shutdown all remaining publishing threads ...");
+                log.logInfo("publish: Signaling shutdown to all remaining publishing threads ...");
             yacyCore.publishThreadGroup.interrupt();
             
-            // waiting some time for the publishThreads to finish handshake
+                // waiting some time for the publishThreads to finish execution
+                Thread.sleep(500);
+                
             int threadCount  = yacyCore.publishThreadGroup.activeCount();    
             Thread[] threadList = new Thread[threadCount];     
             threadCount = yacyCore.publishThreadGroup.enumerate(threadList);
-            try {
+                
                 // we need to use a timeout here because of missing interruptable session threads ...
                 for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ )  {
-                    if (threadList[currentThreadIdx].isAlive()) {
+                    Thread currentThread = threadList[currentThreadIdx];
+                    Long currentThreadID = new Long(currentThread.getId());
+                    
+                    if (currentThread.isAlive()) {
+                        log.logInfo("publish: Closing socket of publishing thread '" + threadList[currentThreadIdx].getName() + "'.");
+                        httpc.closeOpenSockets(currentThreadID);
+                        
                         log.logInfo("publish: Waiting for remaining publishing thread '" + threadList[currentThreadIdx].getName() + "' to finish shutdown");
-                        threadList[currentThreadIdx].join(500);
+                        try { threadList[currentThreadIdx].join(500); }catch (Exception ex) {}
                     }
                 }
             }
             catch (InterruptedException ee) {
-                log.logWarning("Interruption while trying to shutdown all remaining publishing threads.");  
+                log.logWarning("publish: Interruption while trying to shutdown all remaining publishing threads.");  
             }
             
             return 0;

source/de/anomic/yacy/yacyPeerActions.java

@@ -145,6 +145,7 @@ public class yacyPeerActions {
         // - use the superseed to further fill up the seedDB
         int ssc = 0;
 	for (int i = 0; i < superseed.size(); i++) {
+        if (Thread.currentThread().isInterrupted()) break;
 	    seedListFileURL = (String) superseed.any();
 	    if (seedListFileURL.startsWith("http://")) {
 		// load the seed list