Monitoring and limit connection-count for Jetty

htroot/Connections_p.html

@@ -11,26 +11,22 @@
     <script type="text/javascript" src="js/sorttable.js"></script>
     <h2>Server Connection Tracking</h2>
     <h3>Incoming Connections</h3>
-    <p>Showing #[numActiveRunning]# active, #[numActivePending]# pending connections from a max. of #[numMax]# allowed incoming connections.</p>
+    <p>Showing #[numActiveRunning]# active connections from a max. of #[numMax]# allowed incoming connections.</p>
     <table class="sortable" border="0">
       <tr class="TableHeader" valign="bottom">
         <td>Protocol</td>
         <td>Duration</td>
         <td>Source IP[:Port]</td>
-        <td>Dest. IP[:Port]</td>
         <td>Command</td>
-        <td>Used</td>
+        <td>ID</td>
-        <td>Close</td>
       </tr>
       #{list}#
       <tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#" title="#[sessionName]#">
         <td>#[proto]#</td>
-        <td>#(ms)##[duration]#::#[duration]# ms#(/ms)#</td>
+        <td>#[duration]#</td>
         <td>#[source]#</td>
-        <td>#[dest]#</td>
+        <td>#[command]#</td>
-        <td>#(running)#Waiting for new request nr. # #[reqNr]#::#[command]##(/running)#</td>
+        <td>#[id]#</td>
-        <td>#[used]#</td>
-        <td><a href="Connections_p.html?closeServerSession=#[serverSessionID]#">[Close]</a></td>
       </tr>
       #{/list}#
     </table>

htroot/Connections_p.java

@@ -30,52 +30,55 @@
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Set;
 
 import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.http.YaCyHttpServer;
-import net.yacy.search.Switchboard;
 import net.yacy.server.serverObjects;
 import net.yacy.server.serverSwitch;
 
 public final class Connections_p {
 
-    public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) {
+    public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
         // return variable that accumulates replacements
-        final Switchboard sb = (Switchboard) env;
         final serverObjects prop = new serverObjects();
 
         // server sessions
-        // get the serverCore thread
+        List<ConnectionInfo> allConnectionsSorted = new LinkedList<ConnectionInfo>(ConnectionInfo.getServerConnections());
-        final YaCyHttpServer httpd = sb.getHttpServer();
+        Collections.sort(allConnectionsSorted);
-
+        Collections.reverse(allConnectionsSorted); // toggle ascending/descending
-        // waiting for all threads to finish
-        int idx = 0, numActivePending = 0;
-        prop.put("list", idx);
 
-        prop.putNum("numMax", httpd.getMaxSessionCount());
+        int c = 0;
-        prop.putNum("numActiveRunning", httpd.getJobCount());
+        synchronized (allConnectionsSorted) {
-        prop.putNum("numActivePending", numActivePending);
+	        for (final ConnectionInfo conInfo: allConnectionsSorted) {
+	            prop.put("list_" + c + "_proto", conInfo.getProtocol());
+	            prop.putNum("list_" + c + "_duration", conInfo.getLifetime());
+	            prop.put("list_" + c + "_source", conInfo.getTargetHost());
+	            prop.putHTML("list_" + c + "_command", conInfo.getCommand());
+	            prop.put("list_" + c + "_id", conInfo.getID());
+	            c++;
+	        }
+        }
+        prop.put("list", c);
+        prop.putNum("numMax", ConnectionInfo.getServerMaxcount());
+        prop.putNum("numActiveRunning", c);
 
         // client sessions
-        final Set<ConnectionInfo> allConnections = ConnectionInfo.getAllConnections();
         // sorting: sort by initTime, decending
-        List<ConnectionInfo> allConnectionsSorted = new LinkedList<ConnectionInfo>(allConnections);
+        allConnectionsSorted = new LinkedList<ConnectionInfo>(ConnectionInfo.getAllConnections());
         Collections.sort(allConnectionsSorted);
         Collections.reverse(allConnectionsSorted); // toggle ascending/descending
 
-        int c = 0;
+        c = 0;
         synchronized (allConnectionsSorted) {
-        for (final ConnectionInfo conInfo: allConnectionsSorted) {
+	        for (final ConnectionInfo conInfo: allConnectionsSorted) {
-            prop.put("clientList_" + c + "_clientProtocol", conInfo.getProtocol());
+	            prop.put("clientList_" + c + "_clientProtocol", conInfo.getProtocol());
-            prop.putNum("clientList_" + c + "_clientLifetime", conInfo.getLifetime());
+	            prop.putNum("clientList_" + c + "_clientLifetime", conInfo.getLifetime());
-            prop.putNum("clientList_" + c + "_clientUpbytes", conInfo.getUpbytes());
+	            prop.putNum("clientList_" + c + "_clientUpbytes", conInfo.getUpbytes());
-            prop.put("clientList_" + c + "_clientTargetHost", conInfo.getTargetHost());
+	            prop.put("clientList_" + c + "_clientTargetHost", conInfo.getTargetHost());
-            prop.putHTML("clientList_" + c + "_clientCommand", conInfo.getCommand());
+	            prop.putHTML("clientList_" + c + "_clientCommand", conInfo.getCommand());
-            prop.put("clientList_" + c + "_clientID", conInfo.getID());
+	            prop.put("clientList_" + c + "_clientID", conInfo.getID());
-            c++;
+	            c++;
-        }
+	        }
         }
         prop.put("clientList", c);
         prop.put("clientActive", ConnectionInfo.getCount());

htroot/PerformanceQueues_p.java

@@ -28,9 +28,9 @@ import java.io.File;
 import java.util.Iterator;
 import java.util.Map;
 
+import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.cora.protocol.HeaderFramework;
 import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.http.YaCyHttpServer;
 import net.yacy.kelondro.data.word.WordReference;
 import net.yacy.kelondro.rwi.IndexCell;
 import net.yacy.kelondro.util.FileUtils;
@@ -241,14 +241,13 @@ public class PerformanceQueues_p {
             /*
              * configuring the http pool
              */
-            final YaCyHttpServer httpd = sb.getHttpServer();
             try {
                 maxBusy = post.getInt("httpd Session Pool_maxActive", 8);
             } catch (final NumberFormatException e) {
                 maxBusy = 8;
             }
 
-            (httpd).setMaxSessionCount(maxBusy);
+            ConnectionInfo.setServerMaxcount(maxBusy);
 
             // storing the new values into configfile
             sb.setConfig("httpdMaxBusySessions",maxBusy);
@@ -281,10 +280,9 @@ public class PerformanceQueues_p {
         prop.put("pool_0_maxActive", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 0));
         prop.put("pool_0_numActive", sb.crawlQueues.activeWorkerEntries().size());
 
-        final YaCyHttpServer httpd = sb.getHttpServer(); 
         prop.put("pool_1_name", "httpd Session Pool");
-        prop.put("pool_1_maxActive", httpd.getMaxSessionCount());
+        prop.put("pool_1_maxActive", ConnectionInfo.getServerMaxcount());
-        prop.put("pool_1_numActive", httpd.getJobCount());
+        prop.put("pool_1_numActive", ConnectionInfo.getServerCount());
 
         prop.put("pool", "2");
 

htroot/Status.java

@@ -29,10 +29,11 @@
 
 import java.net.InetAddress;
 import java.util.Date;
+
+import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.cora.protocol.Domains;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.cora.util.Memory;
-import net.yacy.http.YaCyHttpServer;
 import net.yacy.kelondro.io.ByteCount;
 import net.yacy.kelondro.util.Formatter;
 import net.yacy.kelondro.util.MemoryControl;
@@ -327,10 +328,8 @@ public class Status
         prop.put("trafficCrawler", Formatter.bytesToString(ByteCount.getAccountCount(ByteCount.CRAWLER)));
 
         // connection information
-        final YaCyHttpServer httpd =  sb.getHttpServer();
+        prop.putNum("connectionsActive", ConnectionInfo.getServerCount());
-
+        prop.putNum("connectionsMax", ConnectionInfo.getServerMaxcount());
-        prop.putNum("connectionsActive", httpd.getJobCount());
-        prop.putNum("connectionsMax", httpd.getMaxSessionCount());
 
         // Queue information
         final int loaderJobCount = sb.crawlQueues.activeWorkerEntries().size();

source/net/yacy/cora/protocol/ConnectionInfo.java

@@ -43,10 +43,13 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
      */
     private final static Set<ConnectionInfo> allConnections = Collections
             .synchronizedSet(new HashSet<ConnectionInfo>());
+    private final static Set<ConnectionInfo> serverConnections = Collections
+            .synchronizedSet(new HashSet<ConnectionInfo>());
     // this is only for statistics, so it can be bigger to see lost connectionInfos
     private final static int staleAfterMillis = 30 * 60000; // 30 minutes
     
     private static int maxcount = 20;
+    private static int serverMaxCount = 20;
 
     private final String protocol;
     private final String targetHost;
@@ -126,6 +129,17 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     public static Set<ConnectionInfo> getAllConnections() {
         return allConnections;
     }
+
+    /**
+     * gets a {@link Set} of all collected server ConnectionInfos
+     * 
+     * Important: iterations must be synchronized!
+     * 
+     * @return the allConnections
+     */
+    public static Set<ConnectionInfo> getServerConnections() {
+        return serverConnections;
+    }
     
     /**
      * gets the number of active client connections
@@ -136,6 +150,15 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     	return getAllConnections().size();
     }
     
+    /**
+     * gets the number of active server connections
+     * 
+     * @return count of active connections
+     */
+    public static int getServerCount() {
+    	return getServerConnections().size();
+    }
+    
     /**
      * gets the usage of the Client connection manager by active connections
      * 
@@ -145,6 +168,13 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     	return getCount() * 100 / getMaxcount();
     }
     
+    /**
+     * @return wether the server max connection-count is reached
+     */
+    public static boolean isServerCountReached() {
+    	return getServerCount() >= getServerMaxcount();
+    }
+    
     /**
      * @return how many bytes queued up
      */
@@ -179,6 +209,26 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     public static void setMaxcount(final int max) {
     	if (max > 0) maxcount = max;
     }
+    
+    /**
+     * gets the max connection count of the Server connection manager
+     * 
+     * @return max connections
+     */
+    public static int getServerMaxcount() {
+    	return serverMaxCount;
+    }
+    
+    /**
+     * gets the max connection count of the Sever connection manager
+     * to be used in statistics
+     * 
+     * @param max connections
+     * @TODO Is it correct to only set if max > 0? What if maxcount is > 0 and max = 0 ?
+     */
+    public static void setServerMaxcount(final int max) {
+    	if (max > 0) serverMaxCount = max;
+    }
 
     /**
      * add a connection to the list of all current connections
@@ -189,6 +239,15 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     	getAllConnections().add(conInfo);
     }
 
+    /**
+     * add a Server connection to the list of all current connections
+     * 
+     * @param conInfo
+     */
+    public static void addServerConnection(final ConnectionInfo conInfo) {
+    	getServerConnections().add(conInfo);
+    }
+
     /**
      * remove a connection from the list of all current connections
      * 
@@ -198,6 +257,15 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     	getAllConnections().remove(conInfo);
     }
 
+    /**
+     * remove a Server connection from the list of all current connections
+     * 
+     * @param conInfo
+     */
+    public static void removeServerConnection(final ConnectionInfo conInfo) {
+    	getServerConnections().remove(conInfo);
+    }
+
     /**
      * connections with same id {@link equals()} another
      * 
@@ -206,13 +274,26 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     public static void removeConnection(final int id) {
         removeConnection(new ConnectionInfo(null, null, null, id, 0, 0));
     }
+
+    /**
+     * connections with same id {@link equals()} another
+     * 
+     * @param id
+     */
+    public static void removeServerConnection(final int id) {
+        removeServerConnection(new ConnectionInfo(null, null, null, id, 0, 0));
+    }
     
     /**
      * removes stale connections
      */
     public static void cleanUp() {
-        Iterator<ConnectionInfo> iter = getAllConnections().iterator();
+    	cleanup(getAllConnections().iterator());
-        synchronized (iter) { 
+    	cleanup(getServerConnections().iterator());
+    }
+    
+    private static void cleanup(final Iterator<ConnectionInfo> iter) {
+    	synchronized (iter) { 
             while (iter.hasNext()) {
                 ConnectionInfo con = iter.next();
                 if(con.getLifetime() > staleAfterMillis) {

source/net/yacy/http/Jetty9HttpServerImpl.java

@@ -167,7 +167,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         // define list of YaCy specific general handlers
         HandlerList handlers = new HandlerList();
         handlers.setHandlers(new Handler[] 
-           {domainHandler, new ProxyCacheHandler(), new ProxyHandler()}); 
+           {new MonitorHandler(), domainHandler, new ProxyCacheHandler(), new ProxyHandler()}); 
 
         // context handler for dispatcher and security (hint: dispatcher requires a context)
         ContextHandler context = new ContextHandler();
@@ -241,11 +241,6 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         server.join();
     }
 
-    @Override
-    public void setMaxSessionCount(int maxBusy) {
-        // TODO:
-    }
-
     /**
      * @return true if ssl/https connector is available
      */
@@ -407,16 +402,6 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
 
     }
 
-    @Override
-    public int getMaxSessionCount() {
-        return server.getThreadPool().getThreads();
-    }
-
-    @Override
-    public int getJobCount() {
-        return getMaxSessionCount() - server.getThreadPool().getIdleThreads(); // TODO:
-    }
-
     @Override
     public String getVersion() {
         return "Jetty " + Server.getVersion();

source/net/yacy/http/MonitorHandler.java

@@ -0,0 +1,80 @@
+/**
+ *  MonitorHandler
+ *  Copyright 2014 by Sebastian Gaebel
+ *  First released 15.05.2014 at http://yacy.net
+ *
+ *  $LastChangedDate$
+ *  $LastChangedRevision$
+ *  $LastChangedBy$
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this program in the file lgpl21.txt
+ *  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.http;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import net.yacy.cora.protocol.ConnectionInfo;
+import net.yacy.cora.protocol.Domains;
+
+import org.eclipse.jetty.io.Connection;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+
+public class MonitorHandler extends AbstractHandler {
+	
+	private final Connection.Listener remover = new Connection.Listener() {
+
+		@Override
+		public void onClosed(Connection c) {
+			ConnectionInfo.removeServerConnection(c.hashCode());
+		}
+
+		@Override
+		public void onOpened(Connection c) {
+		}
+	};
+
+	@Override
+	public void handle(String target, Request baseRequest, HttpServletRequest request,
+            HttpServletResponse response) throws IOException, ServletException {
+		
+		final Connection connection = baseRequest.getHttpChannel().getEndPoint().getConnection();
+		final ConnectionInfo info = new ConnectionInfo(
+				baseRequest.getProtocol(),
+				baseRequest.getRemoteAddr() + ":" + baseRequest.getRemotePort(),
+				baseRequest.getMethod() + " " + baseRequest.getUri().getPathAndParam(),
+				connection.hashCode(),
+				baseRequest.getTimeStamp(),
+				-1);
+		
+		if (ConnectionInfo.getServerConnections().contains(info)) {
+			ConnectionInfo.removeServerConnection(info);
+		} else {
+			connection.addListener(remover);
+		}
+		ConnectionInfo.addServerConnection(info);
+		
+		if (ConnectionInfo.isServerCountReached()) {
+			if (Domains.isLocal(baseRequest.getRemoteAddr(), baseRequest.getRemoteInetSocketAddress().getAddress())) return;
+			response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE,"max. server connections reached (see System Administration -> Performance Settings of Busy Queues -> Thread Pool Settings).");
+            baseRequest.setHandled(true);
+		}
+	}
+}

source/net/yacy/http/YaCyHttpServer.java

@@ -19,10 +19,7 @@ public interface YaCyHttpServer {
     
     abstract void startupServer() throws Exception;
     abstract void stop() throws Exception;
-    abstract void setMaxSessionCount(int cnt);
     abstract InetSocketAddress generateSocketAddress(String port) throws SocketException;
-    abstract int getMaxSessionCount();
-    abstract int getJobCount();
     abstract int getSslPort();
     abstract boolean withSSL();
     abstract void reconnect(int milsec);