diff --git a/htroot/Connections_p.html b/htroot/Connections_p.html index 6fa852901..e359ff28f 100644 --- a/htroot/Connections_p.html +++ b/htroot/Connections_p.html @@ -7,7 +7,8 @@ #%env/templates/header.template%# - #%env/templates/submenuAccessTracker.template%# + #%env/templates/submenuAccessTracker.template%# +

Server Connection Tracking

Incoming Connections

Showing #[numActiveRunning]# active, #[numActivePending]# pending connections from a max. of #[numMax]# allowed incoming connections.

@@ -33,8 +34,28 @@ #{/list}# - + +

Outgoing Connections

+

Showing #[clientActive]# active outgoing connections:

+ + + + + + + + + #{clientList}# + + + + + + + + #{/clientList}# +
ProtocolDurationDest. IP[:Port]CommandID
#[clientProtocol]##[clientLifetime]##[clientTargetHost]##[clientCommand]##[clientID]#
+ #%env/templates/footer.template%# - \ No newline at end of file diff --git a/htroot/Connections_p.java b/htroot/Connections_p.java index 60c636fc2..f3ff7730b 100644 --- a/htroot/Connections_p.java +++ b/htroot/Connections_p.java @@ -49,11 +49,13 @@ import java.io.UnsupportedEncodingException; import java.net.InetAddress; import java.net.URLEncoder; +import java.util.Iterator; import java.util.Properties; import org.apache.commons.pool.impl.GenericObjectPool; import de.anomic.http.httpHeader; +import de.anomic.http.httpc; import de.anomic.http.httpd; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; @@ -238,6 +240,22 @@ public final class Connections_p { prop.put("numActiveRunning",Integer.toString(numActiveRunning)); prop.put("numActivePending",Integer.toString(numActivePending)); + + // client sessions + Iterator i = httpc.activeConnections.iterator(); + int c = 0; + while (i.hasNext()) { + httpc clientConnection = (httpc) i.next(); + prop.put("clientList_" + c + "_clientProtocol", (clientConnection.ssl) ? "HTTPS" : "HTTP"); + prop.put("clientList_" + c + "_clientLifetime", System.currentTimeMillis() - clientConnection.initTime); + prop.put("clientList_" + c + "_clientTargetHost", clientConnection.adressed_host + ":" + clientConnection.adressed_port); + prop.put("clientList_" + c + "_clientCommand", clientConnection.command); + prop.put("clientList_" + c + "_clientID", clientConnection.hashCode()); + c++; + } + prop.put("clientList", c); + prop.put("clientActive", c); + // return rewrite values for templates return prop; } diff --git a/htroot/CrawlURLFetch_p.java b/htroot/CrawlURLFetch_p.java index 92ff415d5..f16e714ed 100644 --- a/htroot/CrawlURLFetch_p.java +++ b/htroot/CrawlURLFetch_p.java @@ -531,6 +531,7 @@ public class CrawlURLFetch_p { if (encoding == null) encoding = "US-ASCII"; r = parseText(new String(sbb.getBytes(), encoding)); } + con.close(); } catch (IOException e) { } return r; } diff --git a/source/de/anomic/data/SitemapParser.java b/source/de/anomic/data/SitemapParser.java index 3bec22e54..d42e77e3e 100644 --- a/source/de/anomic/data/SitemapParser.java +++ b/source/de/anomic/data/SitemapParser.java @@ -209,6 +209,7 @@ public class SitemapParser extends DefaultHandler { + "\n\tLength: " + this.contentLength); SAXParser saxParser = SAXParserFactory.newInstance().newSAXParser(); saxParser.parse(this.counterStream, this); + remote.close(); } catch (Exception e) { this.logger.logWarning("Unable to parse sitemap file " + this.siteMapURL,e); } diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java index f731b4243..3dc260ff0 100644 --- a/source/de/anomic/data/robotsParser.java +++ b/source/de/anomic/data/robotsParser.java @@ -413,6 +413,7 @@ public final class robotsParser{ if (!res.responseHeader.mime().startsWith("text/plain")) { robotsTxt = null; serverLog.logFinest("ROBOTS","Robots.txt from URL '" + robotsURL + "' has wrong mimetype '" + res.responseHeader.mime() + "'."); + con.close(); } else { // getting some metadata @@ -435,10 +436,12 @@ public final class robotsParser{ serverLog.logFinest("ROBOTS","Robots.txt successfully loaded from URL '" + robotsURL + "' in " + (downloadEnd-downloadStart) + " ms."); } } else if (res.status.startsWith("304")) { + con.close(); return null; } else if (res.status.startsWith("3")) { // getting redirection URL String redirectionUrlString = (String) res.responseHeader.get(httpHeader.LOCATION); + con.close(); if (redirectionUrlString==null) { serverLog.logFinest("ROBOTS","robots.txt could not be downloaded from URL '" + robotsURL + "' because of missing redirecton header. [" + res.status + "]."); robotsTxt = null; @@ -455,10 +458,12 @@ public final class robotsParser{ return downloadRobotsTxt(redirectionUrl,redirectionCount,entry); } else if (res.status.startsWith("401") || res.status.startsWith("403")) { + con.close(); accessCompletelyRestricted = true; serverLog.logFinest("ROBOTS","Access to Robots.txt not allowed on URL '" + robotsURL + "'."); } else { serverLog.logFinest("ROBOTS","robots.txt could not be downloaded from URL '" + robotsURL + "'. [" + res.status + "]."); + con.close(); robotsTxt = null; } } catch (Exception e) { diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index 55df8c855..b146a5ee1 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -60,6 +60,7 @@ import java.util.Date; import java.util.Enumeration; import java.util.GregorianCalendar; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.Locale; import java.util.Map; @@ -108,6 +109,8 @@ public final class httpc { private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US); private static final HashMap reverseMappingCache = new HashMap(); private static final HashMap openSocketLookupTable = new HashMap(); + public static final HashSet activeConnections = new HashSet(); // all connections are stored here and deleted when they are finished + public static int objCounter = 0; // will be increased with each object and is use to return a hash code // defined during set-up of switchboard public static boolean yacyDebugMode = false; @@ -177,8 +180,8 @@ public final class httpc { // class variables private Socket socket = null; // client socket for commands private Thread socketOwner = null; - private String adressed_host = null; - private int adressed_port = 80; + public String adressed_host = null; + public int adressed_port = 80; private String target_virtual_host = null; // output and input streams for client control connection @@ -194,6 +197,76 @@ public final class httpc { private String requestPath = null; private boolean allowContentEncoding = true; + public boolean ssl; + public long initTime; + public String command; + + private int hashIndex; + + + /** + * Initialize the httpc-instance with the given data. + * + * @param remoteProxyHost + * @param remoteProxyPort + * @throws IOException + */ + public httpc( + String server, + String vhost, + int port, + int timeout, + boolean ssl, + httpRemoteProxyConfig theRemoteProxyConfig, + String incomingByteCountAccounting, + String outgoingByteCountAccounting + ) throws IOException { + + this.hashIndex = objCounter; + objCounter++; + synchronized (activeConnections) {activeConnections.add(this);} + //System.out.println("*** DEBUG init httpc: " + activeConnections.size() + " connections online"); + + this.ssl = ssl; + this.initTime = System.currentTimeMillis(); + this.command = null; + + if ((theRemoteProxyConfig == null) || + (!theRemoteProxyConfig.useProxy())) { + initN( + server, + vhost, + port, + timeout, + ssl, + incomingByteCountAccounting, + outgoingByteCountAccounting + ); + return; + } + + if (port == -1) { + port = (ssl)? 443 : 80; + } + + String remoteProxyHost = theRemoteProxyConfig.getProxyHost(); + int remoteProxyPort = theRemoteProxyConfig.getProxyPort(); + + this.initN( + remoteProxyHost, + vhost, + remoteProxyPort, + timeout, + ssl, + incomingByteCountAccounting, + outgoingByteCountAccounting); + + this.remoteProxyUse = true; + this.adressed_host = server; + this.adressed_port = port; + this.target_virtual_host = vhost; + this.remoteProxyConfig = theRemoteProxyConfig; + } /** * Convert the status of this class into an String object to output it. @@ -252,62 +325,12 @@ public final class httpc { public static Date nowDate() { return new GregorianCalendar(GMTTimeZone).getTime(); } - - /** - * Initialize the httpc-instance with the given data. - * - * @param remoteProxyHost - * @param remoteProxyPort - * @throws IOException - */ - public httpc( - String server, - String vhost, - int port, - int timeout, - boolean ssl, - httpRemoteProxyConfig theRemoteProxyConfig, - String incomingByteCountAccounting, - String outgoingByteCountAccounting - ) throws IOException { - - if ((theRemoteProxyConfig == null) || - (!theRemoteProxyConfig.useProxy())) { - initN( - server, - vhost, - port, - timeout, - ssl, - incomingByteCountAccounting, - outgoingByteCountAccounting - ); - return; - } - - if (port == -1) { - port = (ssl)? 443 : 80; - } - - String remoteProxyHost = theRemoteProxyConfig.getProxyHost(); - int remoteProxyPort = theRemoteProxyConfig.getProxyPort(); - - this.initN( - remoteProxyHost, - vhost, - remoteProxyPort, - timeout, - ssl, - incomingByteCountAccounting, - outgoingByteCountAccounting); - - this.remoteProxyUse = true; - this.adressed_host = server; - this.adressed_port = port; - this.target_virtual_host = vhost; - this.remoteProxyConfig = theRemoteProxyConfig; + + public int hashCode() { + // return a hash code so it is possible to store objects of httpc objects in a HashSet + return this.hashIndex; } - + /** * Initialize the https-instance with the given data. Opens the sockets to * the remote server and creats input and output streams. @@ -404,6 +427,9 @@ public final class httpc { } public void close() { + synchronized (activeConnections) {activeConnections.remove(this);} + //System.out.println("*** DEBUG close httpc: " + activeConnections.size() + " connections online"); + if (this.clientInput != null) { try {this.clientInput.close();} catch (Exception e) {} this.clientInput = null; @@ -569,7 +595,8 @@ public final class httpc { */ public response GET(String path, httpHeader requestHeader) throws IOException { //serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle)); - try { + this.command = "GET " + path; + try { boolean zipped = (!this.allowContentEncoding) ? false : httpd.shallTransportZipped(path); send(httpHeader.METHOD_GET, path, requestHeader, zipped); response r = new response(zipped); @@ -592,6 +619,7 @@ public final class httpc { * @throws IOException */ public response HEAD(String path, httpHeader requestHeader) throws IOException { + this.command = "HEAD " + path; try { send(httpHeader.METHOD_HEAD, path, requestHeader, false); return new response(false); @@ -612,6 +640,7 @@ public final class httpc { * @throws IOException */ public response POST(String path, httpHeader requestHeader, InputStream ins) throws IOException { + this.command = "POST " + path; try { send(httpHeader.METHOD_POST, path, requestHeader, false); // if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader @@ -653,6 +682,7 @@ public final class httpc { */ public response CONNECT(String remotehost, int remoteport, httpHeader requestHeader) throws IOException { + this.command = "CONNECT " + remotehost + ":" + remoteport; try { send(httpHeader.METHOD_CONNECT, remotehost + ":" + remoteport, requestHeader, false); return new response(false); @@ -786,8 +816,7 @@ public final class httpc { requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password)); } - httpc con = null; - con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null); + httpc con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null); httpc.response res = con.GET(path, requestHeader); if (res.status.startsWith("2")) { @@ -795,10 +824,12 @@ public final class httpc { // stream to byte[] serverByteBuffer sbb = new serverByteBuffer(); res.writeContent(sbb, null); + con.close(); return sbb.getBytes(); } else { // stream to file and return null res.writeContent(null, download); + con.close(); return null; } } @@ -847,11 +878,16 @@ public final class httpc { httpc.response res = con.POST(path, requestHeader, props, files); //System.out.println("response=" + res.toString()); - if (!(res.status.startsWith("2"))) return res.status.getBytes(); + if (!(res.status.startsWith("2"))) { + byte[] status = res.status.getBytes(); + con.close(); + return status; + } // read connection body and return body serverByteBuffer sbb = new serverByteBuffer(); res.writeContent(sbb, null); + con.close(); return sbb.getBytes(); } @@ -993,15 +1029,11 @@ public final class httpc { String realhost = url.getHost(); // start connection - httpc con = null; - con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null); + httpc con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null); httpc.response res = con.HEAD(path, requestHeader); - if (res.status.startsWith("2")) { - // success - return res.responseHeader; - } - // fail - return res.responseHeader; + httpHeader h = res.responseHeader; + con.close(); + return h; } public static byte[] wput( diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 50711a27b..1c9b00f56 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -708,6 +708,8 @@ public final class httpdProxyHandler { conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); } + remote.close(); + if (gzippedOut != null) { gzippedOut.finish(); } @@ -954,11 +956,11 @@ public final class httpdProxyHandler { // sending the server respond back to the client httpd.sendRespondHeader(conProp,respond,httpVer,res.statusCode,res.statusText,res.responseHeader); + respond.flush(); + remote.close(); } catch (Exception e) { handleProxyException(e,remote,conProp,respond,url); } - - respond.flush(); } public static void doPost(Properties conProp, httpHeader requestHeader, OutputStream respond, PushbackInputStream body) throws IOException { @@ -1144,12 +1146,14 @@ public final class httpdProxyHandler { // replace connection details host = switchboard.remoteProxyConfig.getProxyHost(); port = switchboard.remoteProxyConfig.getProxyPort(); + remoteProxy.close(); // go on (see below) } else { // pass error response back to client httpd.sendRespondHeader(conProp,clientOut,httpVersion,response.statusCode,response.statusText,response.responseHeader); //respondHeader(clientOut, response.status, response.responseHeader); forceConnectionClose(conProp); + remoteProxy.close(); return; } } catch (Exception e) { diff --git a/source/de/anomic/plasma/crawler/http/CrawlWorker.java b/source/de/anomic/plasma/crawler/http/CrawlWorker.java index 3fb4a9b0d..831835323 100644 --- a/source/de/anomic/plasma/crawler/http/CrawlWorker.java +++ b/source/de/anomic/plasma/crawler/http/CrawlWorker.java @@ -271,10 +271,12 @@ public final class CrawlWorker extends AbstractCrawlWorker { // we write the new cache entry to file system directly byte[] cacheArray = null; cacheArray = res.writeContent(fos,this.keepInMemory); + remote.close(); htCache.setCacheArray(cacheArray); plasmaHTCache.writeFileAnnouncement(cacheFile); } finally { if (fos!=null)try{fos.close();}catch(Exception e){/* ignore this */} + remote.close(); } // enQueue new entry with response header diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 7620cc745..1c00e7194 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -409,6 +409,7 @@ public final class plasmaCrawlLURL { remove(urlHash); log.logInfo("UrlDB-Entry with urlHash '" + urlHash + "' removed\n\tURL: " + oldUrlStr + "\n\tConnection Status: " + res.status); } + theHttpc.close(); } } catch (Exception e) { remove(urlHash); diff --git a/source/de/anomic/plasma/plasmaParser.java b/source/de/anomic/plasma/plasmaParser.java index 502cbe879..54fd261b8 100644 --- a/source/de/anomic/plasma/plasmaParser.java +++ b/source/de/anomic/plasma/plasmaParser.java @@ -932,6 +932,7 @@ public final class plasmaParser { contentMimeType = res.responseHeader.mime(); charSet = res.responseHeader.getCharacterEncoding(); contentLength = res.responseHeader.contentLength(); + remote.close(); } if ((args.length >= 4)&&(args[2].equalsIgnoreCase("-m"))) { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index bfd7a8f88..54de56870 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -791,9 +791,9 @@ public final class yacySeedDB { throw new IOException("Server returned status: " + res.status); } - // read byte array byte[] content = serverFileUtils.read(res.getContentInputStream()); + remote.close(); // uncompress it if it is gzipped content = serverFileUtils.uncompressGZipArray(content); diff --git a/source/yacy.java b/source/yacy.java index 67dcb4e5c..1ae807c44 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -521,6 +521,7 @@ public final class yacy { con.close(); } else { serverLog.logSevere("REMOTE-SHUTDOWN", "error response from YACY socket: " + res.status); + con.close(); System.exit(-1); } } catch (IOException e) {