diff --git a/htroot/Status.java b/htroot/Status.java index 8dc3fe65f..75cda8736 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -77,7 +77,10 @@ public class Status { ((plasmaSwitchboard)env).pauseCrawling(); } else if (post.containsKey("continuecrawlqueue")) { ((plasmaSwitchboard)env).continueCrawling(); - } + } else if (post.containsKey("ResetTraffic")) { + httpdByteCountInputStream.resetCount(); + httpdByteCountOutputStream.resetCount(); + } prop.put("LOCATION",""); return prop; } @@ -239,6 +242,7 @@ public class Status { // proxy traffic prop.put("trafficIn",bytesToString(httpdByteCountInputStream.getGlobalCount())); prop.put("trafficOut",bytesToString(httpdByteCountOutputStream.getGlobalCount())); + prop.put("trafficCrawler",bytesToString(httpdByteCountInputStream.getAccountCount("CRAWLER"))); // connection information serverCore httpd = (serverCore) env.getThread("10_httpd"); diff --git a/htroot/Status_p.inc b/htroot/Status_p.inc index fc57d1bf7..f03d29e8e 100644 --- a/htroot/Status_p.inc +++ b/htroot/Status_p.inc @@ -55,8 +55,8 @@ Traffic - Out: #[trafficIn]# | In: #[trafficOut]# -   + Proxy: #[trafficOut]# | Crawler: #[trafficCrawler]# + [Reset] Connections Incoming diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index 151ea54f9..45a13b72a 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -42,6 +42,8 @@ package de.anomic.http; import java.io.File; import java.io.FileOutputStream; +import java.io.FilterInputStream; +import java.io.FilterOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -131,6 +133,9 @@ public final class httpc { // output and input streams for client control connection PushbackInputStream clientInput = null; private OutputStream clientOutput = null; + + private httpdByteCountInputStream clientInputByteCount = null; + private httpdByteCountOutputStream clientOutputByteCount = null; private boolean remoteProxyUse = false; private String savedRemoteHost = null; @@ -208,7 +213,9 @@ public final class httpc { int port, int timeout, boolean ssl, - httpRemoteProxyConfig remoteProxyConfig + httpRemoteProxyConfig remoteProxyConfig, + String incomingByteCountAccounting, + String outgoingByteCountAccounting ) throws IOException { httpc newHttpc; @@ -226,7 +233,9 @@ public final class httpc { port, timeout, ssl, - remoteProxyConfig + remoteProxyConfig, + incomingByteCountAccounting, + outgoingByteCountAccounting ); } catch (IOException e) { try{ httpc.theHttpcPool.returnObject(newHttpc); } catch (Exception e1) {} @@ -234,7 +243,27 @@ public final class httpc { } return newHttpc; } + + public static httpc getInstance( + String server, + int port, + int timeout, + boolean ssl, + httpRemoteProxyConfig remoteProxyConfig + ) throws IOException { + return getInstance(server,port,timeout,ssl,remoteProxyConfig,null,null); + } + public static httpc getInstance( + String server, + int port, + int timeout, + boolean ssl + ) throws IOException { + return getInstance(server,port,timeout,ssl,null,null); + } + + /** * This method gets a new httpc instance from the object pool and * initializes it with the given parameters. @@ -246,7 +275,14 @@ public final class httpc { * @throws IOException * @see httpc#init */ - public static httpc getInstance(String server, int port, int timeout, boolean ssl) throws IOException { + public static httpc getInstance( + String server, + int port, + int timeout, + boolean ssl, + String incomingByteCountAccounting, + String outgoingByteCountAccounting + ) throws IOException { httpc newHttpc = null; // fetching a new httpc from the object pool @@ -259,7 +295,7 @@ public final class httpc { // initialize it try { - newHttpc.init(server,port,timeout,ssl); + newHttpc.init(server,port,timeout,ssl,incomingByteCountAccounting,outgoingByteCountAccounting); } catch (IOException e) { try{ httpc.theHttpcPool.returnObject(newHttpc); } catch (Exception e1) {} throw e; @@ -398,7 +434,10 @@ public final class httpc { int port, int timeout, boolean ssl, - httpRemoteProxyConfig theRemoteProxyConfig) throws IOException { + httpRemoteProxyConfig theRemoteProxyConfig, + String incomingByteCountAccounting, + String outgoingByteCountAccounting + ) throws IOException { if (port == -1) { port = (ssl)? 443 : 80; @@ -407,7 +446,7 @@ public final class httpc { String remoteProxyHost = theRemoteProxyConfig.getProxyHost(); int remoteProxyPort = theRemoteProxyConfig.getProxyPort(); - this.init(remoteProxyHost, remoteProxyPort, timeout, ssl); + this.init(remoteProxyHost, remoteProxyPort, timeout, ssl,incomingByteCountAccounting,outgoingByteCountAccounting); this.remoteProxyUse = true; this.savedRemoteHost = server + ((port == 80) ? "" : (":" + port)); @@ -424,7 +463,14 @@ public final class httpc { * @param ssl Wether we should use SSL. * @throws IOException */ - void init(String server, int port, int timeout, boolean ssl) throws IOException { + void init( + String server, + int port, + int timeout, + boolean ssl, + String incomingByteCountAccounting, + String outgoingByteCountAccounting + ) throws IOException { this.handle = System.currentTimeMillis(); //serverLog.logDebug("HTTPC", handle + " initialized"); this.remoteProxyUse = false; @@ -466,10 +512,19 @@ public final class httpc { //socket.setSoLinger(true, timeout); this.socket.setKeepAlive(true); // + if (incomingByteCountAccounting != null) { + this.clientInputByteCount = new httpdByteCountInputStream(this.socket.getInputStream(),incomingByteCountAccounting); + } + // getting input and output streams - this.clientInput = new PushbackInputStream(this.socket.getInputStream()); + this.clientInput = new PushbackInputStream((this.clientInputByteCount!=null)? + this.clientInputByteCount: + this.socket.getInputStream()); this.clientOutput = this.socket.getOutputStream(); + + + // if we reached this point, we should have a connection } catch (UnknownHostException e) { if (this.socket != null) { @@ -479,6 +534,14 @@ public final class httpc { this.socketOwner = null; throw new IOException("unknown host: " + server); } + } + + public long getInputStreamByteCount() { + return (this.clientInputByteCount == null)?0:this.clientInputByteCount.getCount(); + } + + public long getOutputStreamByteCount() { + return (this.clientOutputByteCount == null)?0:this.clientOutputByteCount.getCount(); } /** @@ -501,6 +564,15 @@ public final class httpc { this.socket = null; this.socketOwner = null; } + + if (this.clientInputByteCount != null) { + try { this.clientInputByteCount.finish();} catch (IOException e) {} + this.clientInputByteCount = null; + } + if (this.clientOutputByteCount != null) { + try { this.clientOutputByteCount.finish();} catch (IOException e) {} + this.clientOutputByteCount = null; + } this.host = null; this.timeout = 0; diff --git a/source/de/anomic/http/httpdByteCountInputStream.java b/source/de/anomic/http/httpdByteCountInputStream.java index 93fc22efd..92c345647 100644 --- a/source/de/anomic/http/httpdByteCountInputStream.java +++ b/source/de/anomic/http/httpdByteCountInputStream.java @@ -3,21 +3,26 @@ package de.anomic.http; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; +import java.util.Iterator; public final class httpdByteCountInputStream extends FilterInputStream { private static final Object syncObject = new Object(); + private static final HashMap byteCountInfo = new HashMap(2); private static long globalByteCount = 0; private boolean finished = false; private long byteCount; + private String byteCountAccountName = null; /** * Constructor of this class * @param inputStream the {@link InputStream} to read from */ - public httpdByteCountInputStream(InputStream inputStream) { + public httpdByteCountInputStream(InputStream inputStream, String accountName) { super(inputStream); + this.byteCountAccountName = accountName; } /** @@ -25,9 +30,10 @@ public final class httpdByteCountInputStream extends FilterInputStream { * @param inputStream the {@link InputStream} to read from * @param initByteCount to initialize the bytecount with a given value */ - public httpdByteCountInputStream(InputStream inputStream, int initByteCount) { + public httpdByteCountInputStream(InputStream inputStream, int initByteCount, String accountName) { super(inputStream); this.byteCount = initByteCount; + this.byteCountAccountName = accountName; } public int read(byte[] b) throws IOException { @@ -51,19 +57,55 @@ public final class httpdByteCountInputStream extends FilterInputStream { return this.byteCount; } + public String getAccountName() { + return this.byteCountAccountName; + } + public static long getGlobalCount() { synchronized (syncObject) { return globalByteCount; } } + public static long getAccountCount(String accountName) { + synchronized (syncObject) { + if (byteCountInfo.containsKey(accountName)) { + return ((Long)byteCountInfo.get(accountName)).longValue(); + } + return 0; + } + } + + public void close() throws IOException { + super.close(); + this.finish(); + } + public void finish() throws IOException { + if (this.finished) return; + this.finished = true; synchronized (syncObject) { globalByteCount += this.byteCount; + if (this.byteCountAccountName != null) { + long lastByteCount = 0; + if (byteCountInfo.containsKey(this.byteCountAccountName)) { + lastByteCount = ((Long)byteCountInfo.get(this.byteCountAccountName)).longValue(); + } + lastByteCount += this.byteCount; + byteCountInfo.put(this.byteCountAccountName,new Long(lastByteCount)); + } + } } + public static void resetCount() { + synchronized (syncObject) { + globalByteCount = 0; + byteCountInfo.clear(); + } + } + protected void finalize() throws Throwable { if (!this.finished) finish(); diff --git a/source/de/anomic/http/httpdByteCountOutputStream.java b/source/de/anomic/http/httpdByteCountOutputStream.java index 60a129dfa..0400b6f70 100644 --- a/source/de/anomic/http/httpdByteCountOutputStream.java +++ b/source/de/anomic/http/httpdByteCountOutputStream.java @@ -62,7 +62,15 @@ public final class httpdByteCountOutputStream extends BufferedOutputStream { } } + public static void resetCount() { + synchronized (syncObject) { + globalByteCount = 0; + } + } + public void finish() throws IOException { + if (this.finished) return; + this.finished = true; synchronized (syncObject) { globalByteCount += this.byteCount; diff --git a/source/de/anomic/plasma/plasmaCrawlWorker.java b/source/de/anomic/plasma/plasmaCrawlWorker.java index 9e3d3c783..c9238fc78 100644 --- a/source/de/anomic/plasma/plasmaCrawlWorker.java +++ b/source/de/anomic/plasma/plasmaCrawlWorker.java @@ -354,8 +354,8 @@ public final class plasmaCrawlWorker extends Thread { // open the connection remote = ((theRemoteProxyConfig != null) && (theRemoteProxyConfig.useProxy())) - ? httpc.getInstance(host, port, socketTimeout, ssl, theRemoteProxyConfig) - : httpc.getInstance(host, port, socketTimeout, ssl); + ? httpc.getInstance(host, port, socketTimeout, ssl, theRemoteProxyConfig,"CRAWLER",null) + : httpc.getInstance(host, port, socketTimeout, ssl, "CRAWLER",null); // specifying if content encoding is allowed remote.setAllowContentEncoding(useContentEncodingGzip);