*) Adding Traffic Statistic for Crawler

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@972 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Status.java

@@ -77,7 +77,10 @@ public class Status {
                 ((plasmaSwitchboard)env).pauseCrawling();
             } else if (post.containsKey("continuecrawlqueue")) {
                 ((plasmaSwitchboard)env).continueCrawling();
-            }   
+            } else if (post.containsKey("ResetTraffic")) {
+                httpdByteCountInputStream.resetCount();
+                httpdByteCountOutputStream.resetCount();               
+            }
             prop.put("LOCATION","");
             return prop;
         }
@@ -239,6 +242,7 @@ public class Status {
         // proxy traffic
         prop.put("trafficIn",bytesToString(httpdByteCountInputStream.getGlobalCount()));
         prop.put("trafficOut",bytesToString(httpdByteCountOutputStream.getGlobalCount()));
+        prop.put("trafficCrawler",bytesToString(httpdByteCountInputStream.getAccountCount("CRAWLER")));
 
         // connection information
         serverCore httpd = (serverCore) env.getThread("10_httpd");

htroot/Status_p.inc

@@ -55,8 +55,8 @@
 </tr>
 <tr class="TableCellDark">
 	<td>Traffic</td>
-	<td>Out: #[trafficIn]# | In: #[trafficOut]#</td>
-	<td>&nbsp;</td>
+	<td>Proxy: #[trafficOut]# | Crawler: #[trafficCrawler]#</td>
+	<td>[<a href="Status.html?ResetTraffic=">Reset</a>]</td>
 </tr>
 <tr class="TableCellLight">
 	<td>Connections Incoming</td>

source/de/anomic/http/httpc.java

@@ -42,6 +42,8 @@ package de.anomic.http;
 
 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.FilterInputStream;
+import java.io.FilterOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -131,6 +133,9 @@ public final class httpc {
     // output and input streams for client control connection
     PushbackInputStream clientInput = null;
     private OutputStream clientOutput = null;
+    
+    private httpdByteCountInputStream clientInputByteCount = null;
+    private httpdByteCountOutputStream clientOutputByteCount = null;
 
     private boolean remoteProxyUse = false;
     private String  savedRemoteHost = null;
@@ -208,7 +213,9 @@ public final class httpc {
             int port,
             int timeout,
             boolean ssl,
-            httpRemoteProxyConfig remoteProxyConfig
+            httpRemoteProxyConfig remoteProxyConfig,
+            String incomingByteCountAccounting,
+            String outgoingByteCountAccounting
             ) throws IOException {
 
         httpc newHttpc;
@@ -226,7 +233,9 @@ public final class httpc {
                     port,
                     timeout,
                     ssl,
-                    remoteProxyConfig
+                    remoteProxyConfig,
+                    incomingByteCountAccounting,
+                    outgoingByteCountAccounting
             );
         } catch (IOException e) {
             try{ httpc.theHttpcPool.returnObject(newHttpc); } catch (Exception e1) {}
@@ -234,7 +243,27 @@ public final class httpc {
         }
         return newHttpc;
     }
+    
+    public static httpc getInstance(
+            String server,
+            int port,
+            int timeout,
+            boolean ssl,
+            httpRemoteProxyConfig remoteProxyConfig
+            ) throws IOException {
+        return getInstance(server,port,timeout,ssl,remoteProxyConfig,null,null);
+    }
 
+    public static httpc getInstance(
+            String server, 
+            int port, 
+            int timeout, 
+            boolean ssl
+    ) throws IOException {
+        return getInstance(server,port,timeout,ssl,null,null);
+    }
+
+    
     /**
     * This method gets a new httpc instance from the object pool and
     * initializes it with the given parameters.
@@ -246,7 +275,14 @@ public final class httpc {
     * @throws IOException
     * @see httpc#init
     */
-    public static httpc getInstance(String server, int port, int timeout, boolean ssl) throws IOException {
+    public static httpc getInstance(
+            String server, 
+            int port, 
+            int timeout, 
+            boolean ssl,
+            String incomingByteCountAccounting,
+            String outgoingByteCountAccounting
+    ) throws IOException {
 
         httpc newHttpc = null;
         // fetching a new httpc from the object pool
@@ -259,7 +295,7 @@ public final class httpc {
 
         // initialize it
         try {
-            newHttpc.init(server,port,timeout,ssl);
+            newHttpc.init(server,port,timeout,ssl,incomingByteCountAccounting,outgoingByteCountAccounting);
         } catch (IOException e) {
             try{ httpc.theHttpcPool.returnObject(newHttpc); } catch (Exception e1) {}
             throw e;
@@ -398,7 +434,10 @@ public final class httpc {
             int port, 
             int timeout, 
             boolean ssl,
-            httpRemoteProxyConfig theRemoteProxyConfig) throws IOException {
+            httpRemoteProxyConfig theRemoteProxyConfig,
+            String incomingByteCountAccounting,
+            String outgoingByteCountAccounting            
+    ) throws IOException {
         
         if (port == -1) {
             port = (ssl)? 443 : 80;
@@ -407,7 +446,7 @@ public final class httpc {
         String remoteProxyHost = theRemoteProxyConfig.getProxyHost();
         int    remoteProxyPort = theRemoteProxyConfig.getProxyPort();
         
-        this.init(remoteProxyHost, remoteProxyPort, timeout, ssl);
+        this.init(remoteProxyHost, remoteProxyPort, timeout, ssl,incomingByteCountAccounting,outgoingByteCountAccounting);
         
         this.remoteProxyUse = true;
         this.savedRemoteHost = server + ((port == 80) ? "" : (":" + port));
@@ -424,7 +463,14 @@ public final class httpc {
     * @param ssl Wether we should use SSL.
     * @throws IOException
     */
-    void init(String server, int port, int timeout, boolean ssl) throws IOException {
+    void init(
+            String server, 
+            int port, 
+            int timeout, 
+            boolean ssl,
+            String incomingByteCountAccounting,
+            String outgoingByteCountAccounting
+    ) throws IOException {
         this.handle = System.currentTimeMillis();
         //serverLog.logDebug("HTTPC", handle + " initialized");
         this.remoteProxyUse = false;
@@ -466,10 +512,19 @@ public final class httpc {
             //socket.setSoLinger(true, timeout);
             this.socket.setKeepAlive(true); //
 
+            if (incomingByteCountAccounting != null) {
+                this.clientInputByteCount = new httpdByteCountInputStream(this.socket.getInputStream(),incomingByteCountAccounting);
+            }
+            
             // getting input and output streams
-            this.clientInput  = new PushbackInputStream(this.socket.getInputStream());
+            this.clientInput  = new PushbackInputStream((this.clientInputByteCount!=null)?
+                                this.clientInputByteCount:
+                                this.socket.getInputStream()); 
             this.clientOutput = this.socket.getOutputStream();
             
+
+                                  
+
             // if we reached this point, we should have a connection
         } catch (UnknownHostException e) {
             if (this.socket != null) {
@@ -479,6 +534,14 @@ public final class httpc {
             this.socketOwner = null;
             throw new IOException("unknown host: " + server);
         }
+    }    
+    
+    public long getInputStreamByteCount() {
+        return (this.clientInputByteCount == null)?0:this.clientInputByteCount.getCount();
+    }
+    
+    public long getOutputStreamByteCount() {
+        return (this.clientOutputByteCount == null)?0:this.clientOutputByteCount.getCount();
     }
 
     /**
@@ -501,6 +564,15 @@ public final class httpc {
             this.socket = null;
             this.socketOwner = null;
         }
+        
+        if (this.clientInputByteCount != null) {
+            try { this.clientInputByteCount.finish();} catch (IOException e) {}
+            this.clientInputByteCount = null;
+        }
+        if (this.clientOutputByteCount != null) {
+            try { this.clientOutputByteCount.finish();} catch (IOException e) {}
+            this.clientOutputByteCount = null;
+        }
 
         this.host = null;
         this.timeout = 0;

source/de/anomic/http/httpdByteCountInputStream.java

@@ -3,21 +3,26 @@ package de.anomic.http;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Iterator;
 
 public final class httpdByteCountInputStream extends FilterInputStream {
     
     private static final Object syncObject = new Object();
+    private static final HashMap byteCountInfo = new HashMap(2);
     private static long globalByteCount = 0;
     
     private boolean finished = false;
     private long byteCount;
+    private String byteCountAccountName = null; 
 
     /**
      * Constructor of this class
      * @param inputStream the {@link InputStream} to read from
      */
-    public httpdByteCountInputStream(InputStream inputStream) {
+    public httpdByteCountInputStream(InputStream inputStream, String accountName) {
         super(inputStream);
+        this.byteCountAccountName = accountName;
     }
     
     /**
@@ -25,9 +30,10 @@ public final class httpdByteCountInputStream extends FilterInputStream {
      * @param inputStream the {@link InputStream} to read from
      * @param initByteCount to initialize the bytecount with a given value
      */
-    public httpdByteCountInputStream(InputStream inputStream, int initByteCount) {
+    public httpdByteCountInputStream(InputStream inputStream, int initByteCount, String accountName) {
         super(inputStream);
         this.byteCount = initByteCount;
+        this.byteCountAccountName = accountName;
     }  
     
     public int read(byte[] b) throws IOException {
@@ -51,19 +57,55 @@ public final class httpdByteCountInputStream extends FilterInputStream {
         return this.byteCount;
     }
     
+    public String getAccountName() {
+        return this.byteCountAccountName;
+    }
+    
     public static long getGlobalCount() {
         synchronized (syncObject) {
             return globalByteCount;
         }
     }
     
+    public static long getAccountCount(String accountName) {
+        synchronized (syncObject) {
+            if (byteCountInfo.containsKey(accountName)) {
+                return ((Long)byteCountInfo.get(accountName)).longValue();
+            }
+            return 0;
+        }
+    }
+    
+    public void close() throws IOException {
+        super.close();
+        this.finish();
+    }
+    
     public void finish() throws IOException {
+        if (this.finished) return;
+        
         this.finished = true;
         synchronized (syncObject) {
             globalByteCount += this.byteCount;
+            if (this.byteCountAccountName != null) {
+                long lastByteCount = 0;
+                if (byteCountInfo.containsKey(this.byteCountAccountName)) {
+                    lastByteCount = ((Long)byteCountInfo.get(this.byteCountAccountName)).longValue();
+                }
+                lastByteCount += this.byteCount;
+                byteCountInfo.put(this.byteCountAccountName,new Long(lastByteCount));
+            }
+            
         }        
     }
     
+    public static void resetCount() {
+        synchronized (syncObject) {
+            globalByteCount = 0;
+            byteCountInfo.clear();
+        }
+    }
+    
     protected void finalize() throws Throwable {
         if (!this.finished) 
             finish();

source/de/anomic/http/httpdByteCountOutputStream.java

@@ -62,7 +62,15 @@ public final class httpdByteCountOutputStream extends BufferedOutputStream {
         }
     }
     
+    public static void resetCount() {
+        synchronized (syncObject) {
+            globalByteCount = 0;
+        }
+    }    
+    
     public void finish() throws IOException {
+        if (this.finished) return;
+        
         this.finished = true;
         synchronized (syncObject) {
             globalByteCount += this.byteCount;

source/de/anomic/plasma/plasmaCrawlWorker.java

@@ -354,8 +354,8 @@ public final class plasmaCrawlWorker extends Thread {
                     
             // open the connection
             remote = ((theRemoteProxyConfig != null) && (theRemoteProxyConfig.useProxy())) 
-                   ? httpc.getInstance(host, port, socketTimeout, ssl, theRemoteProxyConfig)
-                   : httpc.getInstance(host, port, socketTimeout, ssl);
+                   ? httpc.getInstance(host, port, socketTimeout, ssl, theRemoteProxyConfig,"CRAWLER",null)
+                   : httpc.getInstance(host, port, socketTimeout, ssl, "CRAWLER",null);
             
             // specifying if content encoding is allowed
             remote.setAllowContentEncoding(useContentEncodingGzip);