boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

counting crawler traffic again:

Browse Source 
fix for http://forum.yacy-websuche.de/viewtopic.php?f=6&t=2808

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7138 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 14 years ago
parent 547d5226ae
commit 17eebd4ef8
7 changed files with 169 additions and 83 deletions
Show Stats Download Patch File Download Diff File

14
htroot/Status.java
Unescape View File

@ -32,8 +32,9 @@ import java.util.Date;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.io.ByteCountInputStream;
import net.yacy.kelondro.io.ByteCountOutputStream;
import net.yacy.kelondro.io.ByteCount;
//import net.yacy.kelondro.io.ByteCountInputStream;
//import net.yacy.kelondro.io.ByteCountOutputStream;
import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl;
@ -83,8 +84,9 @@ public class Status {
sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
redirect = true;
} else if (post.containsKey("ResetTraffic")) {
ByteCountInputStream.resetCount();
ByteCountOutputStream.resetCount();
// ByteCountInputStream.resetCount();
// ByteCountOutputStream.resetCount();
ByteCount.resetCount();
redirect = true;
} else if (post.containsKey("popup")) {
final String trigger_enabled = post.get("popup");
@ -292,8 +294,8 @@ public class Status {
// proxy traffic
//prop.put("trafficIn",bytesToString(httpdByteCountInputStream.getGlobalCount()));
prop.put("trafficProxy", Formatter.bytesToString(ByteCountOutputStream.getAccountCount("PROXY")));
prop.put("trafficCrawler", Formatter.bytesToString(ByteCountInputStream.getAccountCount("CRAWLER")));
prop.put("trafficProxy", Formatter.bytesToString(ByteCount.getAccountCount(ByteCount.PROXY)));
prop.put("trafficCrawler", Formatter.bytesToString(ByteCount.getAccountCount(ByteCount.CRAWLER)));
// connection information
final serverCore httpd = (serverCore) sb.getThread("10_httpd");

11
htroot/api/status_p.java
Unescape View File

@ -1,8 +1,9 @@
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.io.ByteCountInputStream;
import net.yacy.kelondro.io.ByteCountOutputStream;
import net.yacy.kelondro.io.ByteCount;
//import net.yacy.kelondro.io.ByteCountInputStream;
//import net.yacy.kelondro.io.ByteCountOutputStream;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.workflow.WorkflowProcessor;
import de.anomic.search.Segment;
@ -49,9 +50,9 @@ public class status_p {
prop.putNum("processors", WorkflowProcessor.availableCPU);
// proxy traffic
prop.put("trafficIn", ByteCountInputStream.getGlobalCount());
prop.put("trafficProxy", ByteCountOutputStream.getAccountCount("PROXY"));
prop.put("trafficCrawler", ByteCountInputStream.getAccountCount("CRAWLER"));
prop.put("trafficIn", ByteCount.getGlobalCount());
prop.put("trafficProxy", ByteCount.getAccountCount(ByteCount.PROXY));
prop.put("trafficCrawler", ByteCount.getAccountCount(ByteCount.CRAWLER));
// return rewrite properties
return prop;

5
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -43,6 +43,7 @@ import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.kelondro.blob.BEncodedHeap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.io.ByteCount;
import de.anomic.crawler.retrieval.HTTPLoader;
@ -347,6 +348,10 @@ public class RobotsTxt {
// sending the get request
robotsTxt = client.GETbytes(robotsURL.toString());
// statistics:
if (robotsTxt != null) {
ByteCount.addAccountCount(ByteCount.CRAWLER, robotsTxt.length);
}
final int code = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());

9
source/de/anomic/crawler/retrieval/HTTPLoader.java
Unescape View File

@ -34,6 +34,7 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.io.ByteCount;
import net.yacy.kelondro.logging.Log;
import net.yacy.repository.Blacklist;
@ -137,9 +138,10 @@ public final class HTTPLoader {
// the transfer is ok
// we write the new cache entry to file system directly
// TODO: res.setAccountingName("CRAWLER");
// res.setAccountingName("CRAWLER");
// final byte[] responseBody = res.getData();
long contentLength = responseBody.length;
ByteCount.addAccountCount(ByteCount.CRAWLER, contentLength);
// check length again in case it was not possible to get the length before loading
if (maxFileSize > 0 && contentLength > maxFileSize) {
@ -268,8 +270,11 @@ public final class HTTPLoader {
if (responseBody != null && (code == 200 || code == 203)) {
// the transfer is ok
//statistics:
ByteCount.addAccountCount(ByteCount.CRAWLER, responseBody.length);
// we write the new cache entry to file system directly
// TODO: res.setAccountingName("CRAWLER");
// res.setAccountingName("CRAWLER");
// final byte[] responseBody = res.getData();
// create a new cache entry

71
source/net/yacy/kelondro/io/ByteCount.java
Unescape View File

@ -0,0 +1,71 @@
//ByteCount.java
//-----------------------
//(C) by Michael Peter Christen; mc@yacy.net
//first published on http://www.anomic.de
//Frankfurt, Germany, 2004
//
// This file is contributed by Sebastian Gäbel
// last major change: $LastChangedDate: 2010-04-23 11:32:29 +0200 (Fr, 23. Apr 2010) $ by $LastChangedBy: sixcooler $
// Revision: $LastChangedRevision: 6835 $
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
//the Free Software Foundation; either version 2 of the License, or
//(at your option) any later version.
//
//This program is distributed in the hope that it will be useful,
//but WITHOUT ANY WARRANTY; without even the implied warranty of
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//GNU General Public License for more details.
//
//You should have received a copy of the GNU General Public License
//along with this program; if not, write to the Free Software
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.io;
import java.util.HashMap;
public final class ByteCount {
public final static String PROXY = "PROXY";
public final static String CRAWLER = "CRAWLER";
private final static Object syncObject = new Object();
private static long globalCount = 0;
private final static HashMap<String, Long> countMap = new HashMap<String, Long>(2);
public final static long getGlobalCount() {
return globalCount;
}
public final static long getAccountCount(final String accountName) {
synchronized (syncObject) {
if (countMap.containsKey(accountName)) {
return (countMap.get(accountName)).longValue();
}
return 0;
}
}
public final static void addAccountCount(final String accountName, final long count) {
synchronized (syncObject) {
globalCount += count;
if (accountName != null) {
long current = 0;
if (countMap.containsKey(accountName)) {
current = (countMap.get(accountName)).longValue();
}
current += count;
countMap.put(accountName, current);
}
}
}
public final static void resetCount() {
synchronized (syncObject) {
globalCount = 0;
countMap.clear();
}
}
}

71
source/net/yacy/kelondro/io/ByteCountInputStream.java
Unescape View File

@ -29,13 +29,13 @@ package net.yacy.kelondro.io;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
//import java.util.HashMap;
public final class ByteCountInputStream extends FilterInputStream {
private final static Object syncObject = new Object();
private final static HashMap<String, Long> byteCountInfo = new HashMap<String, Long>(2);
private static long globalByteCount = 0;
// private final static Object syncObject = new Object();
// private final static HashMap<String, Long> byteCountInfo = new HashMap<String, Long>(2);
// private static long globalByteCount = 0;
private boolean finished = false;
protected long byteCount;
@ -99,20 +99,20 @@ public final class ByteCountInputStream extends FilterInputStream {
return this.byteCountAccountName;
}
public final static long getGlobalCount() {
synchronized (syncObject) {
return globalByteCount;
}
}
// public final static long getGlobalCount() {
// synchronized (syncObject) {
// return globalByteCount;
// }
// }
public final static long getAccountCount(final String accountName) {
synchronized (syncObject) {
if (byteCountInfo.containsKey(accountName)) {
return (byteCountInfo.get(accountName)).longValue();
}
return 0;
}
}
// public final static long getAccountCount(final String accountName) {
// synchronized (syncObject) {
// if (byteCountInfo.containsKey(accountName)) {
// return (byteCountInfo.get(accountName)).longValue();
// }
// return 0;
// }
// }
public final void close() throws IOException {
super.close();
@ -123,24 +123,25 @@ public final class ByteCountInputStream extends FilterInputStream {
if (this.finished) return;
this.finished = true;
synchronized (syncObject) {
globalByteCount += this.byteCount;
if (this.byteCountAccountName != null) {
long lastByteCount = 0;
if (byteCountInfo.containsKey(this.byteCountAccountName)) {
lastByteCount = byteCountInfo.get(this.byteCountAccountName).longValue();
}
lastByteCount += this.byteCount;
byteCountInfo.put(this.byteCountAccountName, Long.valueOf(lastByteCount));
}
}
ByteCount.addAccountCount(this.byteCountAccountName, this.byteCount);
// synchronized (syncObject) {
// globalByteCount += this.byteCount;
// if (this.byteCountAccountName != null) {
// long lastByteCount = 0;
// if (byteCountInfo.containsKey(this.byteCountAccountName)) {
// lastByteCount = byteCountInfo.get(this.byteCountAccountName).longValue();
// }
// lastByteCount += this.byteCount;
// byteCountInfo.put(this.byteCountAccountName, Long.valueOf(lastByteCount));
// }
//
// }
}
public final static void resetCount() {
synchronized (syncObject) {
globalByteCount = 0;
byteCountInfo.clear();
}
}
// public final static void resetCount() {
// synchronized (syncObject) {
// globalByteCount = 0;
// byteCountInfo.clear();
// }
// }
}

75
source/net/yacy/kelondro/io/ByteCountOutputStream.java
Unescape View File

@ -27,13 +27,13 @@ package net.yacy.kelondro.io;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.HashMap;
//import java.util.HashMap;
public final class ByteCountOutputStream extends BufferedOutputStream {
private final static Object syncObject = new Object();
private static long globalByteCount = 0;
private final static HashMap<String, Long> byteCountInfo = new HashMap<String, Long>(2);
// private final static Object syncObject = new Object();
// private static long globalByteCount = 0;
// private final static HashMap<String, Long> byteCountInfo = new HashMap<String, Long>(2);
protected long byteCount;
protected String byteCountAccountName = null;
@ -92,44 +92,45 @@ public final class ByteCountOutputStream extends BufferedOutputStream {
return this.byteCountAccountName;
}
public final static long getGlobalCount() {
synchronized (syncObject) {
return globalByteCount;
}
}
public final static long getAccountCount(final String accountName) {
synchronized (syncObject) {
if (byteCountInfo.containsKey(accountName)) {
return (byteCountInfo.get(accountName)).longValue();
}
return 0;
}
}
public final static void resetCount() {
synchronized (syncObject) {
globalByteCount = 0;
byteCountInfo.clear();
}
}
// public final static long getGlobalCount() {
// synchronized (syncObject) {
// return globalByteCount;
// }
// }
// public final static long getAccountCount(final String accountName) {
// synchronized (syncObject) {
// if (byteCountInfo.containsKey(accountName)) {
// return (byteCountInfo.get(accountName)).longValue();
// }
// return 0;
// }
// }
// public final static void resetCount() {
// synchronized (syncObject) {
// globalByteCount = 0;
// byteCountInfo.clear();
// }
// }
public final void finish() {
if (this.finished) return;
this.finished = true;
synchronized (syncObject) {
globalByteCount += this.byteCount;
if (this.byteCountAccountName != null) {
long lastByteCount = 0;
if (byteCountInfo.containsKey(this.byteCountAccountName)) {
lastByteCount = (byteCountInfo.get(this.byteCountAccountName)).longValue();
}
lastByteCount += this.byteCount;
byteCountInfo.put(this.byteCountAccountName, Long.valueOf(lastByteCount));
}
}
ByteCount.addAccountCount(this.byteCountAccountName, this.byteCount);
// synchronized (syncObject) {
// globalByteCount += this.byteCount;
// if (this.byteCountAccountName != null) {
// long lastByteCount = 0;
// if (byteCountInfo.containsKey(this.byteCountAccountName)) {
// lastByteCount = (byteCountInfo.get(this.byteCountAccountName)).longValue();
// }
// lastByteCount += this.byteCount;
// byteCountInfo.put(this.byteCountAccountName, Long.valueOf(lastByteCount));
// }
//
// }
}
}

Write Preview
Loading…
Cancel
Save