From 0c3a20d44ff49615786700c71da39f6ea62d2dd7 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 4 Oct 2005 00:28:59 +0000 Subject: [PATCH] more + changed log for better understanding of outOfMemory bug and others git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@846 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.html | 26 +++++++++---------- htroot/Network.java | 2 +- htroot/yacy/transferRWI.java | 10 ++++--- .../de/anomic/plasma/plasmaCrawlWorker.java | 4 +-- .../de/anomic/plasma/plasmaSwitchboard.java | 10 ++++--- 5 files changed, 30 insertions(+), 22 deletions(-) diff --git a/htroot/Network.html b/htroot/Network.html index 285df5b1f..4ad043da5 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -40,11 +40,6 @@ send Message/
show Profile/
edit Wiki
  Name
  -#(complete)#:: -Address
  -Hash
  -Age
  -#(/complete)# Info
  Release/
SVN
  Last
Seen
  < > @@ -58,7 +53,13 @@ Received
URLs
  PPM
  #Seeds
  -#Connects
per hour
  +#(complete)#:: +Address
  +Hash
  +Age
  +con/h

  +#(/complete)# + #{list}# m   @@ -66,13 +67,6 @@ w  #[alert]# #[shortname]# -#(complete)# -:: -http://#[ip]#:#[port]# -#[hash]# -#[age]# -#(/complete)# - #(info)##(direct)#::#(/direct)#::#(direct)#::#(/direct)#::#(direct)#::#(/direct)##(/info)##(acceptcrawl)#::#(/acceptcrawl)##(dhtreceive)#::#(/dhtreceive)# #[version]# #[lastSeen]# @@ -86,7 +80,13 @@ #[rU]# #[ppm]# #[seeds]# +#(complete)# +:: +http://#[ip]#:#[port]# +#[hash]# +#[age]# #[connects]# +#(/complete)# #{/list}# diff --git a/htroot/Network.java b/htroot/Network.java index 08a5c68ed..644365bcb 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -302,6 +302,7 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_complete_port", seed.get("Port", "-") ); prop.put(STR_TABLE_LIST + conCount + "_complete_hash", seed.hash); prop.put(STR_TABLE_LIST + conCount + "_complete_age", seed.getAge()); + prop.put(STR_TABLE_LIST + conCount + "_complete_connects", groupDigits(seed.get("CCount", "0"))); } else { prop.put(STR_TABLE_LIST + conCount + "_complete", 0); } @@ -338,7 +339,6 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_rU", groupDigits(seed.get(yacySeed.URL_IN, "0"))); prop.put(STR_TABLE_LIST + conCount + "_ppm", PPM); prop.put(STR_TABLE_LIST + conCount + "_seeds", seed.get("SCount", "-")); - prop.put(STR_TABLE_LIST + conCount + "_connects", groupDigits(seed.get("CCount", "0"))); conCount++; } // seed != null } // while diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index 0ecae4d5b..1f021026c 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -84,6 +84,10 @@ public class transferRWI { String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion())); if (granted) { + // log value status (currently added to find outOfMemory error + switchboard.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName); + long startProcess = System.currentTimeMillis(); + // decode request Vector v = new Vector(); int s = 0; @@ -94,7 +98,7 @@ public class transferRWI { s = e; while (s < indexes.length) if (indexes[s++] >= 32) {s--; break;} } // the value-vector should now have the same length as entryc - if (v.size() != entryc) System.out.println("ERROR WITH ENTRY COUNTER: v=" + v.size() + ", entryc=" + entryc); + if (v.size() != entryc) switchboard.getLog().logSevere("ERROR WITH ENTRY COUNTER: v=" + v.size() + ", entryc=" + entryc); // now parse the Strings in the value-vector and write index entries String estring; @@ -128,10 +132,10 @@ public class transferRWI { while (it.hasNext()) unknownURLs += "," + (String) it.next(); if (unknownURLs.length() > 0) unknownURLs = unknownURLs.substring(1); if (wordhashes.length == 0) - switchboard.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", requested " + unknownURL.size() + " URLs"); + switchboard.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs"); else { double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[wordhashes.length - 1])) / 2.0; - switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from " + otherPeerName + ", requested " + unknownURL.size() + " URLs"); + switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs"); } result = "ok"; } else { diff --git a/source/de/anomic/plasma/plasmaCrawlWorker.java b/source/de/anomic/plasma/plasmaCrawlWorker.java index 1e83df8ca..62b211665 100644 --- a/source/de/anomic/plasma/plasmaCrawlWorker.java +++ b/source/de/anomic/plasma/plasmaCrawlWorker.java @@ -410,8 +410,8 @@ public final class plasmaCrawlWorker extends Thread { remote = null; // restart crawling with new url - log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + url.toString() + - "\nRedirecting request to: " + redirectionUrl); + log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + url.toString()); + log.logInfo("CRAWLER ..Redirecting request to: " + redirectionUrl); // if we are already doing a shutdown we don't need to retry crawling if (Thread.currentThread().isInterrupted()) { diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 8fa147622..5ef4ca990 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -145,6 +145,7 @@ import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySearch; import de.anomic.yacy.yacySeed; +import de.anomic.yacy.yacyNewsPool; public final class plasmaSwitchboard extends serverAbstractSwitch implements serverSwitch { @@ -173,7 +174,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public String remoteProxyHost; public int remoteProxyPort; public boolean remoteProxyUse; - public static plasmaCrawlRobotsTxt robots; + public static plasmaCrawlRobotsTxt robots; public plasmaCrawlProfile profiles; public plasmaCrawlProfile.entry defaultProxyProfile; public plasmaCrawlProfile.entry defaultRemoteProfile; @@ -185,7 +186,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public long proxyLastAccess; public yacyCore yc; public HashMap indexingTasksInProcess; - public userDB userDB; + public userDB userDB; private static final String STR_PROXYPROFILE = "defaultProxyProfile"; private static final String STR_REMOTEPROFILE = "defaultRemoteProfile"; @@ -700,13 +701,15 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // clean up error stack if ((urlPool.errorURL.stackSize() > 1000)) { + log.logFine("Cleaning Error-URLs report stack, " + urlPool.errorURL.stackSize() + " entries on stack"); urlPool.errorURL.clearStack(); hasDoneSomething = true; } // clean up loadedURL stack for (int i = 1; i <= 6; i++) { if (urlPool.loadedURL.getStackSize(i) > 1000) { - urlPool.loadedURL.clearStack(i); + log.logFine("Cleaning Loaded-URLs report stack, " + urlPool.loadedURL.getStackSize(i) + " entries on stack " + i); + urlPool.loadedURL.clearStack(i); hasDoneSomething = true; } } @@ -715,6 +718,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // clean up news try { + log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack"); if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true; } catch (IOException e) {}