boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more + changed log for better understanding of outOfMemory bug and others

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@846 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent e5f8163203
commit 0c3a20d44f
5 changed files with 30 additions and 22 deletions
Show Stats Download Patch File Download Diff File

26
htroot/Network.html
Unescape View File

@ -40,11 +40,6 @@
<tr class="TableHeader" valign="bottom">
<td class="small">send&nbsp;<b>M</b>essage/<br>show&nbsp;<b>P</b>rofile/<br>edit&nbsp;<b>W</b>iki<br>&nbsp;</td>
<td class="small"><b>Name</b><br>&nbsp;</td>
#(complete)#::
<td class="small"><b>Address</b><br>&nbsp;</td>
<td class="small"><b>Hash</b><br>&nbsp;</td>
<td class="small"><b>Age</b><br>&nbsp;</td>
#(/complete)#
<td class="small"><b>Info</b><br>&nbsp;</td>
<td class="small"><b>Release/<br>SVN</b><br>&nbsp;</td>
<td class="small"><b>Last<br>Seen</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=LastSeen&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=LastSeen&order=down">&gt;</a></td>
@ -58,7 +53,13 @@
<td class="small"><b>Received<br>URLs</b><br>&nbsp;</td>
<td class="small"><b>PPM</b><br>&nbsp;</td>
<td class="small"><b>#Seeds</b><br>&nbsp;</td>
<td class="small"><b>#Connects<br>per hour</b><br>&nbsp;</td></tr>
#(complete)#::
<td class="small"><b>Address</b><br>&nbsp;</td>
<td class="small"><b>Hash</b><br>&nbsp;</td>
<td class="small"><b>Age</b><br>&nbsp;</td>
<td class="small"><b>con/h<br></b><br>&nbsp;</td>
#(/complete)#
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td class="small"><a href="MessageSend_p.html?hash=#[hash]#" class="small" title="Send message to peer #[fullname]#">m</a>&nbsp;&nbsp;
@ -66,13 +67,6 @@
<a href="http://#[fullname]#.yacy/Wiki.html#[updatedWikiPage]#" class="small" title="Read and edit wiki on peer #[fullname]#">w</a>&nbsp;&nbsp;#[alert]#
</td>
<td class="small"><a href="http://www.#[fullname]#.yacy" class="small">#[shortname]#</a></td>
#(complete)#
::
<td class="small"><a href="http://#[ip]#:#[port]#/Network.html?page=1&ip=" class="small">http://#[ip]#:#[port]#</a></td>
<td class=small>#[hash]#</td>
<td class=small>#[age]#</td>
#(/complete)#
<td class="small">#(info)##(direct)#<img border="0" src="/env/grafics/JuniorPassive.gif" title="Type: Junior | Contact: passive"></img>::<img border="0" src="/env/grafics/JuniorDirect.gif" title="Type: Junior | Contact: direct"></img>#(/direct)#::#(direct)#<img border="0" src="/env/grafics/SeniorPassive.gif" title="Type: Senior | Contact: passive"></img>::<img border="0" src="/env/grafics/SeniorDirect.gif" title="Type: Senior | Contact: direct"></img>#(/direct)#::<a href="#[url]#">#(direct)#<img border="0" src="/env/grafics/PrincipalPassive.gif" title="Type: Principal | Contact: passive | Seed download: possible"></img>::<img border="0" src="/env/grafics/PrincipalDirect.gif" title="Type: Principal | Contact: direct | Seed download: possible"></img>#(/direct)#</a>#(/info)##(acceptcrawl)#<img border="0" src="/env/grafics/CrawlNo.gif" title="Accept Crawl: no"></img>::<img border="0" src="/env/grafics/CrawlYes.gif" title="Accept Crawl: yes"></img>#(/acceptcrawl)##(dhtreceive)#<img border="0" src="/env/grafics/DHTReceiveNo.gif" title="DHT Receive: no"></img>::<img border="0" src="/env/grafics/DHTReceiveYes.gif" title="DHT Receive: yes"></img>#(/dhtreceive)#</td>
<td class="small" align="right">#[version]#</td>
<td class="small" align="right">#[lastSeen]#</td>
@ -86,7 +80,13 @@
<td class="small" align="right">#[rU]#</td>
<td class="small" align="right">#[ppm]#</td>
<td class="small" align="right">#[seeds]#</td>
#(complete)#
::
<td class="small"><a href="http://#[ip]#:#[port]#/Network.html?page=1&ip=" class="small">http://#[ip]#:#[port]#</a></td>
<td class=small>#[hash]#</td>
<td class=small>#[age]#</td>
<td class="small" align="right">#[connects]#</td>
#(/complete)#
</tr>
#{/list}#
</table>

2
htroot/Network.java
Unescape View File

@ -302,6 +302,7 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_complete_port", seed.get("Port", "-") );
prop.put(STR_TABLE_LIST + conCount + "_complete_hash", seed.hash);
prop.put(STR_TABLE_LIST + conCount + "_complete_age", seed.getAge());
prop.put(STR_TABLE_LIST + conCount + "_complete_connects", groupDigits(seed.get("CCount", "0")));
} else {
prop.put(STR_TABLE_LIST + conCount + "_complete", 0);
}
@ -338,7 +339,6 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_rU", groupDigits(seed.get(yacySeed.URL_IN, "0")));
prop.put(STR_TABLE_LIST + conCount + "_ppm", PPM);
prop.put(STR_TABLE_LIST + conCount + "_seeds", seed.get("SCount", "-"));
prop.put(STR_TABLE_LIST + conCount + "_connects", groupDigits(seed.get("CCount", "0")));
conCount++;
} // seed != null
} // while

10
htroot/yacy/transferRWI.java
Unescape View File

@ -84,6 +84,10 @@ public class transferRWI {
String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if (granted) {
// log value status (currently added to find outOfMemory error
switchboard.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName);
long startProcess = System.currentTimeMillis();
// decode request
Vector v = new Vector();
int s = 0;
@ -94,7 +98,7 @@ public class transferRWI {
s = e; while (s < indexes.length) if (indexes[s++] >= 32) {s--; break;}
}
// the value-vector should now have the same length as entryc
if (v.size() != entryc) System.out.println("ERROR WITH ENTRY COUNTER: v=" + v.size() + ", entryc=" + entryc);
if (v.size() != entryc) switchboard.getLog().logSevere("ERROR WITH ENTRY COUNTER: v=" + v.size() + ", entryc=" + entryc);
// now parse the Strings in the value-vector and write index entries
String estring;
@ -128,10 +132,10 @@ public class transferRWI {
while (it.hasNext()) unknownURLs += "," + (String) it.next();
if (unknownURLs.length() > 0) unknownURLs = unknownURLs.substring(1);
if (wordhashes.length == 0)
switchboard.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", requested " + unknownURL.size() + " URLs");
switchboard.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs");
else {
double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[wordhashes.length - 1])) / 2.0;
switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from " + otherPeerName + ", requested " + unknownURL.size() + " URLs");
switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs");
}
result = "ok";
} else {

4
source/de/anomic/plasma/plasmaCrawlWorker.java
Unescape View File

@ -410,8 +410,8 @@ public final class plasmaCrawlWorker extends Thread {
remote = null;
// restart crawling with new url
log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + url.toString() +
"\nRedirecting request to: " + redirectionUrl);
log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + url.toString());
log.logInfo("CRAWLER ..Redirecting request to: " + redirectionUrl);
// if we are already doing a shutdown we don't need to retry crawling
if (Thread.currentThread().isInterrupted()) {

10
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -145,6 +145,7 @@ import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySearch;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyNewsPool;
public final class plasmaSwitchboard extends serverAbstractSwitch implements serverSwitch {
@ -173,7 +174,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public String remoteProxyHost;
public int remoteProxyPort;
public boolean remoteProxyUse;
public static plasmaCrawlRobotsTxt robots;
public static plasmaCrawlRobotsTxt robots;
public plasmaCrawlProfile profiles;
public plasmaCrawlProfile.entry defaultProxyProfile;
public plasmaCrawlProfile.entry defaultRemoteProfile;
@ -185,7 +186,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public long proxyLastAccess;
public yacyCore yc;
public HashMap indexingTasksInProcess;
public userDB userDB;
public userDB userDB;
private static final String STR_PROXYPROFILE = "defaultProxyProfile";
private static final String STR_REMOTEPROFILE = "defaultRemoteProfile";
@ -700,13 +701,15 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// clean up error stack
if ((urlPool.errorURL.stackSize() > 1000)) {
log.logFine("Cleaning Error-URLs report stack, " + urlPool.errorURL.stackSize() + " entries on stack");
urlPool.errorURL.clearStack();
hasDoneSomething = true;
}
// clean up loadedURL stack
for (int i = 1; i <= 6; i++) {
if (urlPool.loadedURL.getStackSize(i) > 1000) {
urlPool.loadedURL.clearStack(i);
log.logFine("Cleaning Loaded-URLs report stack, " + urlPool.loadedURL.getStackSize(i) + " entries on stack " + i);
urlPool.loadedURL.clearStack(i);
hasDoneSomething = true;
}
}
@ -715,6 +718,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// clean up news
try {
log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true;
} catch (IOException e) {}

Write Preview
Loading…
Cancel
Save