From db294687ea13ec61e31583c505e65aaba553e902 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 3 Oct 2006 22:55:59 +0000 Subject: [PATCH] enhanced logging - more logging output - fix in log line preparation - added filter to log page - some small bugfixes git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2707 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .classpath | 72 +++++++++---------- htroot/ViewLog_p.html | 1 + htroot/ViewLog_p.java | 20 ++++-- htroot/env/base.css | 2 +- .../kelondro/kelondroRowCollection.java | 2 +- source/de/anomic/kelondro/kelondroRowSet.java | 37 +++++++++- .../de/anomic/plasma/plasmaSearchEvent.java | 11 +-- source/de/anomic/plasma/plasmaWordIndex.java | 3 +- .../de/anomic/server/logging/GuiHandler.java | 2 +- source/de/anomic/yacy/yacyClient.java | 13 ++-- source/de/anomic/yacy/yacySearch.java | 20 +++--- 11 files changed, 117 insertions(+), 66 deletions(-) diff --git a/.classpath b/.classpath index c9eba99d5..28b75dfde 100644 --- a/.classpath +++ b/.classpath @@ -1,36 +1,36 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/htroot/ViewLog_p.html b/htroot/ViewLog_p.html index 93d9138d5..0cc33c260 100644 --- a/htroot/ViewLog_p.html +++ b/htroot/ViewLog_p.html @@ -12,6 +12,7 @@
+
diff --git a/htroot/ViewLog_p.java b/htroot/ViewLog_p.java index 1f59be1f4..050a28ea6 100644 --- a/htroot/ViewLog_p.java +++ b/htroot/ViewLog_p.java @@ -48,6 +48,7 @@ import java.util.logging.Handler; import java.util.logging.Logger; +import java.util.regex.PatternSyntaxException; import de.anomic.http.httpHeader; import de.anomic.server.serverObjects; @@ -60,7 +61,8 @@ public class ViewLog_p { serverObjects prop = new serverObjects(); String[] log = new String[0]; boolean reversed = false; - int lines = 50; + int lines = 200; + String filter = ".*.*"; if(post != null){ if(post.containsKey("mode") && ((String)post.get("mode")).equals("reversed")){ @@ -69,6 +71,9 @@ public class ViewLog_p { if(post.containsKey("lines")){ lines = Integer.parseInt((String)post.get("lines")); } + if(post.containsKey("filter")){ + filter = (String)post.get("filter"); + } } @@ -83,21 +88,26 @@ public class ViewLog_p { prop.put("reverseChecked", reversed ? 1 : 0); prop.put("lines", lines); + prop.put("filter", filter); int level = 0; + int lc = 0; for (int i=0; i < log.length; i++) { - String nextLogLine = log[i]; + String nextLogLine = log[i].trim(); + try {if (!(nextLogLine.matches(filter))) continue;} catch (PatternSyntaxException e) {} + if (nextLogLine.startsWith("E ")) level = 4; else if (nextLogLine.startsWith("W ")) level = 3; else if (nextLogLine.startsWith("S ")) level = 2; else if (nextLogLine.startsWith("I ")) level = 1; else if (nextLogLine.startsWith("D ")) level = 0; - prop.put("log_" + i + "_level",level); - prop.put("log_" + i + "_line", nextLogLine); + prop.put("log_" + lc + "_level",level); + prop.put("log_" + lc + "_line", nextLogLine); + lc++; } - prop.put("log",log.length); + prop.put("log",lc); // return rewrite properties return prop; diff --git a/htroot/env/base.css b/htroot/env/base.css index 277def36d..6108c96e8 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -341,7 +341,7 @@ dl.pairs dd { /* for pages: */ body#ViewLog div.log { - height:480px; + height:600px; overflow:scroll; } diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java index d0d62a940..2fb0a4459 100644 --- a/source/de/anomic/kelondro/kelondroRowCollection.java +++ b/source/de/anomic/kelondro/kelondroRowCollection.java @@ -214,7 +214,7 @@ public class kelondroRowCollection { int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart)); synchronized (chunkcache) { ensureSize(chunkcount + 1); - System.arraycopy(a, 0, chunkcache, rowdef.objectsize() * chunkcount, l); + System.arraycopy(a, astart, chunkcache, rowdef.objectsize() * chunkcount, l); chunkcount++; } this.lastTimeWrote = System.currentTimeMillis(); diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index 68985b1e8..fc00799a1 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -447,7 +447,8 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd } System.out.println("RESULT SIZE: " + c.size()); */ - + /* + // performance test for put long start = System.currentTimeMillis(); kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0); Random random = new Random(0); @@ -459,6 +460,40 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd } System.out.println("RESULT SIZE: " + c.size()); System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); + */ + + // remove test + long start = System.currentTimeMillis(); + kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0); + byte[] key; + int testsize = 5000; + byte[][] delkeys = new byte[testsize / 5][]; + Random random = new Random(0); + for (int i = 0; i < testsize; i++) { + key = randomHash(random); + if (i % 5 != 0) continue; + delkeys[i / 5] = key; + } + random = new Random(0); + for (int i = 0; i < testsize; i++) { + key = randomHash(random); + c.put(c.rowdef.newEntry(new byte[][]{key, key})); + if (i % 1000 == 0) { + for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]); + c.shape(); + } + } + for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]); + c.shape(); + random = new Random(0); + for (int i = 0; i < testsize; i++) { + key = randomHash(random); + if (i % 5 == 0) continue; + if (c.get(key) == null) System.out.println("missing entry " + new String(key)); + } + c.shape(); + System.out.println("RESULT SIZE: " + c.size()); + System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); } public static byte[] randomHash(final long r0, final long r1) { diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index bd4463586..71da580a2 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -70,7 +70,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable { private plasmaCrawlLURL urlStore; private plasmaSnippetCache snippetCache; private indexContainer rcContainers; // cache for results - private int rcContainerCount; + private int rcContainerFlushCount; private Map rcAbstracts; // cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation private plasmaSearchTimingProfile profileLocal, profileGlobal; private boolean postsort; @@ -92,7 +92,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable { this.urlStore = urlStore; this.snippetCache = snippetCache; this.rcContainers = new indexContainer(null); - this.rcContainerCount = 0; + this.rcContainerFlushCount = 0; this.rcAbstracts = new TreeMap(); this.profileLocal = localTiming; this.profileGlobal = remoteTiming; @@ -434,7 +434,8 @@ public final class plasmaSearchEvent extends Thread implements Runnable { // this must be called after search results had been computed // it is wise to call this within a separate thread because // this method waits until all threads are finished - + serverLog.logFine("PLASMA", "STARTED FLUSHING GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords); + int remaining = 0; if (primarySearchThreads == null) return; long starttime = System.currentTimeMillis(); @@ -456,7 +457,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable { //log.logFine("FINISHED FLUSH RESULTS PROCESS for query " + query.hashes(",")); } - serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords); + serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerFlushCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords); // finally delete the temporary index rcContainers = null; @@ -483,7 +484,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable { rcContainers.clear(); } } - rcContainerCount += count; + rcContainerFlushCount += count; } } diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 001f564d8..df03367d6 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -363,8 +363,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { if (container == null) { container = dhtInCache.getContainer(wordHash, urlselection, true, -1); } else { - indexContainer ic = dhtInCache.getContainer(wordHash, urlselection, true, -1); - if (ic != null) container.add(ic, -1); + container.add(dhtInCache.getContainer(wordHash, urlselection, true, -1), -1); } // get from collection index diff --git a/source/de/anomic/server/logging/GuiHandler.java b/source/de/anomic/server/logging/GuiHandler.java index c509ffcc4..b53b86865 100644 --- a/source/de/anomic/server/logging/GuiHandler.java +++ b/source/de/anomic/server/logging/GuiHandler.java @@ -201,7 +201,7 @@ public class GuiHandler extends Handler{ for (int i = 0; i < lineCount; i++) { int ix = (reversed) ? Math.abs((start-i)%this.buffer.length) : - (start+i)%this.buffer.length; + (start - lineCount + i) % this.buffer.length; record = this.buffer[ix]; logMessages.add(logFormatter.format(record)); } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 67ef67438..6da6f90a7 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -364,7 +364,7 @@ public final class yacyClient { } } - public static int search( + public static String[] search( String wordhashes, String urlhashes, String prefer, @@ -454,12 +454,12 @@ public final class yacyClient { } catch (IOException e) { yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes)); yacyCore.peerActions.peerDeparture(targetPeer); - return 0; + return null; } if (result.size() == 0) { yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (zero response), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes)); - return 0; + return null; } // compute all computation times @@ -497,6 +497,7 @@ public final class yacyClient { // insert results to containers plasmaCrawlLURL.Entry urlEntry; + String[] urls = new String[results]; for (int n = 0; n < results; n++) { // get one single search result urlEntry = urlManager.newEntry((String) result.get("resource" + n), true); @@ -540,6 +541,8 @@ public final class yacyClient { for (int m = 0; m < words; m++) { container[m].add(new indexEntry[]{entry}, System.currentTimeMillis()); } + // store url hash for statistics + urls[n] = urlEntry.hash(); } // insert the containers to the index @@ -574,11 +577,11 @@ public final class yacyClient { searchtime = totalrequesttime; } yacyCore.log.logFine("SEARCH " + results + " URLS FROM " + targetPeer.hash + ":" + targetPeer.getName() + ", score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes) + ", duetime=" + duetime + ", searchtime=" + searchtime + ", netdelay=" + (totalrequesttime - searchtime) + ", references=" + result.get("references")); - return results; + return urls; } catch (Exception e) { yacyCore.log.logSevere("yacyClient.search error: '" + targetPeer.get(yacySeed.NAME, "anonymous") + "' failed - " + e); e.printStackTrace(); - return 0; + return null; } } diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 27fa4ce07..2cfb7ac83 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -70,7 +70,7 @@ public class yacySearch extends Thread { final private plasmaURLPattern blacklist; final private plasmaSnippetCache snippetCache; final private yacySeed targetPeer; - private int links; + private String[] urls; private int maxDistance; final private plasmaSearchTimingProfile timingProfile; final private plasmaSearchRankingProfile rankingProfile; @@ -94,18 +94,20 @@ public class yacySearch extends Thread { this.blacklist = blacklist; this.snippetCache = snippetCache; this.targetPeer = targetPeer; - this.links = -1; + this.urls = null; this.maxDistance = maxDistance; this.timingProfile = (plasmaSearchTimingProfile) timingProfile.clone(); this.rankingProfile = rankingProfile; } public void run() { - this.links = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile); - yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes); - if (links != 0) { - yacyCore.seedDB.mySeed.incRI(links); - yacyCore.seedDB.mySeed.incRU(links); + this.urls = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile); + StringBuffer urllist = new StringBuffer(this.urls.length * 13); + for (int i = 0; i < this.urls.length; i++) urllist.append(this.urls[i]).append(' '); + yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + urls.length + " links for word hash " + wordhashes + ": " + new String(urllist)); + if (urls != null) { + yacyCore.seedDB.mySeed.incRI(urls.length); + yacyCore.seedDB.mySeed.incRU(urls.length); } } @@ -117,7 +119,7 @@ public class yacySearch extends Thread { } public int links() { - return this.links; + return this.urls.length; } public plasmaSearchTimingProfile timingProfile() { @@ -241,7 +243,7 @@ public class yacySearch extends Thread { public static int collectedLinks(yacySearch[] searchThreads) { int links = 0; for (int i = 0; i < searchThreads.length; i++) { - if (!(searchThreads[i].isAlive())) links += searchThreads[i].links; + if (!(searchThreads[i].isAlive())) links += searchThreads[i].urls.length; } return links; }