enhanced logging

- more logging output
- fix in log line preparation
- added filter to log page
- some small bugfixes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2707 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/ViewLog_p.html

@@ -12,6 +12,7 @@
       <fieldset>
         <input type="text" name="lines" id="lines" size ="4" value="#[lines]#" maxlength="9" /><label for="lines"> Lines</label>
         <input type="checkbox" name="mode" id="mode" value="reversed" #(reverseChecked)#::checked="checked"#(/reverseChecked)# /><label for="mode">reversed order</label>
+        <input type="text" name="filter" id="filter" size ="30" value="#[filter]#" maxlength="60" />
         <input type="submit" value="refresh" />
       </fieldset>
     </form>

htroot/ViewLog_p.java

@@ -48,6 +48,7 @@
 
 import java.util.logging.Handler;
 import java.util.logging.Logger;
+import java.util.regex.PatternSyntaxException;
 
 import de.anomic.http.httpHeader;
 import de.anomic.server.serverObjects;
@@ -60,7 +61,8 @@ public class ViewLog_p {
         serverObjects prop = new serverObjects();
         String[] log = new String[0];
         boolean reversed = false;
-        int lines = 50;
+        int lines = 200;
+        String filter = ".*.*";
         
         if(post != null){
             if(post.containsKey("mode") && ((String)post.get("mode")).equals("reversed")){
@@ -69,6 +71,9 @@ public class ViewLog_p {
             if(post.containsKey("lines")){
                 lines = Integer.parseInt((String)post.get("lines"));
             }
+            if(post.containsKey("filter")){
+                filter = (String)post.get("filter");
+            }
         }
         
         
@@ -83,21 +88,26 @@ public class ViewLog_p {
         
         prop.put("reverseChecked", reversed ? 1 : 0);
         prop.put("lines", lines);
+        prop.put("filter", filter);
         
 
         int level = 0;
+        int lc = 0;
         for (int i=0; i < log.length; i++) {
-            String nextLogLine = log[i];
+            String nextLogLine = log[i].trim();
+            try {if (!(nextLogLine.matches(filter))) continue;} catch (PatternSyntaxException e) {}
+            
             if (nextLogLine.startsWith("E ")) level = 4;
             else if (nextLogLine.startsWith("W ")) level = 3;
             else if (nextLogLine.startsWith("S ")) level = 2;
             else if (nextLogLine.startsWith("I ")) level = 1;
             else if (nextLogLine.startsWith("D ")) level = 0;
             
-            prop.put("log_" + i + "_level",level);
+            prop.put("log_" + lc + "_level",level);
-            prop.put("log_" + i + "_line", nextLogLine); 
+            prop.put("log_" + lc + "_line", nextLogLine); 
+            lc++;
         }
-        prop.put("log",log.length);
+        prop.put("log",lc);
         
         // return rewrite properties
         return prop;

htroot/env/base.css

@@ -341,7 +341,7 @@ dl.pairs dd {
 /* for pages: */
 
 body#ViewLog div.log {
-  height:480px;
+  height:600px;
   overflow:scroll;  
 }	
 

source/de/anomic/kelondro/kelondroRowCollection.java

@@ -214,7 +214,7 @@ public class kelondroRowCollection {
         int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
         synchronized (chunkcache) {
             ensureSize(chunkcount + 1);
-            System.arraycopy(a, 0, chunkcache, rowdef.objectsize() * chunkcount, l);
+            System.arraycopy(a, astart, chunkcache, rowdef.objectsize() * chunkcount, l);
             chunkcount++;
         }
         this.lastTimeWrote = System.currentTimeMillis();

source/de/anomic/kelondro/kelondroRowSet.java

@@ -447,7 +447,8 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
         }
         System.out.println("RESULT SIZE: " + c.size());
         */
-        
+        /*
+        // performance test for put
         long start = System.currentTimeMillis();
         kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0);
         Random random = new Random(0);
@@ -459,6 +460,40 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
         }
         System.out.println("RESULT SIZE: " + c.size());
         System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
+        */
+        
+        // remove test
+        long start = System.currentTimeMillis();
+        kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0);
+        byte[] key;
+        int testsize = 5000;
+        byte[][] delkeys = new byte[testsize / 5][];
+        Random random = new Random(0);
+        for (int i = 0; i < testsize; i++) {
+            key = randomHash(random);
+            if (i % 5 != 0) continue;
+            delkeys[i / 5] = key;
+        }
+        random = new Random(0);
+        for (int i = 0; i < testsize; i++) {
+            key = randomHash(random);
+            c.put(c.rowdef.newEntry(new byte[][]{key, key}));
+            if (i % 1000 == 0) {
+                for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
+                c.shape();
+            }
+        }
+        for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
+        c.shape();
+        random = new Random(0);
+        for (int i = 0; i < testsize; i++) {
+            key = randomHash(random);
+            if (i % 5 == 0) continue;
+            if (c.get(key) == null) System.out.println("missing entry " + new String(key));
+        }
+        c.shape();
+        System.out.println("RESULT SIZE: " + c.size());
+        System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
     }
     
     public static byte[] randomHash(final long r0, final long r1) {

source/de/anomic/plasma/plasmaSearchEvent.java

@@ -70,7 +70,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
     private plasmaCrawlLURL urlStore;
     private plasmaSnippetCache snippetCache;
     private indexContainer rcContainers; // cache for results
-    private int rcContainerCount;
+    private int rcContainerFlushCount;
     private Map rcAbstracts; // cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
     private plasmaSearchTimingProfile profileLocal, profileGlobal;
     private boolean postsort;
@@ -92,7 +92,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
         this.urlStore = urlStore;
         this.snippetCache = snippetCache;
         this.rcContainers = new indexContainer(null);
-        this.rcContainerCount = 0;
+        this.rcContainerFlushCount = 0;
         this.rcAbstracts = new TreeMap();
         this.profileLocal = localTiming;
         this.profileGlobal = remoteTiming;
@@ -434,6 +434,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
         // this must be called after search results had been computed
         // it is wise to call this within a separate thread because
         // this method waits until all threads are finished
+        serverLog.logFine("PLASMA", "STARTED FLUSHING GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
         
         int remaining = 0;
         if (primarySearchThreads == null) return;
@@ -456,7 +457,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
             //log.logFine("FINISHED FLUSH RESULTS PROCESS for query " + query.hashes(","));
         }
         
-        serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
+        serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerFlushCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
             
         // finally delete the temporary index
         rcContainers = null;
@@ -483,7 +484,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
                 rcContainers.clear();
             }
         }
-        rcContainerCount += count;
+        rcContainerFlushCount += count;
     }
     
 }

source/de/anomic/plasma/plasmaWordIndex.java

@@ -363,8 +363,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
             if (container == null) {
                 container = dhtInCache.getContainer(wordHash, urlselection, true, -1);
             } else {
-                indexContainer ic = dhtInCache.getContainer(wordHash, urlselection, true, -1);
+                container.add(dhtInCache.getContainer(wordHash, urlselection, true, -1), -1);
-                if (ic != null) container.add(ic, -1);
             }
         
             // get from collection index

source/de/anomic/server/logging/GuiHandler.java

@@ -201,7 +201,7 @@ public class GuiHandler extends Handler{
                 for (int i = 0; i < lineCount; i++) {
                     int ix = (reversed) ?
                                 Math.abs((start-i)%this.buffer.length) :
-                                (start+i)%this.buffer.length;
+                                (start - lineCount + i) % this.buffer.length;
                     record = this.buffer[ix];
                     logMessages.add(logFormatter.format(record));                
                 }             

source/de/anomic/yacy/yacyClient.java

@@ -364,7 +364,7 @@ public final class yacyClient {
         }
     }
 
-    public static int search(
+    public static String[] search(
             String wordhashes,
             String urlhashes,
             String prefer,
@@ -454,12 +454,12 @@ public final class yacyClient {
             } catch (IOException e) {
                 yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
                 yacyCore.peerActions.peerDeparture(targetPeer);
-                return 0;
+                return null;
             }
 
             if (result.size() == 0) {
                 yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (zero response), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
-                return 0;
+                return null;
             }
             
             // compute all computation times
@@ -497,6 +497,7 @@ public final class yacyClient {
 
             // insert results to containers
             plasmaCrawlLURL.Entry urlEntry;
+            String[] urls = new String[results];
             for (int n = 0; n < results; n++) {
                 // get one single search result
                 urlEntry = urlManager.newEntry((String) result.get("resource" + n), true);
@@ -540,6 +541,8 @@ public final class yacyClient {
                 for (int m = 0; m < words; m++) {
                     container[m].add(new indexEntry[]{entry}, System.currentTimeMillis());
                 }
+                // store url hash for statistics
+                urls[n] = urlEntry.hash();
             }
 
             // insert the containers to the index
@@ -574,11 +577,11 @@ public final class yacyClient {
                 searchtime = totalrequesttime;
             }
             yacyCore.log.logFine("SEARCH " + results + " URLS FROM " + targetPeer.hash + ":" + targetPeer.getName() + ", score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes) + ", duetime=" + duetime + ", searchtime=" + searchtime + ", netdelay=" + (totalrequesttime - searchtime) + ", references=" + result.get("references"));
-            return results;
+            return urls;
         } catch (Exception e) {
             yacyCore.log.logSevere("yacyClient.search error: '" + targetPeer.get(yacySeed.NAME, "anonymous") + "' failed - " + e);
             e.printStackTrace();
-            return 0;
+            return null;
         }
     }
 

source/de/anomic/yacy/yacySearch.java

@@ -70,7 +70,7 @@ public class yacySearch extends Thread {
     final private plasmaURLPattern blacklist;
     final private plasmaSnippetCache snippetCache;
     final private yacySeed targetPeer;
-    private int links;
+    private String[] urls;
     private int maxDistance;
     final private plasmaSearchTimingProfile timingProfile;
     final private plasmaSearchRankingProfile rankingProfile;
@@ -94,18 +94,20 @@ public class yacySearch extends Thread {
         this.blacklist = blacklist;
         this.snippetCache = snippetCache;
         this.targetPeer = targetPeer;
-        this.links = -1;
+        this.urls = null;
         this.maxDistance = maxDistance;
         this.timingProfile = (plasmaSearchTimingProfile) timingProfile.clone();
         this.rankingProfile = rankingProfile;
     }
 
     public void run() {
-        this.links = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile);
+        this.urls = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile);
-        yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes);
+        StringBuffer urllist = new StringBuffer(this.urls.length * 13);
-        if (links != 0) {
+        for (int i = 0; i < this.urls.length; i++) urllist.append(this.urls[i]).append(' ');
-            yacyCore.seedDB.mySeed.incRI(links);
+        yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + urls.length + " links for word hash " + wordhashes + ": " + new String(urllist));
-            yacyCore.seedDB.mySeed.incRU(links);
+        if (urls != null) {
+            yacyCore.seedDB.mySeed.incRI(urls.length);
+            yacyCore.seedDB.mySeed.incRU(urls.length);
         }
     }
 
@@ -117,7 +119,7 @@ public class yacySearch extends Thread {
     }
 
     public int links() {
-        return this.links;
+        return this.urls.length;
     }
     
     public plasmaSearchTimingProfile timingProfile() {
@@ -241,7 +243,7 @@ public class yacySearch extends Thread {
     public static int collectedLinks(yacySearch[] searchThreads) {
         int links = 0;
         for (int i = 0; i < searchThreads.length; i++) {
-            if (!(searchThreads[i].isAlive())) links += searchThreads[i].links;
+            if (!(searchThreads[i].isAlive())) links += searchThreads[i].urls.length;
         }
         return links;
     }