better assortment monitoring and enhanced profile menue

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@416 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Performance_p.html

@@ -22,8 +22,8 @@
 <td class="small">Idle<br>Cycles</td>
 <td class="small">Busy<br>Cycles</td>
 <td class="small">Short Mem<br>Cycles</td>
-<td class="small">Sleep Time<br>per Cycle<br>(milliseconds)</td>
-<td class="small">Exec Time<br>per Busy-Cycle<br>(milliseconds)</td>
+<td class="small">Sleep Time<br>per Cycle<br>(millis)</td>
+<td class="small">Exec Time<br>per Busy-Cycle<br>(millis)</td>
 <td class="small">Delay between<br>idle loops</td>
 <td class="small">Delay between<br>busy loops</td>
 <td class="small">Minimum of<br>Required Memory</td>
@@ -57,17 +57,13 @@
 Changes take effect immediately</td>
 </tr>
 </table>
-</p></form>
+</form>
+</p>
 
 <p>
 <div class=small><b>Indexing Cache Settings:</b></div>
-<table border="0" cellpadding="5" cellspacing="0" width="100%">
 <form action="Performance_p.html" method="post" enctype="multipart/form-data">
-  <tr valign="top" class="TableCellDark">
-    <td width="120"></td>
-    <td></td>
-    <td></td>
-  </tr>
+<table border="0" cellpadding="5" cellspacing="1" width="100%">
   <tr valign="top" class="TableCellDark">
     <td class=small>Words in RAM Cache:</td>
     <td class=small>#[wordCacheRAMSize]#</td>
@@ -85,13 +81,6 @@ Changes take effect immediately</td>
     If this is a big number, it shows that the caching works efficiently.
     </td>
   </tr>
-  #{assortmentCluster}#
-  <tr valign="top" class="TableCellDark">
-    <td class=small>Assortment #[assortmentSlot]#:</td>
-    <td class=small>#[assortmentSize]#</td>
-    <td class=small>Buffer for words appearing exactly on #[assortmentSlot]# different pages.</td>
-  </tr>
-  #{/assortmentCluster}#
   <tr valign="top" class="TableCellDark">
     <td class=small>Maximum number of Word Caches:</td>
     <td class=small><input name="wordCacheMax" type="text" size="20" maxlength="100" value="#[wordCacheMax]#"></td>
@@ -101,27 +90,16 @@ Changes take effect immediately</td>
     flushed to disc; this may last some minutes.
     </td>
   </tr>
-  <!--
-  <tr valign="top" class="TableCellDark">
-    <td class=small>Maximum waitingtime to flush word cache<br>during shut-down (seconds):</td>
-    <td class=small><input name="maxWaitingWordFlush" type="text" size="6" maxlength="6" value="#[maxWaitingWordFlush]#"></td>
-    <td class=small>
-    This gives the number of seconds that the shutdown
-    may last for the final word flush. Words that cannot be flushed within this
-    time are lost.
-    </td>
-  </tr>
-  -->
   <tr valign="top" class="TableCellLight">
-    <td class=small colspan="3"><input type="submit" name="cacheSizeSubmit" value="Enter New Cache Size">
+    <td class=small colspan="3">
+    <input type="submit" name="cacheSizeSubmit" value="Enter New Cache Size">
     Changes take effect immediately</td>
-    </td>
   </tr>
-
-</form>
 </table>
+</form>
 </p>
 
+<p>
 <div class=small><b>Thread pool settings:</b></div>
 <form action="Performance_p.html" method="post" enctype="multipart/form-data">
 <table border="0" cellpadding="2" cellspacing="1" width="100%">
@@ -149,6 +127,26 @@ Changes take effect immediately</td>
 </table>
 </p></form>
 
+<p>
+<div class=small><b>Index Assortments:</b></div>
+<table border="0" cellpadding="5" cellspacing="1" width="100%">
+  #{assortmentCluster}#
+  <tr valign="top" class="TableCellDark">
+    <td class=small>Assortments #[assortmentSlots]#:</td>
+    <td class=small align="right">#[assortmentSizeA]#</td>
+    <td class=small align="right">#[assortmentSizeB]#</td>
+    <td class=small align="right">#[assortmentSizeC]#</td>
+    <td class=small align="right">#[assortmentSizeD]#</td>
+    <td class=small align="right">#[assortmentSizeE]#</td>
+    <td class=small align="right">#[assortmentSizeF]#</td>
+    <td class=small align="right">#[assortmentSizeG]#</td>
+    <td class=small align="right">#[assortmentSizeH]#</td>
+  </tr>
+  #{/assortmentCluster}#
+</table>
+</p>
+
+
 
 #[footer]#
 </body>

htroot/Performance_p.java

@@ -91,8 +91,8 @@ public class Performance_p {
             thread = switchboard.getThread(threadName);
             
             // set values to templates
-            prop.put("table_" + c + "_threadname", threadName);
-            prop.put("table_" + c + "_shortdescr", thread.getShortDescription());
+            //prop.put("table_" + c + "_threadname", threadName);
+            prop.put("table_" + c + "_shortdescr", (thread.getMonitorURL() == null) ? thread.getShortDescription() : "<a href=\"" + thread.getMonitorURL() + "\" class=?\"small\">" + thread.getShortDescription() + "</a>");
             prop.put("table_" + c + "_longdescr", thread.getLongDescription());
             queuesize = thread.getJobCount();
             prop.put("table_" + c + "_queuesize", (queuesize == Integer.MAX_VALUE) ? "unlimited" : ("" + queuesize));
@@ -186,11 +186,18 @@ public class Performance_p {
         prop.put("wordCacheMax", switchboard.getConfig("wordCacheMax", "10000"));
         
         int[] asizes = switchboard.wordIndex.assortmentSizes();
-        for (int i = 0; i < asizes.length; i++) {
-            prop.put("assortmentCluster_" + i + "_assortmentSlot", i + 1);
-            prop.put("assortmentCluster_" + i + "_assortmentSize", asizes[i]);
+        for (int i = 0; i < asizes.length; i += 8) {
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSlots", (i + 1) + "-" + (i + 8));
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeA", asizes[i]);
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeB", asizes[i + 1]);
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeC", asizes[i + 2]);
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeD", asizes[i + 3]);
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeE", asizes[i + 4]);
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeF", asizes[i + 5]);
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeG", asizes[i + 6]);
+            prop.put("assortmentCluster_" + (i/8) + "_assortmentSizeH", asizes[i + 7]);
         }
-        prop.put("assortmentCluster", asizes.length);
+        prop.put("assortmentCluster", asizes.length / 8);
         
         // table thread pool settings
         GenericObjectPool.Config crawlerPoolConfig = switchboard.cacheLoader.getPoolConfig();

source/de/anomic/plasma/plasmaSnippetCache.java

@@ -202,6 +202,7 @@ public class plasmaSnippetCache {
     }
     
     private String computeSnippet(String[] sentences, Set queryhashes, int minLength, int maxLength) {
+        try {
         if ((sentences == null) || (sentences.length == 0)) return null;
         if ((queryhashes == null) || (queryhashes.size() == 0)) return null;
         kelondroMScoreCluster hitTable = new kelondroMScoreCluster();
@@ -297,6 +298,10 @@ public class plasmaSnippetCache {
         if (maxLength < 20) maxLength = 20;
         String nextSnippet = computeSnippet(sentences, remaininghashes, minLength, maxLength);
         return result + ((nextSnippet == null) ? "" : (" / " + nextSnippet));
+        } catch (IndexOutOfBoundsException e) {
+            e.printStackTrace();
+            return "";
+        }
     }
     
     private HashMap hashSentence(String sentence) {

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -351,36 +351,36 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         System.gc(); // help for profiler
         int indexing_cluster = Integer.parseInt(getConfig("80_indexing_cluster", "1"));
         if (indexing_cluster < 1) indexing_cluster = 1;
-        deployThread("90_cleanup", "Cleanup", "simple cleaning process for monitoring information" ,
+        deployThread("90_cleanup", "Cleanup", "simple cleaning process for monitoring information", null,
                      new serverInstantThread(this, "cleanupJob", "cleanupJobSize"), 10000); // all 5 Minutes
-        deployThread("80_indexing", "Parsing/Indexing", "thread that performes document parsing and indexing" ,
+        deployThread("80_indexing", "Parsing/Indexing", "thread that performes document parsing and indexing", "/IndexCreateIndexingQueue_p.html",
                      new serverInstantThread(this, "deQueue", "queueSize"), 10000);
         
         for (int i = 1; i < indexing_cluster; i++) {
             setConfig((i + 80) + "_indexing_idlesleep", getConfig("80_indexing_idlesleep", ""));
             setConfig((i + 80) + "_indexing_busysleep", getConfig("80_indexing_busysleep", ""));
-            deployThread((i + 80) + "_indexing", "Parsing/Indexing (cluster job)", "thread that performes document parsing and indexing" ,
+            deployThread((i + 80) + "_indexing", "Parsing/Indexing (cluster job)", "thread that performes document parsing and indexing", null,
                      new serverInstantThread(this, "deQueue", "queueSize"), 10000 + (i * 1000));
         }
-        deployThread("70_cachemanager", "Proxy Cache Enqueue", "job takes new proxy files from RAM stack, stores them, and hands over to the Indexing Stack",
+        deployThread("70_cachemanager", "Proxy Cache Enqueue", "job takes new proxy files from RAM stack, stores them, and hands over to the Indexing Stack", null,
                      new serverInstantThread(this, "htEntryStoreJob", "htEntrySize"), 10000);
-        deployThread("62_remotetriggeredcrawl", "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer",
+        deployThread("62_remotetriggeredcrawl", "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null,
                      new serverInstantThread(this, "remoteTriggeredCrawlJob", "remoteTriggeredCrawlJobSize"), 30000);
-        deployThread("61_globalcrawltrigger", "Global Crawl Trigger", "thread that triggeres remote peers for crawling",
+        deployThread("61_globalcrawltrigger", "Global Crawl Trigger", "thread that triggeres remote peers for crawling", "/IndexCreateWWWGlobalQueue_p.html",
                    new serverInstantThread(this, "limitCrawlTriggerJob", "limitCrawlTriggerJobSize"), 30000); // error here?
-        deployThread("50_localcrawl", "Local Crawl", "thread that performes a single crawl step from the local crawl queue",
+        deployThread("50_localcrawl", "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html",
                      new serverInstantThread(this, "coreCrawlJob", "coreCrawlJobSize"), 10000);
-        deployThread("40_peerseedcycle", "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account",
+        deployThread("40_peerseedcycle", "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null,
                      new serverInstantThread(yc, "publishSeedList", null), 180000);
         serverInstantThread peerPing = null;
-        deployThread("30_peerping", "YaCy Core", "this is the p2p-control and peer-ping task",
+        deployThread("30_peerping", "YaCy Core", "this is the p2p-control and peer-ping task", null,
                      peerPing = new serverInstantThread(yc, "peerPing", null), 2000);
         peerPing.setSyncObject(new Object());
         
         indexDistribution = new plasmaWordIndexDistribution(urlPool, wordIndex, log,
                                                             getConfig("allowDistributeIndex", "false").equals("true"));
         indexDistribution.setCounts(100, 1, 8000);
-        deployThread("20_dhtdistribution", "DHT Distribution (currently by juniors only)", "selection, transfer and deletion of index entries that are not searched on your peer, but on others",
+        deployThread("20_dhtdistribution", "DHT Distribution (currently by juniors only)", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null,
                      new serverInstantThread(indexDistribution, "job", null), 120000);
             
         // init migratiion from 0.37 -> 0.38
@@ -389,7 +389,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         if (classicCache.size() > 0) {
             setConfig("99_indexcachemigration_idlesleep" , 10000);
             setConfig("99_indexcachemigration_busysleep" , 40);
-            deployThread("99_indexcachemigration", "index cache migration", "migration of index cache data structures 0.37 -> 0.38",
+            deployThread("99_indexcachemigration", "index cache migration", "migration of index cache data structures 0.37 -> 0.38", null,
             new serverInstantThread(classicCache, "oneStepMigration", "size"), 30000);
         }
         

source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java

@@ -118,13 +118,29 @@ public final class plasmaWordIndexAssortmentCluster {
     private void storeStretched(String wordHash, plasmaWordIndexEntryContainer newContainer) {
 	// this stores the record and stretches the storage over
         // all the assortments that are necessary to fit in the record
+        // IMPORTANT: it must be ensured that the wordHash does not exist in the cluster before
+        // i.e. by calling removeFromAll
 	if (newContainer.size() <= clusterCount) {
             storeForced(wordHash, newContainer);
             return;
         }
+        
+        // calculate appropriate cluster insert point
+        int clusterStart = clusterCount;
+        if ((((byte) wordHash.charAt(0)) & 1) == 1) {
+            // for every second hash, place the entries in the middle of the assortments
+            // this balances the entries within the assortments-cluster
+            int cap = clusterCapacity - newContainer.size() - 2 * clusterCount;
+            while (cap > 0) {
+                cap -= clusterStart;
+                clusterStart--;
+            }
+        }
+        
+        // do the insert
         plasmaWordIndexEntryContainer c;
         Iterator i = newContainer.entries();
-        for (int j = clusterCount; j >= 1; j--) {
+        for (int j = clusterStart; j >= 1; j--) {
             c = new plasmaWordIndexEntryContainer(wordHash);
             for (int k = 0; k < j; k++) {
                 if (i.hasNext()) {

source/de/anomic/plasma/plasmaWordIndexCache.java

@@ -67,7 +67,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
     private static final String oldSingletonFileName = "indexSingletons0.db";
     private static final String newSingletonFileName = "indexAssortment001.db";
     private static final String indexAssortmentClusterPath = "ACLUSTER";
-    private static final int assortmentCount = 50;
+    private static final int assortmentCount = 64;
     private static final int ramCacheLimit = 200;
     
     
@@ -242,63 +242,6 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
         }
         return urlCount;
     }
-
-    /*
-    private void dump(int waitingSeconds) throws IOException {
-        log.logSystem("creating dump for index cache, " + cache.size() + " words (and much more urls)");
-        File indexDumpFile = new File(databaseRoot, indexStackFileName);
-        if (indexDumpFile.exists()) indexDumpFile.delete();
-        kelondroStack dumpStack = new kelondroStack(indexDumpFile, 1024, plasmaWordIndexAssortment.bufferStructureBasis);
-        long startTime = System.currentTimeMillis();
-        long messageTime = System.currentTimeMillis() + 5000;
-        long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
-        synchronized (cache) {
-            Iterator i = cache.entrySet().iterator();
-            Map.Entry entry;
-            String wordHash;
-            plasmaWordIndexEntryContainer container;
-            long updateTime;
-            plasmaWordIndexEntry wordEntry;
-            byte[][] row = new byte[5][];
-            System.gc(); // this can speed up the assortment, because they may better use the cache
-            while (i.hasNext()) {
-                // get entries
-                entry = (Map.Entry) i.next();
-                wordHash = (String) entry.getKey();
-                updateTime = getUpdateTime(wordHash);
-                container = (plasmaWordIndexEntryContainer) entry.getValue();
-
-                // put entries on stack
-                if (container != null) {
-                    Iterator ci = container.entries();
-                    while (ci.hasNext()) {
-                        wordEntry = (plasmaWordIndexEntry) ci.next();
-                        row[0] = wordHash.getBytes();
-                        row[1] = kelondroRecords.long2bytes(container.size(), 4);
-                        row[2] = kelondroRecords.long2bytes(updateTime, 8);
-                        row[3] = wordEntry.getUrlHash().getBytes();
-                        row[4] = wordEntry.toEncodedForm(true).getBytes();
-                        dumpStack.push(row);
-                        urlcount++;
-                    }
-                }
-                wordcount++;
-                i.remove(); // free some mem
-                
-                // write a log
-                if (System.currentTimeMillis() > messageTime) {
-                    System.gc(); // for better statistic
-                    wordsPerSecond = wordcount * 1000 / (1 + System.currentTimeMillis() - startTime);
-                    log.logInfo("dumping status: " + wordcount + " words done, " + (cache.size() / (wordsPerSecond + 1)) + " seconds remaining, free mem = " + (Runtime.getRuntime().freeMemory() / 1024 / 1024) + "MB");
-                    messageTime = System.currentTimeMillis() + 5000;
-                }
-            }
-        }
-	dumpStack.close();
-        log.logSystem("dumped " + urlcount + " word/url relations in " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds");
-    }
-        
-    */
     
     private long restoreStack() throws IOException {
         File indexDumpFile = new File(databaseRoot, indexStackFileName);

source/de/anomic/server/serverAbstractSwitch.java

@@ -257,9 +257,10 @@ public abstract class serverAbstractSwitch implements serverSwitch {
             String threadName,
             String threadShortDescription,
             String threadLongDescription,
+            String threadMonitorURL,
             serverThread newThread,
             long startupDelay) {
-        deployThread(threadName, threadShortDescription, threadLongDescription,
+        deployThread(threadName, threadShortDescription, threadLongDescription, threadMonitorURL,
                      newThread, startupDelay,
                      Long.parseLong(getConfig(threadName + "_idlesleep" ,     "100")), 
                      Long.parseLong(getConfig(threadName + "_busysleep" ,    "1000")),
@@ -270,6 +271,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
             String threadName,
             String threadShortDescription,
             String threadLongDescription,
+            String threadMonitorURL,
             serverThread newThread,
             long startupDelay,
             long initialIdleSleep,
@@ -300,7 +302,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
             setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
         }
         newThread.setLog(log);
-        newThread.setDescription(threadShortDescription, threadLongDescription);
+        newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL);
         workerThreads.put(threadName, newThread);
         // start the thread
         if (workerThreads.containsKey(threadName)) newThread.start();

source/de/anomic/server/serverAbstractThread.java

@@ -59,6 +59,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
     private serverLog log = null;
     private long idletime = 0, busytime = 0, memprereq = 0;
     private String shortDescr = "", longDescr = "";
+    private String monitorURL = null;
     private long threadBlockTimestamp = System.currentTimeMillis();
     private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
     private Object syncObject = null;
@@ -87,10 +88,11 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
         this.idletime += millis;
     }
         
-    public final void setDescription(String shortText, String longText) {
+    public final void setDescription(String shortText, String longText, String monitorURL) {
         // sets a visible description string
         this.shortDescr = shortText;
         this.longDescr  = longText;
+        this.monitorURL = monitorURL;
     }
     
     public final void setStartupSleep(long milliseconds) {
@@ -121,6 +123,10 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
         return this.longDescr;
     }
     
+    public String getMonitorURL() {
+        return this.monitorURL;
+    }
+    
     public final long getIdleCycles() {
         // returns the total number of cycles of job execution with idle-result
         return this.idleCycles;

source/de/anomic/server/serverPortForwardingSch.java

@@ -138,7 +138,7 @@ public class serverPortForwardingSch implements serverPortForwarding{
             // using a timer task to control if the session remains open
             if (sessionWatcher == null) {
                 this.log.logDebug("Deploying port forwarding session watcher thread.");
-                this.switchboard.deployThread("portForwardingWatcher", "Remote Port Forwarding Watcher", "this thread is used to detect broken connections and to re-establish it if necessary.",
+                this.switchboard.deployThread("portForwardingWatcher", "Remote Port Forwarding Watcher", "this thread is used to detect broken connections and to re-establish it if necessary.", null,
                         sessionWatcher = new serverInstantThread(this, "reconnect", null), 30000,30000,30000,1000);
                 sessionWatcher.setSyncObject(new Object());
             }

source/de/anomic/server/serverSwitch.java

@@ -71,6 +71,7 @@ public interface serverSwitch {
     public void deployThread(String threadName,
 			     String threadShortDescription,
 			     String threadLongDescription,
+                             String threadMonitorURL,
                              serverThread newThread,
                              long startupDelay,
                              long initialIdleSleep, long initialBusySleep,

source/de/anomic/server/serverThread.java

@@ -53,7 +53,7 @@ public interface serverThread {
     // these method are implemented by serverThread and do not need to be altered
     // this includes also the run()-Method
     
-    public void setDescription(String shortText, String longText);
+    public void setDescription(String shortText, String longText, String monitorURL);
     // sets a visible description string
     
     public void setStartupSleep(long milliseconds);
@@ -74,6 +74,9 @@ public interface serverThread {
     public String getLongDescription();
     // returns long description string for online display
     
+    public String getMonitorURL();
+    // returns an URL that can be used to monitor the thread and it's queue
+    
     public long getIdleCycles();
     // returns the total number of cycles of job execution with idle-result
     

source/yacy.java

@@ -300,7 +300,7 @@ public final class yacy {
                     serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use.");
                 } else {
                     // first start the server
-                    sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, 0, 0, 0, 0);
+                    sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
                     //server.start();
                     
                     // open the browser window