design update of crawler monitor

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3094 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/WatchCrawler_p.html

@@ -12,48 +12,68 @@
     #%env/templates/header.template%#
     <h2>Crawler Monitor</h2>
     <p>
-      Next update in <span id="nextUpdate" onclick="changeInterval()"></span> seconds.<br />
+      Next update in <span id="nextUpdate" onclick="changeInterval()"></span> seconds.
-      PPM: <span id="ppm">&nbsp;&nbsp;&nbsp;</span><br />
-      Wordcaches: <span id="wordcache">&nbsp;</span>
     </p>
-    <p>
+    
-      Indexing Queue: size=<span id="indexingqueuesize">&nbsp;&nbsp;&nbsp;</span>; max=<span id="indexingqueuemax">&nbsp;&nbsp;&nbsp;</span>
+    <p><table border="0" cellpadding="0" cellspacing="0"><tr>
-    </p>
+
-    <table border="0" cellpadding="2" cellspacing="1" id="indexingTable">
+    <td valign="top"><table border="0" cellpadding="2" cellspacing="1">
       <tbody>
         <tr class="TableHeader">
-          <th>Initiator</th>
+          <th>Queue</th>
-          <th>Depth</th>
-          <th>Modified Date</th>
-          <th>Anchor Name</th>
-          <th>URL</th>
           <th>Size</th>
-          <th>Delete</th>
+          <th>Max</th>
+        </tr>
+        <tr class="TableCellLight">
+          <td align="left">Indexing</td>
+          <td align="right"><span id="indexingqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
+          <td align="right"><span id="indexingqueuemax">&nbsp;&nbsp;&nbsp;</span></td>
+        </tr>
+        <tr class="TableCellLight">
+          <td align="left">Loader</td>
+          <td align="right"><span id="loaderqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
+          <td align="right"><span id="loaderqueuemax">&nbsp;&nbsp;&nbsp;</span></td>
+        </tr>
+        <tr class="TableCellLight">
+          <td align="left">Local Crawler</td>
+          <td align="right"><span id="localcrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
+          <td align="right">unlimited</td>
+        </tr>
+        <tr class="TableCellLight">
+          <td align="left">Remote Crawler</td>
+          <td align="right"><span id="remotecrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
+          <td align="right">unlimited</td>
         </tr>
       </tbody>
-    </table>
+    </table></td>
-    <p>
+    
-      Loader: size=<span id="loaderqueuesize">&nbsp;&nbsp;&nbsp;</span>; max=<span id="loaderqueuemax">&nbsp;&nbsp;&nbsp;</span>
+    <td width="10">&nbsp;</td>
-    </p>
+    
-    <table border="0" cellpadding="2" cellspacing="1" id="loaderTable">
+    <td valign="top"	><table border="0" cellpadding="2" cellspacing="1">
       <tbody>
         <tr class="TableHeader">
-          <th>Initiator</th>
+          <th>Indicator</th>
-          <th>Depth</th>
+          <th colspan="2">Level</th>
-          <th>Modified Date</th>
+        </tr>
-          <th>Anchor Name</th>
+        <tr class="TableCellLight">
-          <th>URL</th>
+          <td align="left">PPM</td>
-          <th>Size</th>
+          <td align="left"><span id="ppmNum">&nbsp;&nbsp;&nbsp;</span></td>
-          <th>Delete</th>
+          <td align="left"><span id="ppmSpan">&nbsp;&nbsp;&nbsp;</span></td>
+        </tr>
+        <tr class="TableCellLight">
+          <td align="left">Word Caches</td>
+          <td align="left"><span id="wordcacheNum">&nbsp;</span></td>
+          <td align="left"><span id="wordcacheSpan">&nbsp;</span></td>
         </tr>
       </tbody>
-    </table>
+    </table></td>
-    <p>
+    
-      Local Crawler: size=<span id="localcrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span>
+    </tr></table></p>
-    </p>
+    
-    <table border="0" cellpadding="2" cellspacing="1" id="localCrawlerTable">
+    <p><table border="0" cellpadding="2" cellspacing="1" id="queueTable">
       <tbody>
         <tr class="TableHeader">
+          <th>Queue</th>
           <th>Initiator</th>
           <th>Depth</th>
           <th>Modified Date</th>
@@ -63,25 +83,8 @@
           <th>Delete</th>
         </tr>
       </tbody>
-    </table>
+    </table></p>
-
+    
-    <p>
-      Remote Crawler: size=<span id="remotecrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span>
-    </p>
-    <table border="0" cellpadding="2" cellspacing="1" id="remoteCrawlerTable">
-      <tbody>
-        <tr class="TableHeader">
-          <th width="30">Initiator</th>
-          <th width="10">Depth</th>
-          <th width="60">Modified Date</th>
-          <th width="100">Anchor Name</th>
-          <th width="300">URL</th>
-          <th width="20">Size</th>
-          <th width="30">Delete</th>
-        </tr>
-      </tbody>
-    </table>
-
     #%env/templates/footer.template%#
   </body>
 </html>

htroot/js/WatchCrawler.js

@@ -64,10 +64,13 @@ function handleStatus(){
 	statusTag=getFirstChild(statusResponse, "status")
 	
 	ppm=getValue(getFirstChild(statusTag, "ppm"))
-	var ppmSpan = document.getElementById("ppm");
+	
+	var ppmNum = document.getElementById("ppmNum");
+	removeAllChildren(ppmNum);
+	ppmNum.appendChild(document.createTextNode(ppm));
+	
+	var ppmSpan = document.getElementById("ppmSpan");
 	removeAllChildren(ppmSpan);
-	ppmSpan.appendChild(document.createTextNode(ppm));
-	ppmSpan.appendChild(document.createElement("br"));
 	for(i=0;i<ppm;i++){
 		img=document.createElement("img");
 		img.setAttribute("src", BAR_IMG1);
@@ -77,15 +80,14 @@ function handleStatus(){
 	var wordCache=getValue(getFirstChild(statusTag, "wordCacheWSize"));
 	var wordCacheMax=getValue(getFirstChild(statusTag, "wordCacheMaxCount"));
 
-	var percent=Math.round(wordCache/wordCacheMax*100);
+	wordCacheNum=document.getElementById("wordcacheNum");
-
+	removeAllChildren(wordCacheNum);
+	wordCacheNum.appendChild(document.createTextNode(wordCache+"/"+wordCacheMax));
 	
-	wordCacheSpan=document.getElementById("wordcache");
+	wordCacheSpan=document.getElementById("wordcacheSpan");
-
 	removeAllChildren(wordCacheSpan);
-	wordCacheSpan.appendChild(document.createTextNode(wordCache+"/"+wordCacheMax));
-	wordCacheSpan.appendChild(document.createElement("br"));
 	var img;
+	var percent=Math.round(wordCache/wordCacheMax*100);
 	for(i=0;i<percent*WORDCACHEBAR_LENGTH;i++){
 		img=document.createElement("img");
 		img.setAttribute("src", BAR_IMG2);
@@ -105,15 +107,19 @@ function handleQueues(){
 	var queuesResponse = queuesRPC.responseXML;
 	xml=getFirstChild(queuesResponse);
 	if(queuesResponse != null){
+		clearTable(document.getElementById("queueTable"), 1);
+	
 		indexingqueue=getFirstChild(xml, "indexingqueue");
-		createIndexingTable(indexingqueue, "indexingTable");
+		updateTable(indexingqueue, "indexing");
+		
 		indexingqueue_size=getValue(getFirstChild(indexingqueue, "size"));
 		indexingqueue_max=getValue(getFirstChild(indexingqueue, "max"));
 		document.getElementById("indexingqueuesize").firstChild.nodeValue=indexingqueue_size;
 		document.getElementById("indexingqueuemax").firstChild.nodeValue=indexingqueue_max;
 		
 		loaderqueue=getFirstChild(xml, "loaderqueue");
-		createIndexingTable(loaderqueue, "loaderTable");
+		updateTable(loaderqueue, "loader");
+		
 		loaderqueue_size=getValue(getFirstChild(loaderqueue, "size"));
 		loaderqueue_max=getValue(getFirstChild(loaderqueue, "max"));
 		document.getElementById("loaderqueuesize").firstChild.nodeValue=loaderqueue_size;
@@ -122,20 +128,20 @@ function handleQueues(){
 		localcrawlerqueue=getFirstChild(xml, "localcrawlerqueue");
 		localcrawlerqueue_size=getValue(getFirstChild(localcrawlerqueue, "size"));
 		document.getElementById("localcrawlerqueuesize").firstChild.nodeValue=localcrawlerqueue_size;
-		createIndexingTable(localcrawlerqueue, "localCrawlerTable");
+		updateTable(localcrawlerqueue, "local crawler");
 		
 		remotecrawlerqueue=getFirstChild(xml, "remotecrawlerqueue");
-		createIndexingTable(remotecrawlerqueue, "remoteCrawlerTable");
+		updateTable(remotecrawlerqueue, "remoteCrawlerTable");
+		
 		remotecrawlerqueue_size=getValue(getFirstChild(remotecrawlerqueue, "size"));
 		document.getElementById("remotecrawlerqueuesize").firstChild.nodeValue=remotecrawlerqueue_size;
-		createRemoteCrawlerTable(remotecrawlerqueue);
+		updateTable(remotecrawlerqueue, "remote crawler");
 	}
 }
 
-function createIndexingTable(indexingqueue, tablename){
+function updateTable(indexingqueue, tablename){
-	indexingTable=document.getElementById(tablename);
+	indexingTable=document.getElementById("queueTable");
 	entries=indexingqueue.getElementsByTagName("entry");
-    clearTable(indexingTable, 1);
         
     dark=false;
     for(i=0;i<entries.length;i++){
@@ -154,7 +160,7 @@ function createIndexingTable(indexingqueue, tablename){
 			deletebutton=createLinkCol("IndexCreateIndexingQueue_p.html?deleteEntry="+hash, DELETE_STRING);
 		else
 			deletebutton=createCol("");
-		row=createIndexingRow(initiator, depth, modified, anchor, url, size, deletebutton);
+		row=createIndexingRow(tablename, initiator, depth, modified, anchor, url, size, deletebutton);
 		
 		//create row
 		if(inProcess){
@@ -167,16 +173,12 @@ function createIndexingTable(indexingqueue, tablename){
         getFirstChild(indexingTable, "tbody").appendChild(row);
         dark=!dark;
     }
-    while(i++ <= 10) {
-    		row=createIndexingRow("", "", "", "", "", "", createCol(""));
-		if(dark) {row.setAttribute("class", "TableCellDark");} else {row.setAttribute("class", "TableCellLight");}
-        getFirstChild(indexingTable, "tbody").appendChild(row);
-        dark=!dark;
-    }
 }
-function createIndexingRow(initiator, depth, modified, anchor, url, size, deletebutton){
+
+function createIndexingRow(queue, initiator, depth, modified, anchor, url, size, deletebutton){
     row=document.createElement("tr");
     row.setAttribute("height", 10);
+    row.appendChild(createCol(queue));
 	row.appendChild(createCol(initiator));
 	row.appendChild(createCol(depth));
 	row.appendChild(createCol(modified));

htroot/xml/queues_p.java

@@ -105,13 +105,9 @@ public class queues_p {
                 if ((switchboard.sbQueue.size() > 0)) {
                     entryList.addAll(switchboard.sbQueue.list(0));
                 }
-                int size=10;
+                int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size());
-                if (post!= null) size = post.getInt("num", 10);
+                if (size > entryList.size()) size = entryList.size();
-                
+
-                if(size>entryList.size()){
-                    size=entryList.size();
-                }
-                
                 for (i = 0; i < size; i++) {
                     boolean inProcess = i < inProcessCount;
                     pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
@@ -140,13 +136,9 @@ public class queues_p {
             prop.put("list-loader", 0);
         } else {
             ThreadGroup loaderThreads = switchboard.cacheLoader.threadStatus();            
-            int threadCount  = loaderThreads.activeCount();
+            Thread[] threadList = new Thread[loaderThreads.activeCount()*2];
-            Thread[] threadList = new Thread[threadCount*2];
+            int size = loaderThreads.enumerate(threadList);
-            threadCount = loaderThreads.enumerate(threadList);
+            
-            int size=10;
-            if(threadCount<size){
-                size=threadCount;
-            }
             int i, count = 0;
             for (i = 0; i < size; i++)  {
                 CrawlWorker theWorker = (CrawlWorker)threadList[i];

source/de/anomic/plasma/plasmaCrawlBalancer.java

@@ -65,7 +65,7 @@ public class plasmaCrawlBalancer {
     }
 
     public void close() {
-        try { flushAll(); } catch (IOException e) {}
+        try { flushSome(domainStacks.size()); } catch (IOException e) {}
         try { stack.close(); } catch (IOException e) {}
         stack = null;
     }
@@ -111,8 +111,8 @@ public class plasmaCrawlBalancer {
         }
     }
     
-    private void flushAll() throws IOException {
+    private void flushSome(int count) throws IOException {
-        while (domainStacks.size() > 0) flushOnce();
+        while ((domainStacks.size() > 0) && (count-- > 0)) flushOnce();
     }
     
     public void add(String domain, byte[] hash) throws IOException {
@@ -150,7 +150,7 @@ public class plasmaCrawlBalancer {
     }
     
     public byte[] top(int dist) throws IOException {
-        flushAll();
+        flushSome(1 + dist - stack.size()); // flush only that much as we need to display
         synchronized (domainStacks) {
             return stack.top(dist).getColBytes(0);
         }

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -168,7 +168,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
     
     // load slots
     public static int crawlSlots            = 10;
-    public static int indexingSlots         = 100;
+    public static int indexingSlots         = 30;
     public static int stackCrawlSlots       = 1000000;
     
     public static int maxCRLDump            = 500000;
@@ -472,7 +472,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         this.sbQueue = new plasmaSwitchboardQueue(this.cacheManager, this.wordIndex.loadedURL, new File(this.plasmaPath, "switchboardQueue1.stack"), this.profiles);
         
         // setting the indexing queue slots
-        indexingSlots = (int) getConfigLong("indexer.slots", 100);
+        indexingSlots = (int) getConfigLong("indexer.slots", 30);
         
         // create in process list
         this.indexingTasksInProcess = new HashMap();

source/de/anomic/yacy/yacyClient.java

@@ -458,7 +458,7 @@ public final class yacyClient {
                     ), "UTF-8"
                 );
             } catch (IOException e) {
-                yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
+                yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes.substring(0, 12)));
                 yacyCore.peerActions.peerDeparture(targetPeer);
                 return null;
             }

yacy.init

@@ -662,7 +662,7 @@ stacker.MaxIdleThreads = 10
 stacker.MinIdleThreads = 5
 
 # maximum size of indexing queue
-indexer.slots = 100
+indexer.slots = 30
 
 # specifies if yacy should set it's own referer if no referer URL
 # was set by the client.