From 9c681cc00d8fba4c5a80f2bbb36bb4dc6ba73248 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 23 Jul 2013 19:10:11 +0200 Subject: [PATCH] added segment sizes, postprocessing status and cpu load to crawler monitor --- htroot/Crawler_p.html | 43 +++++++++++++++++++++++++++---------------- htroot/Crawler_p.java | 3 +++ htroot/js/Crawler.js | 24 ++++++++++++++++++------ 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/htroot/Crawler_p.html b/htroot/Crawler_p.html index 5394a44b6..4c40af46d 100644 --- a/htroot/Crawler_p.html +++ b/htroot/Crawler_p.html @@ -21,14 +21,14 @@ #%env/templates/submenuCrawlMonitor.template%#

Crawler

-
+
Queues - - - + + + @@ -75,41 +75,45 @@
QueueSizePause/ResumeQueue
 		Size
 		Pause/
Resume
Local Crawler
 #[queuemessage]#
-
+
Index Size - - + + + - + + - + + - + +
DatabaseEntriesDatabase
 		Entries
 		Seg-
ments
Documents
solr search api		#[urlpublictextSize]##[urlpublictextSize]##[urlpublictextSegmentCount]#
Webgraph Edges
solr search api		#[webgraphSize]##[webgraphSize]##[webgraphSegmentCount]#
RWIs
(P2P Chunks)		#[rwipublictextSize]##[rwipublictextSize]##[rwipublictextSegmentCount]#
-
+
Progress
- - + + - - - + + + + + + + + diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java index 5253d9e8f..1cd75ef15 100644 --- a/htroot/Crawler_p.java +++ b/htroot/Crawler_p.java @@ -73,8 +73,11 @@ public class Crawler_p { final serverObjects prop = new serverObjects(); prop.put("rejected", 0); prop.put("urlpublictextSize", 0); + prop.put("urlpublictextSegmentCount", 0); prop.put("webgraphSize", 0); + prop.put("webgraphSegmentCount", 0); prop.put("rwipublictextSize", 0); + prop.put("rwipublictextSegmentCount", 0); prop.put("list", "0"); prop.put("loaderSize", 0); prop.put("loaderMax", 0); diff --git a/htroot/js/Crawler.js b/htroot/js/Crawler.js index cb9f4d279..c7b085594 100644 --- a/htroot/js/Crawler.js +++ b/htroot/js/Crawler.js @@ -89,12 +89,24 @@ function handleStatus(){ trafCrawlerSpan.appendChild(document.createTextNode(Math.round((trafficCrawlerValue) / 1024 / 10.24) / 100)); dbsize=getFirstChild(statusTag, "dbsize"); - urlpublictextSize=getValue(getFirstChild(dbsize, "urlpublictext")); - rwipublictextSize=getValue(getFirstChild(dbsize, "rwipublictext")); - webgraphSize=getValue(getFirstChild(dbsize, "webgraph")); - document.getElementById("urldbsize").firstChild.nodeValue=urlpublictextSize; - document.getElementById("webgraphsize").firstChild.nodeValue=webgraphSize; - document.getElementById("rwidbsize").firstChild.nodeValue=rwipublictextSize; + urlpublictext=getValue(getFirstChild(dbsize, "urlpublictext")); + urlpublictextSegmentCount=getValue(getFirstChild(dbsize, "urlpublictextSegmentCount")); + webgraph=getValue(getFirstChild(dbsize, "webgraph")); + webgraphSegmentCount=getValue(getFirstChild(dbsize, "webgraphSegmentCount")); + rwipublictext=getValue(getFirstChild(dbsize, "rwipublictext")); + rwipublictextSegmentCount=getValue(getFirstChild(dbsize, "rwipublictextSegmentCount")); + document.getElementById("urlpublictextSize").firstChild.nodeValue=urlpublictext; + document.getElementById("urlpublictextSegmentCount").firstChild.nodeValue=urlpublictextSegmentCount; + document.getElementById("webgraphSize").firstChild.nodeValue=webgraph; + document.getElementById("webgraphSegmentCount").firstChild.nodeValue=webgraphSegmentCount; + document.getElementById("rwipublictextSize").firstChild.nodeValue=rwipublictext; + document.getElementById("rwipublictextSegmentCount").firstChild.nodeValue=rwipublictextSegmentCount; + + postprocessing=getFirstChild(statusTag, "postprocessing"); + document.getElementById("postprocessing").firstChild.nodeValue=getValue(getFirstChild(postprocessing, "status")); + + load=getFirstChild(statusTag, "load"); + document.getElementById("load").firstChild.nodeValue=getValue(load); loaderqueue=getFirstChild(statusTag, "loaderqueue"); loaderqueue_size=getValue(getFirstChild(loaderqueue, "size"));
IndicatorLevelIndicator
 		Level
 
Speed + PPM @@ -117,14 +121,21 @@
PPM (Pages Per Minute)    +    
Traffic (Crawler)     MB 
Load   Postprocessing