From bf7a69197d4e6acaa92ae3b1aacc927e8a9b2a51 Mon Sep 17 00:00:00 2001 From: karlchenofhell Date: Thu, 22 Feb 2007 22:26:11 +0000 Subject: [PATCH] - fix for possible NPE in queues_p - WatchCrawler_p: - display crawler traffic - pause/resume local- and global crawler git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3389 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/WatchCrawler_p.html | 18 ++++++++++++++++++ htroot/WatchCrawler_p.java | 21 ++++++++++++++++++++- htroot/js/WatchCrawler.js | 28 ++++++++++++++++++++++++++++ htroot/xml/queues_p.java | 16 ++++++++++------ htroot/xml/queues_p.xml | 2 ++ 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/htroot/WatchCrawler_p.html b/htroot/WatchCrawler_p.html index 3b3b7aaaf..78dcbd071 100644 --- a/htroot/WatchCrawler_p.html +++ b/htroot/WatchCrawler_p.html @@ -16,26 +16,39 @@ Queue Size +   Max Indexing     +       Loader     +       Local Crawler     + + + + + unlimited Remote Crawler     + + + + + unlimited @@ -69,6 +82,11 @@         + + Traffic (Crawler) +     MB +   + RWI RAM (Word Cache)   diff --git a/htroot/WatchCrawler_p.java b/htroot/WatchCrawler_p.java index 1df8c75bd..94023d55e 100644 --- a/htroot/WatchCrawler_p.java +++ b/htroot/WatchCrawler_p.java @@ -73,6 +73,26 @@ public class WatchCrawler_p { if (handle != null) switchboard.profiles.removeEntry(handle); } + if (post.containsKey("continue")) { + // continue queue + String queue = post.get("continue", ""); + if (queue.equals("localcrawler")) { + switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + } else if (queue.equals("remotecrawler")) { + switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER); + } + } + + if (post.containsKey("pause")) { + // pause queue + String queue = post.get("pause", ""); + if (queue.equals("localcrawler")) { + switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + } else if (queue.equals("remotecrawler")) { + switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER); + } + } + if (post.containsKey("crawlingstart")) { // init crawl if (yacyCore.seedDB == null) { @@ -301,7 +321,6 @@ public class WatchCrawler_p { } } } - } // crawl profiles diff --git a/htroot/js/WatchCrawler.js b/htroot/js/WatchCrawler.js index 1853cb3c0..e094b41e0 100644 --- a/htroot/js/WatchCrawler.js +++ b/htroot/js/WatchCrawler.js @@ -77,6 +77,13 @@ function handleStatus(){ ppmSpan.appendChild(img); } + // traffic output (no bar up to now) + traffic = getFirstChild(statusTag, "traffic"); + trafficCrawler = getValue(getFirstChild(traffic, "crawler")); + trafCrawlerSpan = document.getElementById("trafficCrawler"); + removeAllChildren(trafCrawlerSpan); + trafCrawlerSpan.appendChild(document.createTextNode(Math.round((trafficCrawler) / 1024 / 10.24) / 100)); + var wordCache=getValue(getFirstChild(statusTag, "wordCacheWSize")); var wordCacheMax=getValue(getFirstChild(statusTag, "wordCacheMaxCount")); @@ -133,18 +140,39 @@ function handleQueues(){ localcrawlerqueue=getFirstChild(xml, "localcrawlerqueue"); localcrawlerqueue_size=getValue(getFirstChild(localcrawlerqueue, "size")); + localcrawlerqueue_state=getValue(getFirstChild(localcrawlerqueue, "state")); document.getElementById("localcrawlerqueuesize").firstChild.nodeValue=localcrawlerqueue_size; + putQueueState("localcrawler", localcrawlerqueue_state); + updateTable(localcrawlerqueue, "local crawler"); remotecrawlerqueue=getFirstChild(xml, "remotecrawlerqueue"); updateTable(remotecrawlerqueue, "remoteCrawlerTable"); remotecrawlerqueue_size=getValue(getFirstChild(remotecrawlerqueue, "size")); + remotecrawlerqueue_state=getValue(getFirstChild(remotecrawlerqueue, "state")); document.getElementById("remotecrawlerqueuesize").firstChild.nodeValue=remotecrawlerqueue_size; + putQueueState("remotecrawler", remotecrawlerqueue_state); updateTable(remotecrawlerqueue, "remote crawler"); } } +function putQueueState(queue, state) { + a = document.getElementById(queue + "stateA"); + img = document.getElementById(queue + "stateIMG"); + if (state == "paused") { + a.href = "WatchCrawler_p.html?continue=" + queue; + a.title = "Continue this queue"; + img.src = "/env/grafics/start.gif"; + img.alt = "Continue this queue"; + } else { + a.href = "WatchCrawler_p.html?pause=" + queue; + a.title = "Pause this queue"; + img.src = "/env/grafics/stop.gif"; + img.alt = "Pause this queue"; + } +} + function updateTable(indexingqueue, tablename){ indexingTable=document.getElementById("queueTable"); entries=indexingqueue.getElementsByTagName("entry"); diff --git a/htroot/xml/queues_p.java b/htroot/xml/queues_p.java index 5f88df466..8800ca63f 100644 --- a/htroot/xml/queues_p.java +++ b/htroot/xml/queues_p.java @@ -66,6 +66,9 @@ import de.anomic.yacy.yacySeed; public class queues_p { + public static final String STATE_RUNNING = "running"; + public static final String STATE_PAUSED = "paused"; + private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US); private static String daydate(Date date) { if (date == null) return ""; @@ -83,7 +86,7 @@ public class queues_p { yacySeed initiator; //indexing queue - prop.put("indexingSize", switchboard.getThread("80_indexing").getJobCount()+switchboard.indexingTasksInProcess.size()); + prop.put("indexingSize", switchboard.getThread(plasmaSwitchboard.INDEXER).getJobCount()+switchboard.indexingTasksInProcess.size()); prop.put("indexingMax", plasmaSwitchboard.indexingSlots); prop.put("urlpublictextSize", switchboard.wordIndex.loadedURL.size()); prop.put("rwipublictextSize", switchboard.wordIndex.size()); @@ -114,9 +117,9 @@ public class queues_p { for (i = 0; i < size; i++) { boolean inProcess = i < inProcessCount; pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i); - long entrySize = pcentry.size(); - totalSize += entrySize; if ((pcentry != null)&&(pcentry.url() != null)) { + long entrySize = pcentry.size(); + totalSize += entrySize; initiator = yacyCore.seedDB.getConnected(pcentry.initiator()); prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted"); prop.putSafeXML("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : wikiCode.replaceHTML(initiator.getName()))); @@ -160,13 +163,14 @@ public class queues_p { } //local crawl queue - prop.put("localCrawlSize", Integer.toString(switchboard.getThread("50_localcrawl").getJobCount())); + prop.put("localCrawlSize", Integer.toString(switchboard.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount())); + prop.put("localCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING); int stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE); addNTable(prop, "list-local", switchboard.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize))); //global crawl queue - prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("61_globalcrawltrigger").getJobCount())); - //prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("62_remotetriggeredcrawl").getJobCount())); + prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount())); + prop.put("remoteCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER) ? STATE_PAUSED : STATE_RUNNING); stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT); if (stackSize == 0) { prop.put("list-remote", 0); diff --git a/htroot/xml/queues_p.xml b/htroot/xml/queues_p.xml index 716064067..341fbf49f 100644 --- a/htroot/xml/queues_p.xml +++ b/htroot/xml/queues_p.xml @@ -34,6 +34,7 @@ #[localCrawlSize]# + #[localCrawlState]# #{list-local}# #[profile]# @@ -49,6 +50,7 @@ #[remoteCrawlSize]# + #[remoteCrawlState]# #{list-remote}# #[profile]#