From c846e9ca14d4d04c755e5690efe805fb6ff8c631 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 25 May 2012 01:45:38 +0200 Subject: [PATCH] redesign of the crawler monitor page: show crawled pages instead of queue of urls that shall be crawled --- htroot/Crawler_p.html | 47 ++++++++----------- .../templates/submenuCrawlMonitor.template | 2 +- htroot/js/Crawler.js | 23 ++++++--- source/net/yacy/cora/document/RSSMessage.java | 10 ++++ source/net/yacy/search/Switchboard.java | 4 +- 5 files changed, 47 insertions(+), 39 deletions(-) diff --git a/htroot/Crawler_p.html b/htroot/Crawler_p.html index 4b706f93f..2087bda8f 100644 --- a/htroot/Crawler_p.html +++ b/htroot/Crawler_p.html @@ -1,27 +1,21 @@ -YaCy '#[clientname]#': Crawler Queues +YaCy '#[clientname]#': Crawler #%env/templates/metas.template%# + + - - + #%env/templates/header.template%# #%env/templates/submenuCrawlMonitor.template%# -

Crawler Queues

+

Crawler

-
+
Queues @@ -74,7 +68,7 @@
-
+
Index Size @@ -93,13 +87,13 @@
-
+
Progress
- + @@ -113,23 +107,20 @@ - + - - - - -
IndicatorIndicator Level
PPM (Pages Per Minute)          
Traffic (Crawler)     MB  
RWI RAM (Word Cache)  
-

#(info)# +

+

+#(info)# :: @@ -155,11 +146,11 @@ Crawling of "#[crawlingURL]#" started. Please wait some seconds, it may take some seconds until the first result appears there. If you crawl any un-wanted pages, you can delete them here.
- #(/info)#

-
-

+#(/info)# +

+

See an access timing

#(crawlProfilesShow)#::
@@ -187,13 +178,13 @@ #{/list}# +

Crawled Pages

+

#(/crawlProfilesShow)# -

See an access timing

- - + #%env/templates/footer.template%# diff --git a/htroot/env/templates/submenuCrawlMonitor.template b/htroot/env/templates/submenuCrawlMonitor.template index 992c36bd5..cf471166b 100644 --- a/htroot/env/templates/submenuCrawlMonitor.template +++ b/htroot/env/templates/submenuCrawlMonitor.template @@ -5,7 +5,7 @@

Processing Monitor

diff --git a/htroot/js/Crawler.js b/htroot/js/Crawler.js index 9747dba41..fd5c55d0e 100644 --- a/htroot/js/Crawler.js +++ b/htroot/js/Crawler.js @@ -3,9 +3,8 @@ BAR_IMG1="/env/grafics/green-block.png"; BAR_IMG2="/env/grafics/red-block.png"; WORDCACHEBAR_LENGTH=1/4; - var statusRPC; -var refreshInterval=3; +var refreshInterval=2; var wait=0; var changing=false; //change the interval var statusLoaded=true; @@ -50,6 +49,7 @@ function refresh(){ wait=refreshInterval; statusLoaded=false; requestStatus(); + getRSS("/api/feed.xml?count=20&set=REMOTEINDEXING,LOCALINDEXING&time=" + (new Date()).getTime()); } function requestStatus(){ @@ -90,10 +90,6 @@ function handleStatus(){ var wordCacheSize=getValue(getFirstChild(statusTag, "wordCacheSize")); var wordCacheMaxSize=getValue(getFirstChild(statusTag, "wordCacheMaxSize")); - wordCacheNum=document.getElementById("wordcacheNum"); - removeAllChildren(wordCacheNum); - wordCacheNum.appendChild(document.createTextNode(wordCacheSize+"/"+wordCacheMaxSize)); - wordCacheSpan=document.getElementById("wordcacheSpan"); removeAllChildren(wordCacheSpan); var img; @@ -164,7 +160,6 @@ function putQueueState(queue, state) { } } - function shortenURL(url) { if (url.length > 80) { return url.substr(0, 80) + "..."; @@ -187,3 +182,17 @@ function createIndexingRow(queue, profile, initiator, depth, modified, anchor, u row.appendChild(deletebutton); return row; } + +crawllist_head = ""; +crawllist_body = ""; +crawllist_tail = "
TitleURL
"; +function showRSS(RSS) { + var doc = document.getElementById("crawllist"); + if (doc != null) { + for (var i=0; i" + RSS.items[i].description + "" + RSS.items[i].link + "" + crawllist_body; + } + doc.innerHTML = crawllist_head + crawllist_body + crawllist_tail; + } + return true; +} diff --git a/source/net/yacy/cora/document/RSSMessage.java b/source/net/yacy/cora/document/RSSMessage.java index 683c512f2..e8a11c3bc 100644 --- a/source/net/yacy/cora/document/RSSMessage.java +++ b/source/net/yacy/cora/document/RSSMessage.java @@ -36,6 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.kelondro.data.meta.DigestURI; public class RSSMessage implements Hit, Comparable, Comparator { @@ -105,6 +106,15 @@ public class RSSMessage implements Hit, Comparable, Comparator(); + this.map.put("title", title); + this.map.put("description", description); + this.map.put("link", link.toNormalform(true, false)); + this.map.put("pubDate", ISO8601Formatter.FORMATTER.format()); + this.map.put("guid", ASCII.String(link.hash())); + } public RSSMessage() { this.map = new ConcurrentHashMap(); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 171a13905..72030985d 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2621,9 +2621,7 @@ public final class Switchboard extends serverSwitch ASCII.getBytes(this.peers.mySeed().hash)) ? EventChannel.LOCALINDEXING : EventChannel.REMOTEINDEXING); - feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url().toNormalform( - true, - false))); + feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url())); } catch ( final IOException e ) { //if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase); addURLtoErrorDB(