diff --git a/htroot/Crawler_p.html b/htroot/Crawler_p.html index 81f144ecb..007adae4d 100644 --- a/htroot/Crawler_p.html +++ b/htroot/Crawler_p.html @@ -1,4 +1,4 @@ - + YaCy '#[clientname]#': Crawler @@ -28,13 +28,14 @@
Queues - + - + + + @@ -88,12 +89,14 @@
Index Size
Queue
 		 Size
 		 -
Local Crawler #[localCrawlSize]#
- + + + @@ -121,11 +124,13 @@ Progress
Database
 		 Entries
 		 Seg-
ments
Documents
solr search api		 #[urlpublictextSize]#
- + + +
Indicator
 		 Level
 
Speed / PPM
(Pages Per Minute)		 @@ -219,33 +224,39 @@ window.setInterval("setTableSize()", 1000); #(crawlProfilesShow)#::
- Running Crawls (#[count]#) + Running Crawls (#[count]#) #(linkstructure)# diff --git a/htroot/js/Crawler.js b/htroot/js/Crawler.js index a7bcd65a1..1b4ea60d5 100644 --- a/htroot/js/Crawler.js +++ b/htroot/js/Crawler.js @@ -28,13 +28,35 @@ var refreshInterval=2; var wait=0; var changing=false; //change the interval var statusLoaded=true; +/* Running crawls table DOM element */ +var crawlsTable; +/* Size of the running crawls table header */ +var crawlsHeadLength; +/* Running crawls legend DOM element */ +var runningCrawlsLegend; +/* true when debug is enabled */ +var debug; function initCrawler(){ + initCrawlProfiles(); + refresh(); //loadInterval=window.setInterval("refresh()", refreshInterval*1000); countInterval=window.setInterval("countdown()", 1000); } +/** + * Init variables used to refresh the running crawls table + */ +function initCrawlProfiles() { + debug = document.getElementById("headerDebug") != null; + crawlsTable = document.getElementById("crawlProfiles"); + if(crawlsTable != null && crawlsTable.rows != null) { + crawlsHeadLength = crawlsTable.tHead != null ? crawlsTable.tHead.rows.length : 0; + } + runningCrawlsLegend = document.getElementById("runningCrawlsLegend"); +} + function changeInterval(){ if(!changing){ window.clearInterval(countInterval); @@ -84,9 +106,9 @@ function handleStatus(){ return; } var statusResponse = statusRPC.responseXML; - statusTag=getFirstChild(statusResponse, "status"); + var statusTag = getFirstChild(statusResponse, "status"); - ppm=getValue(getFirstChild(statusTag, "ppm")); + var ppm = getValue(getFirstChild(statusTag, "ppm")); var ppmNum = document.getElementById("ppmNum"); removeAllChildren(ppmNum); @@ -102,21 +124,21 @@ function handleStatus(){ // ppmBar end // traffic output (no bar up to now) - traffic = getFirstChild(statusTag, "traffic"); - trafficCrawlerValue = getValue(getFirstChild(traffic, "crawler")); - trafCrawlerSpan = document.getElementById("trafficCrawler"); + var traffic = getFirstChild(statusTag, "traffic"); + var trafficCrawlerValue = getValue(getFirstChild(traffic, "crawler")); + var trafCrawlerSpan = document.getElementById("trafficCrawler"); removeAllChildren(trafCrawlerSpan); trafCrawlerSpan.appendChild(document.createTextNode(Math.round((trafficCrawlerValue) / 1024 / 10.24) / 100)); - dbsize=getFirstChild(statusTag, "dbsize"); - urlpublictext=getValue(getFirstChild(dbsize, "urlpublictext")); - urlpublictextSegmentCount=getValue(getFirstChild(dbsize, "urlpublictextSegmentCount")); - webgraph=getValue(getFirstChild(dbsize, "webgraph")); - webgraphSegmentCount=getValue(getFirstChild(dbsize, "webgraphSegmentCount")); - citation=getValue(getFirstChild(dbsize, "citation")); - citationSegmentCount=getValue(getFirstChild(dbsize, "citationSegmentCount")); - rwipublictext=getValue(getFirstChild(dbsize, "rwipublictext")); - rwipublictextSegmentCount=getValue(getFirstChild(dbsize, "rwipublictextSegmentCount")); + var dbsize = getFirstChild(statusTag, "dbsize"); + var urlpublictext = getValue(getFirstChild(dbsize, "urlpublictext")); + var urlpublictextSegmentCount = getValue(getFirstChild(dbsize, "urlpublictextSegmentCount")); + var webgraph = getValue(getFirstChild(dbsize, "webgraph")); + var webgraphSegmentCount = getValue(getFirstChild(dbsize, "webgraphSegmentCount")); + var citation = getValue(getFirstChild(dbsize, "citation")); + var citationSegmentCount = getValue(getFirstChild(dbsize, "citationSegmentCount")); + var rwipublictext = getValue(getFirstChild(dbsize, "rwipublictext")); + var rwipublictextSegmentCount = getValue(getFirstChild(dbsize, "rwipublictextSegmentCount")); document.getElementById("urlpublictextSize").firstChild.nodeValue=urlpublictext; document.getElementById("urlpublictextSegmentCount").firstChild.nodeValue=urlpublictextSegmentCount; document.getElementById("webgraphSize").firstChild.nodeValue=webgraph; @@ -125,55 +147,223 @@ function handleStatus(){ document.getElementById("citationSegmentCount").firstChild.nodeValue=citationSegmentCount; document.getElementById("rwipublictextSize").firstChild.nodeValue=rwipublictext; document.getElementById("rwipublictextSegmentCount").firstChild.nodeValue=rwipublictextSegmentCount; + + refreshRunningCrawls(statusTag); - postprocessing=getFirstChild(statusTag, "postprocessing"); + var postprocessing = getFirstChild(statusTag, "postprocessing"); document.getElementById("postprocessing_status").firstChild.nodeValue=getValue(getFirstChild(postprocessing, "status")); document.getElementById("postprocessing_collection").firstChild.nodeValue=getValue(getFirstChild(postprocessing, "collectionRemainingCount")); document.getElementById("postprocessing_webgraph").firstChild.nodeValue=getValue(getFirstChild(postprocessing, "webgraphRemainingCount")); document.getElementById("postprocessing_remainingTimeMinutes").firstChild.nodeValue=getValue(getFirstChild(postprocessing, "remainingTimeMinutes")); document.getElementById("postprocessing_remainingTimeSeconds").firstChild.nodeValue=getValue(getFirstChild(postprocessing, "remainingTimeSeconds")); - postprocessingElapsedTime=parseInt(getValue(getFirstChild(postprocessing, "ElapsedTime"))); - postprocessingRemainingTime=parseInt(getValue(getFirstChild(postprocessing, "RemainingTime"))); - p = 100 * postprocessingElapsedTime / (postprocessingElapsedTime + postprocessingRemainingTime) || 0; + var postprocessingElapsedTime = parseInt(getValue(getFirstChild(postprocessing, "ElapsedTime"))); + var postprocessingRemainingTime = parseInt(getValue(getFirstChild(postprocessing, "RemainingTime"))); + var p = 100 * postprocessingElapsedTime / (postprocessingElapsedTime + postprocessingRemainingTime) || 0; document.getElementById("postprocessing_bar").firstChild.setAttribute("value", p); //document.getElementById("postprocessing_speed").firstChild.nodeValue=getValue(getFirstChild(postprocessing, "speed")); - load=getFirstChild(statusTag, "load"); + var load = getFirstChild(statusTag, "load"); document.getElementById("load").firstChild.nodeValue=getValue(load); - loaderqueue=getFirstChild(statusTag, "loaderqueue"); - loaderqueue_size=getValue(getFirstChild(loaderqueue, "size")); - loaderqueue_max=getValue(getFirstChild(loaderqueue, "max")); + var loaderqueue = getFirstChild(statusTag, "loaderqueue"); + var loaderqueue_size = getValue(getFirstChild(loaderqueue, "size")); + var loaderqueue_max = getValue(getFirstChild(loaderqueue, "max")); document.getElementById("loaderqueuesize").firstChild.nodeValue=loaderqueue_size; document.getElementById("loaderqueuemax").firstChild.nodeValue=loaderqueue_max; - localcrawlerqueue=getFirstChild(statusTag, "localcrawlerqueue"); - localcrawlerqueue_size=getValue(getFirstChild(localcrawlerqueue, "size")); - localcrawlerqueue_state=getValue(getFirstChild(localcrawlerqueue, "state")); + var localcrawlerqueue = getFirstChild(statusTag, "localcrawlerqueue"); + var localcrawlerqueue_size = getValue(getFirstChild(localcrawlerqueue, "size")); + var localcrawlerqueue_state = getValue(getFirstChild(localcrawlerqueue, "state")); document.getElementById("localcrawlerqueuesize").firstChild.nodeValue=localcrawlerqueue_size; putQueueState("localcrawler", localcrawlerqueue_state); - limitcrawlerqueue=getFirstChild(statusTag, "limitcrawlerqueue"); - limitcrawlerqueue_size=getValue(getFirstChild(limitcrawlerqueue, "size")); - limitcrawlerqueue_state=getValue(getFirstChild(limitcrawlerqueue, "state")); + var limitcrawlerqueue = getFirstChild(statusTag, "limitcrawlerqueue"); + var limitcrawlerqueue_size = getValue(getFirstChild(limitcrawlerqueue, "size")); + var limitcrawlerqueue_state = getValue(getFirstChild(limitcrawlerqueue, "state")); document.getElementById("limitcrawlerqueuesize").firstChild.nodeValue=limitcrawlerqueue_size; putQueueState("limitcrawler", limitcrawlerqueue_state); - remotecrawlerqueue=getFirstChild(statusTag, "remotecrawlerqueue"); - remotecrawlerqueue_size=getValue(getFirstChild(remotecrawlerqueue, "size")); - remotecrawlerqueue_state=getValue(getFirstChild(remotecrawlerqueue, "state")); + var remotecrawlerqueue = getFirstChild(statusTag, "remotecrawlerqueue"); + var remotecrawlerqueue_size = getValue(getFirstChild(remotecrawlerqueue, "size")); + var remotecrawlerqueue_state = getValue(getFirstChild(remotecrawlerqueue, "state")); document.getElementById("remotecrawlerqueuesize").firstChild.nodeValue=remotecrawlerqueue_size; putQueueState("remotecrawler", remotecrawlerqueue_state); - noloadcrawlerqueue=getFirstChild(statusTag, "noloadcrawlerqueue"); - noloadcrawlerqueue_size=getValue(getFirstChild(noloadcrawlerqueue, "size")); - noloadcrawlerqueue_state=getValue(getFirstChild(noloadcrawlerqueue, "state")); + var noloadcrawlerqueue = getFirstChild(statusTag, "noloadcrawlerqueue"); + var noloadcrawlerqueue_size = getValue(getFirstChild(noloadcrawlerqueue, "size")); + var noloadcrawlerqueue_state = getValue(getFirstChild(noloadcrawlerqueue, "state")); document.getElementById("noloadcrawlerqueuesize").firstChild.nodeValue=noloadcrawlerqueue_size; putQueueState("noloadcrawler", noloadcrawlerqueue_state); statusLoaded=true; } +/** + * Insert a new crawl line to the end of the running crawls table + * @param table crawls table HTML DOM node + * @param crawl crawl profile node from status_p.xml + * @param handle {String} identifier of the running crawl profile + * @param status {String} running status of the crawl profile + */ +function insertCrawlRaw(table, crawl, handle, status) { + /* Insert a row in the table at the end */ + var newRow = table.insertRow(); + newRow.className = ((table.rows.length - crawlsHeadLength) % 2) == 0 ? "TableCellLight" : "TableCellDark"; + newRow.id = handle; + + /* Insert name cell */ + var newCell = newRow.insertCell(); + var newText = document.createTextNode(getValue(getFirstChild(crawl, "name"))); + newCell.appendChild(newText); + + if(debug) { + /* Insert count cell when debug is enabled */ + newCell = newRow.insertCell(); + newCell.textContent = getValue(getFirstChild(crawl, "count")); + } + + /* Insert status cell */ + newCell = newRow.insertCell(); + newCell.id = handle + "_status_cell"; + + if(status == "alive") { + var newDiv = document.createElement("div"); + newDiv.id = handle + "_status"; + newDiv.style = "text-decoration:blink;float:left;"; + + newText = document.createTextNode("Running"); + newDiv.appendChild(newText); + + newCell.appendChild(newDiv); + + var newForm = document.createElement("form"); + newForm.id = handle + "_terminate"; + newForm.style = "float:left;"; + newForm.action = "Crawler_p.html"; + newForm.method = "get"; + newForm.enctype="multipart/form-data"; + newForm["accept-charset"]="UTF-8"; + + newDiv = document.createElement("div"); + + var newInput = document.createElement("input"); + newInput.type = "hidden"; + newInput.name = "handle"; + newInput.value = handle; + + newDiv.appendChild(newInput); + + newInput = document.createElement("input"); + newInput.type = "submit"; + newInput.name = "terminate"; + newInput.value = "Terminate"; + newInput.className = "btn btn-danger btn-xs"; + + newDiv.appendChild(newInput); + + newForm.appendChild(newDiv); + + newCell.appendChild(newForm); + } +} + +/** + * Refresh status cell text and terminate button presence + * @param handle name of the crawl + * @param status current crawl status label + */ +function refreshStatusCell(handle, status) { + var handleStatus = document.getElementById(handle + "_status"); + if(handleStatus != null) { + handleStatus.textContent = status; + } + var terminateForm = document.getElementById(handle + "_terminate"); + if(terminateForm != null && terminateForm.parentElement) { + terminateForm.parentElement.removeChild(terminateForm); + } +} + +/** + * Refresh the count in running crawls legend + * @param legend the HTML DOM legend element + * @param crawls crawls node from xml api status_p.xml + */ +function refreshCrawlsLegend(legend, crawls) { + var count = crawls.getAttribute("count"); + if(count && legend != null) { + legend.textContent = "Running Crawls (" + count + ")"; + } +} + +/** + * Refresh dark/light rows style + * @param table running crawls table + */ +function refreshRowsStyle(table, headLength) { + for(var i = headLength; i < table.rows.length; i++) { + raw = table.rows[i]; + raw.className = ((i - headLength) % 2) == 0 ? "TableCellLight" : "TableCellDark"; + } +} + +/** + * Refresh running crawls table + * + * @param statusTag + * status tag from xml api status_p.xml + */ +function refreshRunningCrawls(statusTag) { + var crawls = getFirstChild(statusTag, "crawls"); + /* crawls node should be present even when no crawl is running */ + if(crawls != null) { + /* Update the table when present */ + if(crawlsTable != null && crawlsTable.rows != null) { + var processedHandles = {}, crawlNode = getFirstChild(crawls, "crawl"); + + if(crawlNode) { + var handle, rowIndex, handleCell; + /* Loop on crawl node elements from xml */ + for(; crawlNode; crawlNode = getNextSibling(crawlNode, "crawl")) { + handle = getValue(getFirstChild(crawlNode, "handle")); + if(handle != null) { + processedHandles[handle] = crawlNode; + status = getValue(getFirstChild(crawlNode, "status")); + /* Let's try to get the crawls table cell with id prefixed by this handle */ + handleCell = document.getElementById(handle + "_status_cell"); + if(handleCell == null) { + insertCrawlRaw(crawlsTable, crawlNode, handle, status); + refreshCrawlsLegend(runningCrawlsLegend, crawls); + refreshRowsStyle(crawlsTable, crawlsHeadLength); + } else if(status != "alive"){ + refreshStatusCell(handle, status); + } + } + } + } + + /* Collect raws to delete */ + var raw, rawsToDelete = []; + for(var i = crawlsHeadLength; i < crawlsTable.rows.length; i++) { + raw = crawlsTable.rows[i]; + if(processedHandles[raw.id] == null) { + rawsToDelete.push(raw); + } + } + + /* Delete raws */ + for(var i = 0; i < rawsToDelete.length; i++) { + raw = rawsToDelete[i]; + raw.parentElement.removeChild(raw); + } + /* Refresh legend and rows style (dark/light alternate) */ + if(rawsToDelete.length > 0) { + refreshCrawlsLegend(runningCrawlsLegend, crawls); + refreshRowsStyle(crawlsTable, crawlsHeadLength); + } + } + } +} + function putQueueState(queue, state) { a = document.getElementById(queue + "stateA"); img = document.getElementById(queue + "stateIMG");
- +
- - + + - - - #(debug)#::#(/debug)# - - + + + + #(debug)#::#(/debug)# + + + + #{list}# - - - #(debug)#::#(/debug)# - - - #{/list}# + + + #(debug)#::#(/debug)# + + + #{/list}# +
NameCountStatus
NameCountStatus
#[name]##[count]##(terminateButton)#:: -
Running
-
- - -
- #(/terminateButton)# -
#[name]##[count]##(terminateButton)#:: +
Running
+
+
+ + +
+
+ #(/terminateButton)# +