boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fix for possible NPE in queues_p

Browse Source 
- WatchCrawler_p:
  - display crawler traffic
  - pause/resume local- and global crawler


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3389 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
karlchenofhell 18 years ago
parent 9702d3abba
commit bf7a69197d
5 changed files with 78 additions and 7 deletions
Show Stats Download Patch File Download Diff File

18
htroot/WatchCrawler_p.html
Unescape View File

@ -16,26 +16,39 @@
<tr class="TableHeader">
<th>Queue</th>
<th>Size</th>
<th>&nbsp;</th>
<th>Max</th>
</tr>
<tr class="TableCellLight">
<td align="left">Indexing</td>
<td align="right"><span id="indexingqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
<td>&nbsp;</td>
<td align="right"><span id="indexingqueuemax">&nbsp;&nbsp;&nbsp;</span></td>
</tr>
<tr class="TableCellLight">
<td align="left">Loader</td>
<td align="right"><span id="loaderqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
<td>&nbsp;</td>
<td align="right"><span id="loaderqueuemax">&nbsp;&nbsp;&nbsp;</span></td>
</tr>
<tr class="TableCellLight">
<td align="left">Local Crawler</td>
<td align="right"><span id="localcrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
<td>
<a href="" id="localcrawlerstateA">
<img src="" alt="" style="width:12px; height:12px;" id="localcrawlerstateIMG" />
</a>
</td>
<td align="right">unlimited</td>
</tr>
<tr class="TableCellLight">
<td align="left">Remote Crawler</td>
<td align="right"><span id="remotecrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td>
<td>
<a href="" title="" id="remotecrawlerstateA">
<img src="" alt="" style="width:12px; height:12px;" id="remotecrawlerstateIMG" />
</a>
</td>
<td align="right">unlimited</td>
</tr>
</tbody>
@ -69,6 +82,11 @@
<td align="left"><span id="ppmNum">&nbsp;&nbsp;&nbsp;</span></td>
<td align="left"><span id="ppmSpan">&nbsp;&nbsp;&nbsp;</span></td>
</tr>
<tr class="TableCellLight">
<td align="left">Traffic (Crawler)</td>
<td align="left"><span id="trafficCrawler">&nbsp;&nbsp;&nbsp;</span> MB</td>
<td>&nbsp;</td>
</tr>
<tr class="TableCellLight">
<td align="left">RWI RAM (Word Cache)</td>
<td align="left"><span id="wordcacheNum">&nbsp;</span></td>

21
htroot/WatchCrawler_p.java
Unescape View File

@ -73,6 +73,26 @@ public class WatchCrawler_p {
if (handle != null) switchboard.profiles.removeEntry(handle);
}
if (post.containsKey("continue")) {
// continue queue
String queue = post.get("continue", "");
if (queue.equals("localcrawler")) {
switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
} else if (queue.equals("remotecrawler")) {
switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
}
}
if (post.containsKey("pause")) {
// pause queue
String queue = post.get("pause", "");
if (queue.equals("localcrawler")) {
switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
} else if (queue.equals("remotecrawler")) {
switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
}
}
if (post.containsKey("crawlingstart")) {
// init crawl
if (yacyCore.seedDB == null) {
@ -301,7 +321,6 @@ public class WatchCrawler_p {
}
}
}
}
// crawl profiles

28
htroot/js/WatchCrawler.js
Unescape View File

@ -77,6 +77,13 @@ function handleStatus(){
ppmSpan.appendChild(img);
}
// traffic output (no bar up to now)
traffic = getFirstChild(statusTag, "traffic");
trafficCrawler = getValue(getFirstChild(traffic, "crawler"));
trafCrawlerSpan = document.getElementById("trafficCrawler");
removeAllChildren(trafCrawlerSpan);
trafCrawlerSpan.appendChild(document.createTextNode(Math.round((trafficCrawler) / 1024 / 10.24) / 100));
var wordCache=getValue(getFirstChild(statusTag, "wordCacheWSize"));
var wordCacheMax=getValue(getFirstChild(statusTag, "wordCacheMaxCount"));
@ -133,18 +140,39 @@ function handleQueues(){
localcrawlerqueue=getFirstChild(xml, "localcrawlerqueue");
localcrawlerqueue_size=getValue(getFirstChild(localcrawlerqueue, "size"));
localcrawlerqueue_state=getValue(getFirstChild(localcrawlerqueue, "state"));
document.getElementById("localcrawlerqueuesize").firstChild.nodeValue=localcrawlerqueue_size;
putQueueState("localcrawler", localcrawlerqueue_state);
updateTable(localcrawlerqueue, "local crawler");
remotecrawlerqueue=getFirstChild(xml, "remotecrawlerqueue");
updateTable(remotecrawlerqueue, "remoteCrawlerTable");
remotecrawlerqueue_size=getValue(getFirstChild(remotecrawlerqueue, "size"));
remotecrawlerqueue_state=getValue(getFirstChild(remotecrawlerqueue, "state"));
document.getElementById("remotecrawlerqueuesize").firstChild.nodeValue=remotecrawlerqueue_size;
putQueueState("remotecrawler", remotecrawlerqueue_state);
updateTable(remotecrawlerqueue, "remote crawler");
}
}
function putQueueState(queue, state) {
a = document.getElementById(queue + "stateA");
img = document.getElementById(queue + "stateIMG");
if (state == "paused") {
a.href = "WatchCrawler_p.html?continue=" + queue;
a.title = "Continue this queue";
img.src = "/env/grafics/start.gif";
img.alt = "Continue this queue";
} else {
a.href = "WatchCrawler_p.html?pause=" + queue;
a.title = "Pause this queue";
img.src = "/env/grafics/stop.gif";
img.alt = "Pause this queue";
}
}
function updateTable(indexingqueue, tablename){
indexingTable=document.getElementById("queueTable");
entries=indexingqueue.getElementsByTagName("entry");

16
htroot/xml/queues_p.java
Unescape View File

@ -66,6 +66,9 @@ import de.anomic.yacy.yacySeed;
public class queues_p {
public static final String STATE_RUNNING = "running";
public static final String STATE_PAUSED = "paused";
private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US);
private static String daydate(Date date) {
if (date == null) return "";
@ -83,7 +86,7 @@ public class queues_p {
yacySeed initiator;
//indexing queue
prop.put("indexingSize", switchboard.getThread("80_indexing").getJobCount()+switchboard.indexingTasksInProcess.size());
prop.put("indexingSize", switchboard.getThread(plasmaSwitchboard.INDEXER).getJobCount()+switchboard.indexingTasksInProcess.size());
prop.put("indexingMax", plasmaSwitchboard.indexingSlots);
prop.put("urlpublictextSize", switchboard.wordIndex.loadedURL.size());
prop.put("rwipublictextSize", switchboard.wordIndex.size());
@ -114,9 +117,9 @@ public class queues_p {
for (i = 0; i < size; i++) {
boolean inProcess = i < inProcessCount;
pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i);
long entrySize = pcentry.size();
totalSize += entrySize;
if ((pcentry != null)&&(pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted");
prop.putSafeXML("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : wikiCode.replaceHTML(initiator.getName())));
@ -160,13 +163,14 @@ public class queues_p {
}
//local crawl queue
prop.put("localCrawlSize", Integer.toString(switchboard.getThread("50_localcrawl").getJobCount()));
prop.put("localCrawlSize", Integer.toString(switchboard.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
prop.put("localCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
int stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
addNTable(prop, "list-local", switchboard.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
//global crawl queue
prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("61_globalcrawltrigger").getJobCount()));
//prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("62_remotetriggeredcrawl").getJobCount()));
prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount()));
prop.put("remoteCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER) ? STATE_PAUSED : STATE_RUNNING);
stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
if (stackSize == 0) {
prop.put("list-remote", 0);

2
htroot/xml/queues_p.xml
Unescape View File

@ -34,6 +34,7 @@
</loaderqueue>
<localcrawlerqueue>
<size>#[localCrawlSize]#</size>
<state>#[localCrawlState]#</state>
#{list-local}#
<entry>
<profile>#[profile]#</profile>
@ -49,6 +50,7 @@
</localcrawlerqueue>
<remotecrawlerqueue>
<size>#[remoteCrawlSize]#</size>
<state>#[remoteCrawlState]#</state>
#{list-remote}#
<entry>
<profile>#[profile]#</profile>

Write Preview
Loading…
Cancel
Save