diff --git a/htroot/WatchCrawler_p.html b/htroot/WatchCrawler_p.html index 44b2b8bfa..3152412e0 100644 --- a/htroot/WatchCrawler_p.html +++ b/htroot/WatchCrawler_p.html @@ -12,48 +12,68 @@ #%env/templates/header.template%#

Crawler Monitor

- Next update in seconds.
- PPM:    
- Wordcaches:   + Next update in seconds.

-

- Indexing Queue: size=   ; max=    -

- + +

+ + + + + + + +
- - - - - + - + + + + + + + + + + + + + + + + + + + + + -
InitiatorDepthModified DateAnchor NameURLQueue SizeDeleteMax
Indexing      
Loader      
Local Crawler   unlimited
Remote Crawler   unlimited
-

- Loader: size=   ; max=    -

- +
  - - - - - - - + + + + + + + + + + + + -
InitiatorDepthModified DateAnchor NameURLSizeDeleteIndicatorLevel
PPM      
Word Caches  
-

- Local Crawler: size=    -

- +

+ +

+ @@ -63,25 +83,8 @@ -
Queue Initiator Depth Modified DateDelete
- -

- Remote Crawler: size=    -

- - - - - - - - - - - - -
InitiatorDepthModified DateAnchor NameURLSizeDelete
- +

+ #%env/templates/footer.template%# diff --git a/htroot/js/WatchCrawler.js b/htroot/js/WatchCrawler.js index 710390425..ab8658ec0 100644 --- a/htroot/js/WatchCrawler.js +++ b/htroot/js/WatchCrawler.js @@ -64,10 +64,13 @@ function handleStatus(){ statusTag=getFirstChild(statusResponse, "status") ppm=getValue(getFirstChild(statusTag, "ppm")) - var ppmSpan = document.getElementById("ppm"); + + var ppmNum = document.getElementById("ppmNum"); + removeAllChildren(ppmNum); + ppmNum.appendChild(document.createTextNode(ppm)); + + var ppmSpan = document.getElementById("ppmSpan"); removeAllChildren(ppmSpan); - ppmSpan.appendChild(document.createTextNode(ppm)); - ppmSpan.appendChild(document.createElement("br")); for(i=0;i 0)) { entryList.addAll(switchboard.sbQueue.list(0)); } - int size=10; - if (post!= null) size = post.getInt("num", 10); - - if(size>entryList.size()){ - size=entryList.size(); - } - + int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size()); + if (size > entryList.size()) size = entryList.size(); + for (i = 0; i < size; i++) { boolean inProcess = i < inProcessCount; pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i); @@ -140,13 +136,9 @@ public class queues_p { prop.put("list-loader", 0); } else { ThreadGroup loaderThreads = switchboard.cacheLoader.threadStatus(); - int threadCount = loaderThreads.activeCount(); - Thread[] threadList = new Thread[threadCount*2]; - threadCount = loaderThreads.enumerate(threadList); - int size=10; - if(threadCount 0) flushOnce(); + private void flushSome(int count) throws IOException { + while ((domainStacks.size() > 0) && (count-- > 0)) flushOnce(); } public void add(String domain, byte[] hash) throws IOException { @@ -150,7 +150,7 @@ public class plasmaCrawlBalancer { } public byte[] top(int dist) throws IOException { - flushAll(); + flushSome(1 + dist - stack.size()); // flush only that much as we need to display synchronized (domainStacks) { return stack.top(dist).getColBytes(0); } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 5cf7e82fd..07d60fd83 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -168,7 +168,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // load slots public static int crawlSlots = 10; - public static int indexingSlots = 100; + public static int indexingSlots = 30; public static int stackCrawlSlots = 1000000; public static int maxCRLDump = 500000; @@ -472,7 +472,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.sbQueue = new plasmaSwitchboardQueue(this.cacheManager, this.wordIndex.loadedURL, new File(this.plasmaPath, "switchboardQueue1.stack"), this.profiles); // setting the indexing queue slots - indexingSlots = (int) getConfigLong("indexer.slots", 100); + indexingSlots = (int) getConfigLong("indexer.slots", 30); // create in process list this.indexingTasksInProcess = new HashMap(); diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 1dcea1c73..8378022ef 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -458,7 +458,7 @@ public final class yacyClient { ), "UTF-8" ); } catch (IOException e) { - yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes)); + yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes.substring(0, 12))); yacyCore.peerActions.peerDeparture(targetPeer); return null; } diff --git a/yacy.init b/yacy.init index 94f54166c..518644847 100644 --- a/yacy.init +++ b/yacy.init @@ -662,7 +662,7 @@ stacker.MaxIdleThreads = 10 stacker.MinIdleThreads = 5 # maximum size of indexing queue -indexer.slots = 100 +indexer.slots = 30 # specifies if yacy should set it's own referer if no referer URL # was set by the client.