Let auto-disabled crawls recover if low resource condition vanished.

Analog to autodisabled DHT switch autodisabled crawls back on upon mem ok
by remembering the autodisable by conf parameter.
pull/1/head
reger 10 years ago
parent b32e0b5457
commit 42b0672be3

@ -4,7 +4,7 @@
<groupId>net.yacy</groupId> <groupId>net.yacy</groupId>
<artifactId>yacycore</artifactId> <artifactId>yacycore</artifactId>
<version>1.81</version> <version>1.83</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<description>YaCy - a Peer to Peer Web Search Engine</description> <description>YaCy - a Peer to Peer Web Search Engine</description>
<name>YaCy</name> <name>YaCy</name>

@ -92,10 +92,12 @@ public class ResourceObserver {
if (!this.sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) { if (!this.sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
log.info("pausing local crawls"); log.info("pausing local crawls");
this.sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "resource observer:" + reason); this.sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "resource observer:" + reason);
this.sb.setConfig(SwitchboardConstants.CRAWLJOB_LOCAL_AUTODISABLED,true);
} }
if (!this.sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) { if (!this.sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
log.info("pausing remote triggered crawls"); log.info("pausing remote triggered crawls");
this.sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "resource observer:" + reason); this.sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "resource observer:" + reason);
this.sb.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_AUTODISABLED,true);
} }
if ((this.normalizedDiskFree == Space.EXHAUSTED || this.normalizedMemoryFree != Space.AMPLE) && this.sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) { if ((this.normalizedDiskFree == Space.EXHAUSTED || this.normalizedMemoryFree != Space.AMPLE) && this.sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) {
@ -220,6 +222,14 @@ public class ResourceObserver {
this.sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true); this.sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
this.sb.peers.mySeed().setFlagAcceptRemoteIndex(true); this.sb.peers.mySeed().setFlagAcceptRemoteIndex(true);
this.sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false); this.sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false);
} else if (this.sb.getConfigBool(SwitchboardConstants.CRAWLJOB_LOCAL_AUTODISABLED, false)) {
log.info("continue paused local crawls");
this.sb.setConfig(SwitchboardConstants.CRAWLJOB_LOCAL_AUTODISABLED,false);
this.sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
} else if (this.sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE_AUTODISABLED, false)) {
log.info("continue paused remote triggered crawls");
this.sb.setConfig(SwitchboardConstants.CRAWLJOB_REMOTE_AUTODISABLED,false);
this.sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
} }
log.info("resources ok"); log.info("resources ok");
} }

@ -475,7 +475,9 @@ public final class SwitchboardConstants {
public static final String RESOURCE_DISK_USED_MAX_OVERSHOT = "resource.disk.used.max.overshot"; public static final String RESOURCE_DISK_USED_MAX_OVERSHOT = "resource.disk.used.max.overshot";
public static final String MEMORY_ACCEPTDHT = "memory.acceptDHTabove"; public static final String MEMORY_ACCEPTDHT = "memory.acceptDHTabove";
public static final String INDEX_RECEIVE_AUTODISABLED = "memory.disabledDHT"; public static final String INDEX_RECEIVE_AUTODISABLED = "memory.disabledDHT"; // set if DHT was disabled by ResourceObserver
public static final String CRAWLJOB_LOCAL_AUTODISABLED = "memory.disabledLocalCrawler"; // set if local crawl was disabled by ResourceObserver
public static final String CRAWLJOB_REMOTE_AUTODISABLED = "memory.disabledRemoteCrawler"; // set if remote crawl was disabled by ResourceObserver
/* /*
* Some constants * Some constants

Loading…
Cancel
Save