diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index cf538c261..fe542f696 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -204,7 +204,7 @@ public class CrawlStartScanner_p final DigestURI url = pkmap.get(pk); if ( url != null ) { String path = - "/Crawler_p.html?createBookmark=off&xsstopw=off&crawlingDomMaxPages=10000&intention=&range=domain&indexMedia=on&recrawl=nodoubles&xdstopw=off&storeHTCache=on&sitemapURL=&repeat_time=7&crawlingQ=on&cachePolicy=iffresh&indexText=on&crawlingMode=url&mustnotmatch=&crawlingDomFilterDepth=1&crawlingDomFilterCheck=off&crawlingstart=Start%20New%20Crawl&xpstopw=off&repeat_unit=seldays&crawlingDepth=99"; + "/Crawler_p.html?createBookmark=off&xsstopw=off&crawlingDomMaxPages=10000&intention=&range=domain&indexMedia=on&recrawl=nodoubles&xdstopw=off&storeHTCache=on&sitemapURL=&repeat_time=7&crawlingQ=on&cachePolicy=iffresh&indexText=on&crawlingMode=url&mustnotmatch=&crawlingDomFilterDepth=1&crawlingDomFilterCheck=off&crawlingstart=Start%20New%20Crawl&xpstopw=off&repeat_unit=seldays&crawlingDepth=99&directDocByURL=off"; path += "&crawlingURL=" + url.toNormalform(true, false); WorkTables.execAPICall( "localhost", diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java index dff58b49b..ae8814eb1 100644 --- a/htroot/Crawler_p.java +++ b/htroot/Crawler_p.java @@ -85,8 +85,10 @@ public class Crawler_p { prop.put("localCrawlState", ""); prop.put("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize()); prop.put("limitCrawlState", ""); - prop.put("remoteCrawlSize", sb.crawlQueues.limitCrawlJobSize()); + prop.put("remoteCrawlSize", sb.crawlQueues.remoteTriggeredCrawlJobSize()); prop.put("remoteCrawlState", ""); + prop.put("noloadCrawlSize", sb.crawlQueues.noloadCrawlJobSize()); + prop.put("noloadCrawlState", ""); prop.put("list-remote", 0); prop.put("forwardToCrawlStart", "0"); diff --git a/htroot/api/queues_p.java b/htroot/api/queues_p.java index e630ccbdd..2c9b81bb4 100755 --- a/htroot/api/queues_p.java +++ b/htroot/api/queues_p.java @@ -79,7 +79,7 @@ public class queues_p { prop.put("limitCrawlState", STATE_RUNNING); stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.StackType.LIMIT); - //global crawl queue + //remote crawl queue prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()); prop.put("remoteCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL) ? STATE_PAUSED : STATE_RUNNING); stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.StackType.LIMIT); @@ -89,6 +89,12 @@ public class queues_p { } else { addNTable(sb, prop, "list-remote", sb.crawlQueues.noticeURL.top(NoticedURL.StackType.LIMIT, Math.min(10, stackSize))); } + + //noload crawl queue + prop.putNum("noloadCrawlSize", sb.crawlQueues.noloadCrawlJobSize()); + prop.put("noloadCrawlState", STATE_RUNNING); + //stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.StackType.NOLOAD); + // return rewrite properties return prop; diff --git a/htroot/api/queues_p.xml b/htroot/api/queues_p.xml index 018ec13e0..d8d3d8abb 100644 --- a/htroot/api/queues_p.xml +++ b/htroot/api/queues_p.xml @@ -64,4 +64,8 @@ #{/list-remote}# + + #[noloadCrawlSize]# + #[noloadCrawlState]# + diff --git a/htroot/api/status_p.java b/htroot/api/status_p.java index 0faad5718..ce26f1b65 100644 --- a/htroot/api/status_p.java +++ b/htroot/api/status_p.java @@ -38,6 +38,7 @@ public class status_p { prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount()); prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize()); prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()); + prop.putNum("noloadCrawlSize", sb.crawlQueues.noloadCrawlJobSize()); prop.putNum("loaderSize", sb.crawlQueues.workerSize()); prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10)); diff --git a/htroot/api/status_p.xml b/htroot/api/status_p.xml index 0ab1ec071..a8f4ae4eb 100644 --- a/htroot/api/status_p.xml +++ b/htroot/api/status_p.xml @@ -17,6 +17,9 @@ #[remoteCrawlSize]# + + #[noloadCrawlSize]# + #[freeMemory]# diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index 935636d4c..3be5e3968 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -499,6 +499,10 @@ public class CrawlQueues { return this.noticeURL.stackSize(NoticedURL.StackType.LIMIT); } + public int noloadCrawlJobSize() { + return this.noticeURL.stackSize(NoticedURL.StackType.NOLOAD); + } + public int remoteTriggeredCrawlJobSize() { return this.noticeURL.stackSize(NoticedURL.StackType.REMOTE); } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 01cabe127..a4cc76b07 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -1284,6 +1284,7 @@ public final class Switchboard extends serverSwitch { this.crawlQueues.coreCrawlJobSize() > 0 || this.crawlQueues.limitCrawlJobSize() > 0 || this.crawlQueues.remoteTriggeredCrawlJobSize() > 0 || + this.crawlQueues.noloadCrawlJobSize() > 0 || (this.crawlStacker != null && !this.crawlStacker.isEmpty()) || this.crawlQueues.noticeURL.notEmpty()) return false;