From 71ed8e5e07b7617124ebc6ac3b759b05518bfd3e Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Wed, 7 Nov 2012 12:52:19 +0100 Subject: [PATCH] bugfixes for crawler --- htroot/Crawler_p.html | 7 ++++++- source/net/yacy/crawler/robots/RobotsTxt.java | 3 ++- source/net/yacy/search/Switchboard.java | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/htroot/Crawler_p.html b/htroot/Crawler_p.html index 402c5a4fc..ec9fe4181 100644 --- a/htroot/Crawler_p.html +++ b/htroot/Crawler_p.html @@ -11,6 +11,12 @@ + +
+API +Click on this API button to see an XML with information about the crawler status +
+ #%env/templates/header.template%# #%env/templates/submenuCrawlMonitor.template%#

Crawler

@@ -150,7 +156,6 @@

-

See an access timing

#(crawlProfilesShow)#::
diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.java index aa4b55f65..b680c9cf7 100644 --- a/source/net/yacy/crawler/robots/RobotsTxt.java +++ b/source/net/yacy/crawler/robots/RobotsTxt.java @@ -168,7 +168,8 @@ public class RobotsTxt { Request request = new Request(robotsURL, null); try { response = this.loader.load(request, CacheStrategy.NOCACHE, null, 0); - } catch (IOException e) { + } catch (Throwable e) { + log.info("Trying to download the robots.txt file from URL '" + robotsURL + "' failed - " + e.getMessage()); response = null; } } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index dbb059283..4d78abe4c 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2277,7 +2277,7 @@ public final class Switchboard extends serverSwitch { status[SwitchboardConstants.CRAWLJOB_STATUS] = Boolean.TRUE; } setConfig(jobType + "_isPaused", "true"); - setConfig(jobType + "_isPaused_cause", "cause"); + setConfig(jobType + "_isPaused_cause", cause); log.logWarning("Crawl job '" + jobType + "' is paused: " + cause); }