bugfixes for crawler

13 years ago · 71ed8e5e07
parent 29fbbb49dc
commit 71ed8e5e07
3 changed files with 9 additions and 3 deletions
--- a/htroot/Crawler_p.html
+++ b/htroot/Crawler_p.html
@ -11,6 +11,12 @@
 <script type="text/javascript" src="/js/Crawler.js"></script>
 </head>
 <body id="Crawler" onload="initCrawler();">
+
+<div id="api">
+<a href="/api/status_p.xml" id="apilink"><img src="/env/grafics/api.png" width="60" height="40" alt="API"/></a>
+<span>Click on this API button to see an XML with information about the crawler status</span>
+</div>
+
 #%env/templates/header.template%#
 #%env/templates/submenuCrawlMonitor.template%#
 <h2>Crawler</h2>
@ -150,7 +156,6 @@
 </p>
 <!-- crawl queues -->

-<p>See an <a href="/api/latency_p.xml">access timing</a></p>
 <!-- crawl profile list -->
 #(crawlProfilesShow)#::
 <fieldset>
--- a/source/net/yacy/crawler/robots/RobotsTxt.java
+++ b/source/net/yacy/crawler/robots/RobotsTxt.java
@ -168,7 +168,8 @@ public class RobotsTxt {
                    Request request = new Request(robotsURL, null);
                    try {
                        response = this.loader.load(request, CacheStrategy.NOCACHE, null, 0);
-                    } catch (IOException e) {
+                    } catch (Throwable e) {
+                        log.info("Trying to download the robots.txt file from URL '" + robotsURL + "' failed - " + e.getMessage());
                        response = null;
                    }
                }
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@ -2277,7 +2277,7 @@ public final class Switchboard extends serverSwitch {
            status[SwitchboardConstants.CRAWLJOB_STATUS] = Boolean.TRUE;
        }
        setConfig(jobType + "_isPaused", "true");
-        setConfig(jobType + "_isPaused_cause", "cause");
+        setConfig(jobType + "_isPaused_cause", cause);
        log.logWarning("Crawl job '" + jobType + "' is paused: " + cause);
    }