boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

bugfixes for crawler

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 29fbbb49dc
commit 71ed8e5e07
3 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File

7
htroot/Crawler_p.html
Unescape View File

@ -11,6 +11,12 @@
<script type="text/javascript" src="/js/Crawler.js"></script> <script type="text/javascript" src="/js/Crawler.js"></script>
</head> </head>
<body id="Crawler" onload="initCrawler();"> <body id="Crawler" onload="initCrawler();">
<div id="api">
<a href="/api/status_p.xml" id="apilink"><img src="/env/grafics/api.png" width="60" height="40" alt="API"/></a>
<span>Click on this API button to see an XML with information about the crawler status</span>
</div>
#%env/templates/header.template%# #%env/templates/header.template%#
#%env/templates/submenuCrawlMonitor.template%# #%env/templates/submenuCrawlMonitor.template%#
<h2>Crawler</h2> <h2>Crawler</h2>
@ -150,7 +156,6 @@
</p> </p>
<!-- crawl queues --> <!-- crawl queues -->
<p>See an <a href="/api/latency_p.xml">access timing</a></p>
<!-- crawl profile list --> <!-- crawl profile list -->
#(crawlProfilesShow)#:: #(crawlProfilesShow)#::
<fieldset> <fieldset>

3
source/net/yacy/crawler/robots/RobotsTxt.java
Unescape View File

@ -168,7 +168,8 @@ public class RobotsTxt {
Request request = new Request(robotsURL, null); Request request = new Request(robotsURL, null);
try { try {
response = this.loader.load(request, CacheStrategy.NOCACHE, null, 0); response = this.loader.load(request, CacheStrategy.NOCACHE, null, 0);
} catch (IOException e) { } catch (Throwable e) {
log.info("Trying to download the robots.txt file from URL '" + robotsURL + "' failed - " + e.getMessage());
response = null; response = null;
} }
} }

2
source/net/yacy/search/Switchboard.java
Unescape View File

@ -2277,7 +2277,7 @@ public final class Switchboard extends serverSwitch {
status[SwitchboardConstants.CRAWLJOB_STATUS] = Boolean.TRUE; status[SwitchboardConstants.CRAWLJOB_STATUS] = Boolean.TRUE;
} }
setConfig(jobType + "_isPaused", "true"); setConfig(jobType + "_isPaused", "true");
setConfig(jobType + "_isPaused_cause", "cause"); setConfig(jobType + "_isPaused_cause", cause);
log.logWarning("Crawl job '" + jobType + "' is paused: " + cause); log.logWarning("Crawl job '" + jobType + "' is paused: " + cause);
} }

Write Preview
Loading…
Cancel
Save