From ced1a96f9c11414debb298c42caffb2fb4507b0f Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 25 Feb 2014 02:16:22 +0100 Subject: [PATCH] fixed error cache --- source/net/yacy/crawler/Balancer.java | 2 +- source/net/yacy/crawler/data/CrawlQueues.java | 5 ----- source/net/yacy/search/index/ErrorCache.java | 4 +++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/source/net/yacy/crawler/Balancer.java b/source/net/yacy/crawler/Balancer.java index 0e4da872f..86cdac873 100644 --- a/source/net/yacy/crawler/Balancer.java +++ b/source/net/yacy/crawler/Balancer.java @@ -413,7 +413,7 @@ public class Balancer { Latency.updateAfterSelection(crawlEntry.url(), profileEntry == null ? 0 : robotsTime); if (delay && sleeptime > 0) { // force a busy waiting here - // in best case, this should never happen if the balancer works propertly + // in best case, this should never happen if the balancer works properly // this is only to protection against the worst case, where the crawler could // behave in a DoS-manner ConcurrentLog.info("BALANCER", "forcing crawl-delay of " + sleeptime + " milliseconds for " + crawlEntry.url().getHost() + ": " + Latency.waitingRemainingExplain(crawlEntry.url(), robots, agent) + ", domainStacks.size() = " + this.domainStacks.size() + ", domainStacksInitSize = " + this.domStackInitSize); diff --git a/source/net/yacy/crawler/data/CrawlQueues.java b/source/net/yacy/crawler/data/CrawlQueues.java index 3baf9d61a..af4c20c06 100644 --- a/source/net/yacy/crawler/data/CrawlQueues.java +++ b/source/net/yacy/crawler/data/CrawlQueues.java @@ -59,7 +59,6 @@ import net.yacy.search.IndexingQueueEntry; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.ErrorCache; -import net.yacy.search.schema.CollectionConfiguration; public class CrawlQueues { @@ -181,10 +180,6 @@ public class CrawlQueues { if (u != null) { return u; } - CollectionConfiguration.FailDoc ee = this.errorURL.get(ASCII.String(urlhash)); - if (ee != null) { - return ee.getDigestURL(); - } for (final Loader w: this.workers.values()) { if (Base64Order.enhancedCoder.equal(w.request.url().hash(), urlhash)) { return w.request.url(); diff --git a/source/net/yacy/search/index/ErrorCache.java b/source/net/yacy/search/index/ErrorCache.java index b181765c4..9a69c37e8 100644 --- a/source/net/yacy/search/index/ErrorCache.java +++ b/source/net/yacy/search/index/ErrorCache.java @@ -166,7 +166,9 @@ public class ErrorCache { } if (failDoc != null) return failDoc; try { - SolrDocument doc = this.fulltext.getDefaultConnector().getDocumentById(urlhash); + final SolrDocumentList docs = this.fulltext.getDefaultConnector().getDocumentListByQuery(CollectionSchema.id + ":\"" + urlhash + "\" AND " + CollectionSchema.failtype_s.getSolrFieldName() + ":[* TO *]", 0, 1); + if (docs == null || docs.isEmpty()) return null; + SolrDocument doc = docs.get(0); if (doc == null) return null; return new CollectionConfiguration.FailDoc(doc); } catch (final IOException e) {