|
|
@ -106,12 +106,25 @@ public class ErrorCache {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
|
|
* Adds a error document to the Solr index (marked as failed by httpstatus_i <> 200)
|
|
|
|
|
|
|
|
* and caches recently added failed docs (up to maxStackSize = 1000)
|
|
|
|
|
|
|
|
*
|
|
|
|
|
|
|
|
* @param url failed url
|
|
|
|
|
|
|
|
* @param crawldepth info crawldepth
|
|
|
|
|
|
|
|
* @param profile info of collection
|
|
|
|
|
|
|
|
* @param failCategory .store to index otherwise cache only
|
|
|
|
|
|
|
|
* @param anycause info cause-string
|
|
|
|
|
|
|
|
* @param httpcode http response code
|
|
|
|
|
|
|
|
*/
|
|
|
|
public void push(final DigestURL url, final int crawldepth, final CrawlProfile profile, final FailCategory failCategory, String anycause, final int httpcode) {
|
|
|
|
public void push(final DigestURL url, final int crawldepth, final CrawlProfile profile, final FailCategory failCategory, String anycause, final int httpcode) {
|
|
|
|
// assert executor != null; // null == proxy !
|
|
|
|
// assert executor != null; // null == proxy !
|
|
|
|
assert failCategory.store || httpcode == -1 : "failCategory=" + failCategory.name();
|
|
|
|
assert failCategory.store || httpcode == -1 : "failCategory=" + failCategory.name();
|
|
|
|
if (anycause == null) anycause = "unknown";
|
|
|
|
if (anycause == null) anycause = "unknown";
|
|
|
|
final String reason = anycause + ((httpcode >= 0) ? " (http return code = " + httpcode + ")" : "");
|
|
|
|
final String reason = anycause + ((httpcode >= 0) ? " (http return code = " + httpcode + ")" : "");
|
|
|
|
if (!reason.startsWith("double")) log.info(url.toNormalform(true) + " - " + reason);
|
|
|
|
if (!reason.startsWith("double")) log.info(url.toNormalform(true) + " - " + reason);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (!this.cache.containsKey(ASCII.String(url.hash()))) { // no further action if in error-cache
|
|
|
|
CollectionConfiguration.FailDoc failDoc = new CollectionConfiguration.FailDoc(
|
|
|
|
CollectionConfiguration.FailDoc failDoc = new CollectionConfiguration.FailDoc(
|
|
|
|
url, profile == null ? null : profile.collections(),
|
|
|
|
url, profile == null ? null : profile.collections(),
|
|
|
|
failCategory.name() + " " + reason, failCategory.failType,
|
|
|
|
failCategory.name() + " " + reason, failCategory.failType,
|
|
|
@ -136,6 +149,7 @@ public class ErrorCache {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
checkStackSize();
|
|
|
|
checkStackSize();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private void checkStackSize() {
|
|
|
|
private void checkStackSize() {
|
|
|
|
synchronized (this.cache) {
|
|
|
|
synchronized (this.cache) {
|
|
|
|