diff --git a/defaults/yacy.init b/defaults/yacy.init index bcdec2847..343a5dbf2 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -588,6 +588,8 @@ performanceIO=10 # properties for tasks that are performed during cleanup cleanup.deletionProcessedNews = true cleanup.deletionPublishedNews = true +cleanup.failedSearchURLtimeout = 86400000 + # default memory settings for startup of yacy # is valid in unix/shell and windows environments but diff --git a/source/de/anomic/data/WorkTables.java b/source/de/anomic/data/WorkTables.java index c63982a49..7b373b037 100644 --- a/source/de/anomic/data/WorkTables.java +++ b/source/de/anomic/data/WorkTables.java @@ -329,6 +329,27 @@ public class WorkTables extends Tables { } } + /** + * cleanup cached failed searchs older then timeout + */ + public void cleanFailURLS(long timeout) { + if (timeout >= 0) { + try { + Iterator iter = this.iterator(WorkTables.TABLE_SEARCH_FAILURE_NAME); + while (iter.hasNext()) { + Row row = iter.next(); + Date date = new Date(); + date = row.get(TABLE_SEARCH_FAILURE_COL_DATE, date); + if(date.before(new Date(System.currentTimeMillis() - timeout))) { + this.delete(TABLE_SEARCH_FAILURE_NAME, row.getPK()); + } + } + } catch (IOException e) { + Log.logException(e); + } + } + } + public static Map commentCache(Switchboard sb) { Map comments = new TreeMap(Base64Order.enhancedCoder); Iterator i; diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index c72ffbc12..f30b3cca6 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -1662,6 +1662,9 @@ public final class Switchboard extends serverSwitch { // after all clean up is done, check the resource usage observer.resourceObserverJob(); + // cleanup cached search failures + this.tables.cleanFailURLS(this.getConfigLong("cleanup.failedSearchURLtimeout", -1)); + return true; } catch (final InterruptedException e) { this.log.logInfo("cleanupJob: Shutdown detected");