From 4e29e9712a87093d9f0c26d7582607615cf9002b Mon Sep 17 00:00:00 2001
From: f1ori <f1ori@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Mon, 17 Jan 2011 15:04:00 +0000
Subject: [PATCH] * create cleanupjob for cached failed urls

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7437 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 defaults/yacy.init                       |  2 ++
 source/de/anomic/data/WorkTables.java    | 21 +++++++++++++++++++++
 source/de/anomic/search/Switchboard.java |  3 +++
 3 files changed, 26 insertions(+)

diff --git a/defaults/yacy.init b/defaults/yacy.init
index bcdec2847..343a5dbf2 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -588,6 +588,8 @@ performanceIO=10
 # properties for tasks that are performed during cleanup
 cleanup.deletionProcessedNews = true
 cleanup.deletionPublishedNews = true
+cleanup.failedSearchURLtimeout = 86400000
+
 
 # default memory settings for startup of yacy
 # is valid in unix/shell and windows environments but
diff --git a/source/de/anomic/data/WorkTables.java b/source/de/anomic/data/WorkTables.java
index c63982a49..7b373b037 100644
--- a/source/de/anomic/data/WorkTables.java
+++ b/source/de/anomic/data/WorkTables.java
@@ -329,6 +329,27 @@ public class WorkTables extends Tables {
         }
     }
     
+    /**
+     * cleanup cached failed searchs older then timeout
+     */
+    public void cleanFailURLS(long timeout) {
+    	if (timeout >= 0) {
+    		try {
+				Iterator<Row> iter = this.iterator(WorkTables.TABLE_SEARCH_FAILURE_NAME);
+				while (iter.hasNext()) {
+					Row row = iter.next();
+					Date date = new Date();
+					date = row.get(TABLE_SEARCH_FAILURE_COL_DATE, date);
+					if(date.before(new Date(System.currentTimeMillis() - timeout))) {
+						this.delete(TABLE_SEARCH_FAILURE_NAME, row.getPK());
+					}
+				}
+			} catch (IOException e) {
+	            Log.logException(e);
+			}
+    	}
+    }
+    
     public static Map<byte[], String> commentCache(Switchboard sb) {
         Map<byte[], String> comments = new TreeMap<byte[], String>(Base64Order.enhancedCoder);
         Iterator<Tables.Row> i;
diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java
index c72ffbc12..f30b3cca6 100644
--- a/source/de/anomic/search/Switchboard.java
+++ b/source/de/anomic/search/Switchboard.java
@@ -1662,6 +1662,9 @@ public final class Switchboard extends serverSwitch {
             // after all clean up is done, check the resource usage
             observer.resourceObserverJob();
             
+            // cleanup cached search failures
+            this.tables.cleanFailURLS(this.getConfigLong("cleanup.failedSearchURLtimeout", -1));
+            
             return true;
         } catch (final InterruptedException e) {
             this.log.logInfo("cleanupJob: Shutdown detected");