* create cleanupjob for cached failed urls

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7437 6c8d7289-2bf4-0310-a012-ef5d649a1542

defaults/yacy.init

@@ -588,6 +588,8 @@ performanceIO=10
 # properties for tasks that are performed during cleanup
 cleanup.deletionProcessedNews = true
 cleanup.deletionPublishedNews = true
+cleanup.failedSearchURLtimeout = 86400000
+
 
 # default memory settings for startup of yacy
 # is valid in unix/shell and windows environments but

source/de/anomic/data/WorkTables.java

@@ -329,6 +329,27 @@ public class WorkTables extends Tables {
         }
     }
     
+    /**
+     * cleanup cached failed searchs older then timeout
+     */
+    public void cleanFailURLS(long timeout) {
+    	if (timeout >= 0) {
+    		try {
+				Iterator<Row> iter = this.iterator(WorkTables.TABLE_SEARCH_FAILURE_NAME);
+				while (iter.hasNext()) {
+					Row row = iter.next();
+					Date date = new Date();
+					date = row.get(TABLE_SEARCH_FAILURE_COL_DATE, date);
+					if(date.before(new Date(System.currentTimeMillis() - timeout))) {
+						this.delete(TABLE_SEARCH_FAILURE_NAME, row.getPK());
+					}
+				}
+			} catch (IOException e) {
+	            Log.logException(e);
+			}
+    	}
+    }
+    
     public static Map<byte[], String> commentCache(Switchboard sb) {
         Map<byte[], String> comments = new TreeMap<byte[], String>(Base64Order.enhancedCoder);
         Iterator<Tables.Row> i;

source/de/anomic/search/Switchboard.java

@@ -1662,6 +1662,9 @@ public final class Switchboard extends serverSwitch {
             // after all clean up is done, check the resource usage
             observer.resourceObserverJob();
             
+            // cleanup cached search failures
+            this.tables.cleanFailURLS(this.getConfigLong("cleanup.failedSearchURLtimeout", -1));
+            
             return true;
         } catch (final InterruptedException e) {
             this.log.logInfo("cleanupJob: Shutdown detected");