From 49b79987c91c2f4734384d8f68a806e5a5370dbb Mon Sep 17 00:00:00 2001 From: reger Date: Thu, 4 Jun 2015 22:44:01 +0200 Subject: [PATCH] remove obsolete searchfl work table was used to register urls with not complete words in snippet but is never accessed --- htroot/ConfigHTCache_p.html | 4 +- htroot/ConfigHTCache_p.java | 3 -- htroot/IndexControlURLs_p.html | 3 +- htroot/IndexControlURLs_p.java | 3 -- source/net/yacy/data/WorkTables.java | 52 -------------------- source/net/yacy/search/ResourceObserver.java | 7 +-- source/net/yacy/search/Switchboard.java | 6 --- 7 files changed, 4 insertions(+), 74 deletions(-) diff --git a/htroot/ConfigHTCache_p.html b/htroot/ConfigHTCache_p.html index 563e60c53..2354a52f9 100644 --- a/htroot/ConfigHTCache_p.html +++ b/htroot/ConfigHTCache_p.html @@ -33,10 +33,10 @@
Cache Deletion
Delete HTTP & FTP Cache
Delete robots.txt Cache
- Delete cached snippet-fetching failures during search


+

diff --git a/htroot/ConfigHTCache_p.java b/htroot/ConfigHTCache_p.java index 133ae4b74..aa5c0cec3 100644 --- a/htroot/ConfigHTCache_p.java +++ b/htroot/ConfigHTCache_p.java @@ -71,9 +71,6 @@ public class ConfigHTCache_p { if ("on".equals(post.get("deleteRobots", ""))) { sb.robots.clear(); } - if ("on".equals(post.get("deleteSearchFl", ""))) { - sb.tables.clear(WorkTables.TABLE_SEARCH_FAILURE_NAME); - } } prop.put("HTCachePath", env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT)); diff --git a/htroot/IndexControlURLs_p.html b/htroot/IndexControlURLs_p.html index 0d245a6ab..c0be6f055 100644 --- a/htroot/IndexControlURLs_p.html +++ b/htroot/IndexControlURLs_p.html @@ -98,7 +98,7 @@ function updatepage(str) {
Index Deletion
Delete local search index (embedded Solr and old Metadata)
#(cleanupsolr)#:: Delete remote solr index
#(/cleanupsolr)# #(cleanuprwi)#:: Delete RWI Index (DHT transmission words)
#(/cleanuprwi)# @@ -107,7 +107,6 @@ function updatepage(str) { Delete HTTP & FTP Cache
Stop Crawler and delete Crawl Queues
Delete robots.txt Cache
- Delete cached snippet-fetching failures during search

diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index bf35bda5f..d74cf0212 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -174,9 +174,6 @@ public class IndexControlURLs_p { if ( post.get("deleteRobots", "").equals("on") ) { try {sb.robots.clear();} catch (final IOException e) {} } - if ( post.get("deleteSearchFl", "").equals("on") ) { - sb.tables.clear(WorkTables.TABLE_SEARCH_FAILURE_NAME); - } post.remove("deletecomplete"); } diff --git a/source/net/yacy/data/WorkTables.java b/source/net/yacy/data/WorkTables.java index 21f156e30..fc140a1ea 100644 --- a/source/net/yacy/data/WorkTables.java +++ b/source/net/yacy/data/WorkTables.java @@ -87,12 +87,6 @@ public class WorkTables extends Tables { public final static String TABLE_ACTIVECRAWLS_NAME = "crawljobsActive"; public final static String TABLE_PASSIVECRAWLS_NAME = "crawljobsPassive"; - public final static String TABLE_SEARCH_FAILURE_NAME = "searchfl"; - public final static String TABLE_SEARCH_FAILURE_COL_URL = "url"; - public final static String TABLE_SEARCH_FAILURE_COL_DATE = "date"; - public final static String TABLE_SEARCH_FAILURE_COL_WORDS = "words"; - public final static String TABLE_SEARCH_FAILURE_COL_COMMENT = "comment"; - public YMarkTables bookmarks; public WorkTables(final File workPath) { @@ -353,52 +347,6 @@ public class WorkTables extends Tables { indexCell.removeDelayed(word, url.hash()); } } - - // insert information about changed url into database - try { - // create and insert new entry - Data data = new Data(); - byte[] date = UTF8.getBytes(GenericFormatter.SHORT_MILSEC_FORMATTER.format()); - data.put(TABLE_SEARCH_FAILURE_COL_URL, url.toNormalform(true)); - data.put(TABLE_SEARCH_FAILURE_COL_DATE, date); - data.put(TABLE_SEARCH_FAILURE_COL_WORDS, queryHashes.export()); - data.put(TABLE_SEARCH_FAILURE_COL_COMMENT, UTF8.getBytes(reason)); - super.insert(TABLE_SEARCH_FAILURE_NAME, url.hash(), data); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } - - public boolean failURLsContains(byte[] urlhash) { - try { - return super.has(TABLE_SEARCH_FAILURE_NAME, urlhash); - } catch (final IOException e) { - ConcurrentLog.logException(e); - return false; - } - } - - /** - * cleanup cached failed searchs older then timeout - */ - public void cleanFailURLS(long timeout) { - if (timeout >= 0) { - try { - Row row; - Date date; - Iterator iter = this.iterator(WorkTables.TABLE_SEARCH_FAILURE_NAME); - while (iter.hasNext()) { - row = iter.next(); - date = new Date(); - date = row.get(TABLE_SEARCH_FAILURE_COL_DATE, date); - if(date.before(new Date(System.currentTimeMillis() - timeout))) { - this.delete(TABLE_SEARCH_FAILURE_NAME, row.getPK()); - } - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } } public static Map commentCache(Switchboard sb) { diff --git a/source/net/yacy/search/ResourceObserver.java b/source/net/yacy/search/ResourceObserver.java index df0daa4e1..0bc27cba4 100644 --- a/source/net/yacy/search/ResourceObserver.java +++ b/source/net/yacy/search/ResourceObserver.java @@ -114,12 +114,7 @@ public class ResourceObserver { shrinkmethods: while (true /*this is not a loop, just a construct that we can leave with a break*/) { // delete old releases //if (getNormalizedDiskFree() == Space.AMPLE && getNormalizedDiskUsed(false) == Space.AMPLE) break; - - // delete fetched snippets - log.info("DISK SPACE EXHAUSTED - deleting snippet cache"); - sb.tables.clear(WorkTables.TABLE_SEARCH_FAILURE_NAME); - if (getNormalizedDiskFree() == Space.AMPLE && getNormalizedDiskUsed(false) == Space.AMPLE) break; - + // clear HTCACHE log.info("DISK SPACE EXHAUSTED - deleting HTCACHE"); Cache.clear(); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 4d9464ed0..38bf5fbae 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2408,12 +2408,6 @@ public final class Switchboard extends serverSwitch { // after all clean up is done, check the resource usage this.observer.resourceObserverJob(); - - // cleanup cached search failures - if ( getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) - && this.peers.mySeed().getFlagAcceptRemoteIndex() ) { - this.tables.cleanFailURLS(getConfigLong("cleanup.failedSearchURLtimeout", -1)); - } // clean up profiles checkInterruption();