From aef9dd0350656dec42b57dc826e5707c956fcba6 Mon Sep 17 00:00:00 2001 From: Roland 'Quix0r' Haeder Date: Tue, 10 Jul 2012 13:08:16 +0200 Subject: [PATCH] - removed cleaning of blacklist cache on startup - added cleaning of blacklist cache if cache is modified in interface - extended cache saving to all cache types - moved cache location to DATA/LISTS - fixed static file path which was relative to the application path but should be relative to data path - which is different in debian and mac implementations --- bin/localsearch.sh | 4 +- htroot/Blacklist_p.java | 11 ++++- source/de/anomic/data/ListManager.java | 1 - source/net/yacy/repository/Blacklist.java | 59 ++++++++++++----------- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/bin/localsearch.sh b/bin/localsearch.sh index 50b737616..8acf586ba 100755 --- a/bin/localsearch.sh +++ b/bin/localsearch.sh @@ -2,9 +2,9 @@ cd "`dirname $0`" port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2) if which curl &>/dev/null; then - curl -s "http://localhost:$port/yacysearch.rss?resource=local&verify=ifexist&query=$1" | awk '/^/{ gsub("","" );gsub("<\/link>","" ); print $0 }' + curl -s "http://localhost:$port/yacysearch.rss?resource=local&verify=false&query=$1" | awk '/^/{ gsub("","" );gsub("<\/link>","" ); print $0 }' elif which wget &>/dev/null; then - wget -q -O - "http://localhost:$port/yacysearch.rss?resource=local&verify=ifexist&query=$1" | awk '/^/{ gsub("","" );gsub("<\/link>","" ); print $0 }' + wget -q -O - "http://localhost:$port/yacysearch.rss?resource=local&verify=false&query=$1" | awk '/^/{ gsub("","" );gsub("<\/link>","" ); print $0 }' else echo "Neither curl nor wget installed!" exit 1 diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index 5d0ada5fc..8d2d130ad 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -216,7 +216,8 @@ public class Blacklist_p { ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse); } } - + + Switchboard.urlBlacklist.clear(); ListManager.reloadBlacklists(); } else if (post.containsKey("shareList")) { @@ -256,6 +257,8 @@ public class Blacklist_p { } } } + + Switchboard.urlBlacklist.clear(); ListManager.reloadBlacklists(); } else if (post.containsKey("addBlacklistEntry")) { @@ -275,6 +278,8 @@ public class Blacklist_p { prop.put("LOCATION", temp); return prop; } + + Switchboard.urlBlacklist.clear(); ListManager.reloadBlacklists(); } else if ("moveBlacklistEntry".equals(action)) { @@ -307,6 +312,8 @@ public class Blacklist_p { } } } + + Switchboard.urlBlacklist.clear(); ListManager.reloadBlacklists(); } else if ("editBlacklistEntry".equals(action)) { @@ -346,6 +353,8 @@ public class Blacklist_p { } } } + + Switchboard.urlBlacklist.clear(); ListManager.reloadBlacklists(); prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse); diff --git a/source/de/anomic/data/ListManager.java b/source/de/anomic/data/ListManager.java index 051580a8c..46a9e7a8b 100644 --- a/source/de/anomic/data/ListManager.java +++ b/source/de/anomic/data/ListManager.java @@ -207,7 +207,6 @@ public class ListManager { blacklistFiles.add(blFile); } - Switchboard.urlBlacklist.clear(); Switchboard.urlBlacklist.loadList( blacklistFiles.toArray(new BlacklistFile[blacklistFiles.size()]), "/"); diff --git a/source/net/yacy/repository/Blacklist.java b/source/net/yacy/repository/Blacklist.java index b4934e030..750ab6abf 100644 --- a/source/net/yacy/repository/Blacklist.java +++ b/source/net/yacy/repository/Blacklist.java @@ -51,11 +51,10 @@ import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.SetTools; +import net.yacy.search.Switchboard; public class Blacklist { - - private static final File BLACKLIST_DHT_CACHEFILE = new File("DATA/WORK/blacklistCache_DHT.ser"); - + public enum BlacklistType { DHT, CRAWLER, PROXY, SEARCH, SURFTIPS, NEWS; @@ -94,28 +93,22 @@ public class Blacklist { private File blacklistRootPath = null; private final ConcurrentMap cachedUrlHashs; - private final ConcurrentMap>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here - private final ConcurrentMap>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here - + private final ConcurrentMap>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here + private final ConcurrentMap>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here public Blacklist(final File rootPath) { setRootPath(rootPath); // prepare the data structure - this.hostpaths_matchable = new ConcurrentHashMap>>(); - this.hostpaths_notmatchable = new ConcurrentHashMap>>(); + this.hostpaths_matchable = new ConcurrentHashMap>>(); + this.hostpaths_notmatchable = new ConcurrentHashMap>>(); this.cachedUrlHashs = new ConcurrentHashMap(); for (final BlacklistType blacklistType : BlacklistType.values()) { this.hostpaths_matchable.put(blacklistType, new ConcurrentHashMap>()); this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap>()); - this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap>()); - if (blacklistType.equals(BlacklistType.DHT)) { - loadDHTCache(); - } else { - this.cachedUrlHashs.put(blacklistType, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0)); - } + loadDHTCache(blacklistType); } } @@ -127,8 +120,10 @@ public class Blacklist { public synchronized void close() { Log.logFine("Blacklist", "Shutting down blacklists ..."); - // Save DHT cache - saveDHTCache(); + // Save cache + for (final BlacklistType blacklistType : BlacklistType.values()) { + saveDHTCache(blacklistType); + } Log.logFine("Blacklist", "All blacklists has been shutdown."); } @@ -147,7 +142,7 @@ public class Blacklist { this.blacklistRootPath = rootPath; } - protected ConcurrentMap> getBlacklistMap(final BlacklistType blacklistType, final boolean matchable) { + protected Map> getBlacklistMap(final BlacklistType blacklistType, final boolean matchable) { return (matchable) ? this.hostpaths_matchable.get(blacklistType) : this.hostpaths_notmatchable.get(blacklistType); } @@ -156,7 +151,7 @@ public class Blacklist { } public void clear() { - for (final ConcurrentMap> entry : this.hostpaths_matchable.values()) { + for (final Map> entry : this.hostpaths_matchable.values()) { entry.clear(); } for (final Map> entry : this.hostpaths_notmatchable.values()) { @@ -195,8 +190,8 @@ public class Blacklist { * @param sep */ private void loadList(final BlacklistFile blFile, final String sep) { - final ConcurrentMap> blacklistMapMatch = getBlacklistMap(blFile.getType(), true); - final ConcurrentMap> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false); + final Map> blacklistMapMatch = getBlacklistMap(blFile.getType(), true); + final Map> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false); Set>> loadedBlacklist; Map.Entry> loadedEntry; List paths; @@ -528,10 +523,15 @@ public class Blacklist { return blacklist != null && blacklist.contains(newEntry); } - public final void saveDHTCache() { + private static File DHTCacheFile(BlacklistType type) { + String BLACKLIST_DHT_CACHEFILE_NAME = "DATA/LISTS/blacklist_" + type.name() + "_Cache.ser"; + return new File(Switchboard.getSwitchboard().dataPath, BLACKLIST_DHT_CACHEFILE_NAME); + } + + private final void saveDHTCache(BlacklistType type) { try { - final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(BLACKLIST_DHT_CACHEFILE)); - out.writeObject(getCacheUrlHashsSet(BlacklistType.DHT)); + final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(DHTCacheFile(type))); + out.writeObject(getCacheUrlHashsSet(type)); out.close(); } catch (final IOException e) { @@ -539,13 +539,15 @@ public class Blacklist { } } - public final void loadDHTCache() { + private final void loadDHTCache(BlacklistType type) { try { - if (BLACKLIST_DHT_CACHEFILE.exists()) { - final ObjectInputStream in = new ObjectInputStream(new FileInputStream(BLACKLIST_DHT_CACHEFILE)); - this.cachedUrlHashs.put(BlacklistType.DHT, (HandleSet) in.readObject()); + File cachefile = DHTCacheFile(type); + if (cachefile.exists()) { + final ObjectInputStream in = new ObjectInputStream(new FileInputStream(cachefile)); + this.cachedUrlHashs.put(type, (HandleSet) in.readObject()); in.close(); - return; + } else { + this.cachedUrlHashs.put(type, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0)); } } catch (final ClassNotFoundException e) { Log.logException(e); @@ -554,6 +556,5 @@ public class Blacklist { } catch (final IOException e) { Log.logException(e); } - this.cachedUrlHashs.put(BlacklistType.DHT, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0)); } }