boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- removed cleaning of blacklist cache on startup

Browse Source 
- added cleaning of blacklist cache if cache is modified in interface
- extended cache saving to all cache types
- moved cache location to DATA/LISTS
- fixed static file path which was relative to the application path but
should be relative to data path - which is different in debian and mac
implementations
pull/1/head
Roland 'Quix0r' Haeder 13 years ago committed by orbiter
parent c7afa8bc48
commit aef9dd0350
4 changed files with 42 additions and 33 deletions
Show Stats Download Patch File Download Diff File

4
bin/localsearch.sh
Unescape View File

@ -2,9 +2,9 @@
cd "`dirname $0`" cd "`dirname $0`"
port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2) port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
if which curl &>/dev/null; then if which curl &>/dev/null; then
curl -s "http://localhost:$port/yacysearch.rss?resource=local&verify=ifexist&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }' curl -s "http://localhost:$port/yacysearch.rss?resource=local&verify=false&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
elif which wget &>/dev/null; then elif which wget &>/dev/null; then
wget -q -O - "http://localhost:$port/yacysearch.rss?resource=local&verify=ifexist&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }' wget -q -O - "http://localhost:$port/yacysearch.rss?resource=local&verify=false&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
else else
echo "Neither curl nor wget installed!" echo "Neither curl nor wget installed!"
exit 1 exit 1

11
htroot/Blacklist_p.java
Unescape View File

@ -216,7 +216,8 @@ public class Blacklist_p {
ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse); ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse);
} }
} }
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists(); ListManager.reloadBlacklists();
} else if (post.containsKey("shareList")) { } else if (post.containsKey("shareList")) {
@ -256,6 +257,8 @@ public class Blacklist_p {
} }
} }
} }
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists(); ListManager.reloadBlacklists();
} else if (post.containsKey("addBlacklistEntry")) { } else if (post.containsKey("addBlacklistEntry")) {
@ -275,6 +278,8 @@ public class Blacklist_p {
prop.put("LOCATION", temp); prop.put("LOCATION", temp);
return prop; return prop;
} }
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists(); ListManager.reloadBlacklists();
} else if ("moveBlacklistEntry".equals(action)) { } else if ("moveBlacklistEntry".equals(action)) {
@ -307,6 +312,8 @@ public class Blacklist_p {
} }
} }
} }
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists(); ListManager.reloadBlacklists();
} else if ("editBlacklistEntry".equals(action)) { } else if ("editBlacklistEntry".equals(action)) {
@ -346,6 +353,8 @@ public class Blacklist_p {
} }
} }
} }
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists(); ListManager.reloadBlacklists();
prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse); prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse);

1
source/de/anomic/data/ListManager.java
Unescape View File

@ -207,7 +207,6 @@ public class ListManager {
blacklistFiles.add(blFile); blacklistFiles.add(blFile);
} }
Switchboard.urlBlacklist.clear();
Switchboard.urlBlacklist.loadList( Switchboard.urlBlacklist.loadList(
blacklistFiles.toArray(new BlacklistFile[blacklistFiles.size()]), blacklistFiles.toArray(new BlacklistFile[blacklistFiles.size()]),
"/"); "/");

59
source/net/yacy/repository/Blacklist.java
Unescape View File

@ -51,11 +51,10 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.SetTools; import net.yacy.kelondro.util.SetTools;
import net.yacy.search.Switchboard;
public class Blacklist { public class Blacklist {
private static final File BLACKLIST_DHT_CACHEFILE = new File("DATA/WORK/blacklistCache_DHT.ser");
public enum BlacklistType { public enum BlacklistType {
DHT, CRAWLER, PROXY, SEARCH, SURFTIPS, NEWS; DHT, CRAWLER, PROXY, SEARCH, SURFTIPS, NEWS;
@ -94,28 +93,22 @@ public class Blacklist {
private File blacklistRootPath = null; private File blacklistRootPath = null;
private final ConcurrentMap<BlacklistType, HandleSet> cachedUrlHashs; private final ConcurrentMap<BlacklistType, HandleSet> cachedUrlHashs;
private final ConcurrentMap<BlacklistType, ConcurrentMap<String, List<Pattern>>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here private final ConcurrentMap<BlacklistType, Map<String, List<Pattern>>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
private final ConcurrentMap<BlacklistType, ConcurrentMap<String, List<Pattern>>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here private final ConcurrentMap<BlacklistType, Map<String, List<Pattern>>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
public Blacklist(final File rootPath) { public Blacklist(final File rootPath) {
setRootPath(rootPath); setRootPath(rootPath);
// prepare the data structure // prepare the data structure
this.hostpaths_matchable = new ConcurrentHashMap<BlacklistType, ConcurrentMap<String, List<Pattern>>>(); this.hostpaths_matchable = new ConcurrentHashMap<BlacklistType, Map<String, List<Pattern>>>();
this.hostpaths_notmatchable = new ConcurrentHashMap<BlacklistType, ConcurrentMap<String, List<Pattern>>>(); this.hostpaths_notmatchable = new ConcurrentHashMap<BlacklistType, Map<String, List<Pattern>>>();
this.cachedUrlHashs = new ConcurrentHashMap<BlacklistType, HandleSet>(); this.cachedUrlHashs = new ConcurrentHashMap<BlacklistType, HandleSet>();
for (final BlacklistType blacklistType : BlacklistType.values()) { for (final BlacklistType blacklistType : BlacklistType.values()) {
this.hostpaths_matchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>()); this.hostpaths_matchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>());
this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>()); this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>());
this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>()); loadDHTCache(blacklistType);
if (blacklistType.equals(BlacklistType.DHT)) {
loadDHTCache();
} else {
this.cachedUrlHashs.put(blacklistType, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
}
} }
} }
@ -127,8 +120,10 @@ public class Blacklist {
public synchronized void close() { public synchronized void close() {
Log.logFine("Blacklist", "Shutting down blacklists ..."); Log.logFine("Blacklist", "Shutting down blacklists ...");
// Save DHT cache // Save cache
saveDHTCache(); for (final BlacklistType blacklistType : BlacklistType.values()) {
saveDHTCache(blacklistType);
}
Log.logFine("Blacklist", "All blacklists has been shutdown."); Log.logFine("Blacklist", "All blacklists has been shutdown.");
} }
@ -147,7 +142,7 @@ public class Blacklist {
this.blacklistRootPath = rootPath; this.blacklistRootPath = rootPath;
} }
protected ConcurrentMap<String, List<Pattern>> getBlacklistMap(final BlacklistType blacklistType, final boolean matchable) { protected Map<String, List<Pattern>> getBlacklistMap(final BlacklistType blacklistType, final boolean matchable) {
return (matchable) ? this.hostpaths_matchable.get(blacklistType) : this.hostpaths_notmatchable.get(blacklistType); return (matchable) ? this.hostpaths_matchable.get(blacklistType) : this.hostpaths_notmatchable.get(blacklistType);
} }
@ -156,7 +151,7 @@ public class Blacklist {
} }
public void clear() { public void clear() {
for (final ConcurrentMap<String, List<Pattern>> entry : this.hostpaths_matchable.values()) { for (final Map<String, List<Pattern>> entry : this.hostpaths_matchable.values()) {
entry.clear(); entry.clear();
} }
for (final Map<String, List<Pattern>> entry : this.hostpaths_notmatchable.values()) { for (final Map<String, List<Pattern>> entry : this.hostpaths_notmatchable.values()) {
@ -195,8 +190,8 @@ public class Blacklist {
* @param sep * @param sep
*/ */
private void loadList(final BlacklistFile blFile, final String sep) { private void loadList(final BlacklistFile blFile, final String sep) {
final ConcurrentMap<String, List<Pattern>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true); final Map<String, List<Pattern>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true);
final ConcurrentMap<String, List<Pattern>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false); final Map<String, List<Pattern>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false);
Set<Map.Entry<String, List<String>>> loadedBlacklist; Set<Map.Entry<String, List<String>>> loadedBlacklist;
Map.Entry<String, List<String>> loadedEntry; Map.Entry<String, List<String>> loadedEntry;
List<Pattern> paths; List<Pattern> paths;
@ -528,10 +523,15 @@ public class Blacklist {
return blacklist != null && blacklist.contains(newEntry); return blacklist != null && blacklist.contains(newEntry);
} }
public final void saveDHTCache() { private static File DHTCacheFile(BlacklistType type) {
String BLACKLIST_DHT_CACHEFILE_NAME = "DATA/LISTS/blacklist_" + type.name() + "_Cache.ser";
return new File(Switchboard.getSwitchboard().dataPath, BLACKLIST_DHT_CACHEFILE_NAME);
}
private final void saveDHTCache(BlacklistType type) {
try { try {
final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(BLACKLIST_DHT_CACHEFILE)); final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(DHTCacheFile(type)));
out.writeObject(getCacheUrlHashsSet(BlacklistType.DHT)); out.writeObject(getCacheUrlHashsSet(type));
out.close(); out.close();
} catch (final IOException e) { } catch (final IOException e) {
@ -539,13 +539,15 @@ public class Blacklist {
} }
} }
public final void loadDHTCache() { private final void loadDHTCache(BlacklistType type) {
try { try {
if (BLACKLIST_DHT_CACHEFILE.exists()) { File cachefile = DHTCacheFile(type);
final ObjectInputStream in = new ObjectInputStream(new FileInputStream(BLACKLIST_DHT_CACHEFILE)); if (cachefile.exists()) {
this.cachedUrlHashs.put(BlacklistType.DHT, (HandleSet) in.readObject()); final ObjectInputStream in = new ObjectInputStream(new FileInputStream(cachefile));
this.cachedUrlHashs.put(type, (HandleSet) in.readObject());
in.close(); in.close();
return; } else {
this.cachedUrlHashs.put(type, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
} }
} catch (final ClassNotFoundException e) { } catch (final ClassNotFoundException e) {
Log.logException(e); Log.logException(e);
@ -554,6 +556,5 @@ public class Blacklist {
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
this.cachedUrlHashs.put(BlacklistType.DHT, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
} }
} }

Write Preview
Loading…
Cancel
Save