boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- removed cleaning of blacklist cache on startup

Browse Source 
- added cleaning of blacklist cache if cache is modified in interface
- extended cache saving to all cache types
- moved cache location to DATA/LISTS
- fixed static file path which was relative to the application path but
should be relative to data path - which is different in debian and mac
implementations
pull/1/head
Roland 'Quix0r' Haeder 13 years ago committed by orbiter
parent c7afa8bc48
commit aef9dd0350
4 changed files with 42 additions and 33 deletions
Show Stats Download Patch File Download Diff File

4
bin/localsearch.sh
Unescape View File

@ -2,9 +2,9 @@
cd "`dirname $0`"
port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
if which curl &>/dev/null; then
curl -s "http://localhost:$port/yacysearch.rss?resource=local&verify=ifexist&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
curl -s "http://localhost:$port/yacysearch.rss?resource=local&verify=false&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
elif which wget &>/dev/null; then
wget -q -O - "http://localhost:$port/yacysearch.rss?resource=local&verify=ifexist&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
wget -q -O - "http://localhost:$port/yacysearch.rss?resource=local&verify=false&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
else
echo "Neither curl nor wget installed!"
exit 1

11
htroot/Blacklist_p.java
Unescape View File

@ -216,7 +216,8 @@ public class Blacklist_p {
ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse);
}
}
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
} else if (post.containsKey("shareList")) {
@ -256,6 +257,8 @@ public class Blacklist_p {
}
}
}
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
} else if (post.containsKey("addBlacklistEntry")) {
@ -275,6 +278,8 @@ public class Blacklist_p {
prop.put("LOCATION", temp);
return prop;
}
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
} else if ("moveBlacklistEntry".equals(action)) {
@ -307,6 +312,8 @@ public class Blacklist_p {
}
}
}
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
} else if ("editBlacklistEntry".equals(action)) {
@ -346,6 +353,8 @@ public class Blacklist_p {
}
}
}
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse);

1
source/de/anomic/data/ListManager.java
Unescape View File

@ -207,7 +207,6 @@ public class ListManager {
blacklistFiles.add(blFile);
}
Switchboard.urlBlacklist.clear();
Switchboard.urlBlacklist.loadList(
blacklistFiles.toArray(new BlacklistFile[blacklistFiles.size()]),
"/");

59
source/net/yacy/repository/Blacklist.java
Unescape View File

@ -51,11 +51,10 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.SetTools;
import net.yacy.search.Switchboard;
public class Blacklist {
private static final File BLACKLIST_DHT_CACHEFILE = new File("DATA/WORK/blacklistCache_DHT.ser");
public enum BlacklistType {
DHT, CRAWLER, PROXY, SEARCH, SURFTIPS, NEWS;
@ -94,28 +93,22 @@ public class Blacklist {
private File blacklistRootPath = null;
private final ConcurrentMap<BlacklistType, HandleSet> cachedUrlHashs;
private final ConcurrentMap<BlacklistType, ConcurrentMap<String, List<Pattern>>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
private final ConcurrentMap<BlacklistType, ConcurrentMap<String, List<Pattern>>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
private final ConcurrentMap<BlacklistType, Map<String, List<Pattern>>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
private final ConcurrentMap<BlacklistType, Map<String, List<Pattern>>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
public Blacklist(final File rootPath) {
setRootPath(rootPath);
// prepare the data structure
this.hostpaths_matchable = new ConcurrentHashMap<BlacklistType, ConcurrentMap<String, List<Pattern>>>();
this.hostpaths_notmatchable = new ConcurrentHashMap<BlacklistType, ConcurrentMap<String, List<Pattern>>>();
this.hostpaths_matchable = new ConcurrentHashMap<BlacklistType, Map<String, List<Pattern>>>();
this.hostpaths_notmatchable = new ConcurrentHashMap<BlacklistType, Map<String, List<Pattern>>>();
this.cachedUrlHashs = new ConcurrentHashMap<BlacklistType, HandleSet>();
for (final BlacklistType blacklistType : BlacklistType.values()) {
this.hostpaths_matchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>());
this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>());
this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>());
if (blacklistType.equals(BlacklistType.DHT)) {
loadDHTCache();
} else {
this.cachedUrlHashs.put(blacklistType, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
}
loadDHTCache(blacklistType);
}
}
@ -127,8 +120,10 @@ public class Blacklist {
public synchronized void close() {
Log.logFine("Blacklist", "Shutting down blacklists ...");
// Save DHT cache
saveDHTCache();
// Save cache
for (final BlacklistType blacklistType : BlacklistType.values()) {
saveDHTCache(blacklistType);
}
Log.logFine("Blacklist", "All blacklists has been shutdown.");
}
@ -147,7 +142,7 @@ public class Blacklist {
this.blacklistRootPath = rootPath;
}
protected ConcurrentMap<String, List<Pattern>> getBlacklistMap(final BlacklistType blacklistType, final boolean matchable) {
protected Map<String, List<Pattern>> getBlacklistMap(final BlacklistType blacklistType, final boolean matchable) {
return (matchable) ? this.hostpaths_matchable.get(blacklistType) : this.hostpaths_notmatchable.get(blacklistType);
}
@ -156,7 +151,7 @@ public class Blacklist {
}
public void clear() {
for (final ConcurrentMap<String, List<Pattern>> entry : this.hostpaths_matchable.values()) {
for (final Map<String, List<Pattern>> entry : this.hostpaths_matchable.values()) {
entry.clear();
}
for (final Map<String, List<Pattern>> entry : this.hostpaths_notmatchable.values()) {
@ -195,8 +190,8 @@ public class Blacklist {
* @param sep
*/
private void loadList(final BlacklistFile blFile, final String sep) {
final ConcurrentMap<String, List<Pattern>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true);
final ConcurrentMap<String, List<Pattern>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false);
final Map<String, List<Pattern>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true);
final Map<String, List<Pattern>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false);
Set<Map.Entry<String, List<String>>> loadedBlacklist;
Map.Entry<String, List<String>> loadedEntry;
List<Pattern> paths;
@ -528,10 +523,15 @@ public class Blacklist {
return blacklist != null && blacklist.contains(newEntry);
}
public final void saveDHTCache() {
private static File DHTCacheFile(BlacklistType type) {
String BLACKLIST_DHT_CACHEFILE_NAME = "DATA/LISTS/blacklist_" + type.name() + "_Cache.ser";
return new File(Switchboard.getSwitchboard().dataPath, BLACKLIST_DHT_CACHEFILE_NAME);
}
private final void saveDHTCache(BlacklistType type) {
try {
final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(BLACKLIST_DHT_CACHEFILE));
out.writeObject(getCacheUrlHashsSet(BlacklistType.DHT));
final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(DHTCacheFile(type)));
out.writeObject(getCacheUrlHashsSet(type));
out.close();
} catch (final IOException e) {
@ -539,13 +539,15 @@ public class Blacklist {
}
}
public final void loadDHTCache() {
private final void loadDHTCache(BlacklistType type) {
try {
if (BLACKLIST_DHT_CACHEFILE.exists()) {
final ObjectInputStream in = new ObjectInputStream(new FileInputStream(BLACKLIST_DHT_CACHEFILE));
this.cachedUrlHashs.put(BlacklistType.DHT, (HandleSet) in.readObject());
File cachefile = DHTCacheFile(type);
if (cachefile.exists()) {
final ObjectInputStream in = new ObjectInputStream(new FileInputStream(cachefile));
this.cachedUrlHashs.put(type, (HandleSet) in.readObject());
in.close();
return;
} else {
this.cachedUrlHashs.put(type, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
}
} catch (final ClassNotFoundException e) {
Log.logException(e);
@ -554,6 +556,5 @@ public class Blacklist {
} catch (final IOException e) {
Log.logException(e);
}
this.cachedUrlHashs.put(BlacklistType.DHT, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
}
}

Write Preview
Loading…
Cancel
Save