|
|
|
|
@ -30,9 +30,6 @@
|
|
|
|
|
// if the shell's current path is HTROOT
|
|
|
|
|
|
|
|
|
|
import java.io.File;
|
|
|
|
|
import java.io.FileWriter;
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.io.PrintWriter;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Collection;
|
|
|
|
|
import java.util.HashMap;
|
|
|
|
|
@ -52,7 +49,6 @@ import net.yacy.repository.Blacklist;
|
|
|
|
|
import net.yacy.repository.Blacklist.BlacklistError;
|
|
|
|
|
import net.yacy.repository.Blacklist.BlacklistType;
|
|
|
|
|
import net.yacy.search.Switchboard;
|
|
|
|
|
import net.yacy.search.SwitchboardConstants;
|
|
|
|
|
import net.yacy.search.query.SearchEventCache;
|
|
|
|
|
import net.yacy.server.serverObjects;
|
|
|
|
|
import net.yacy.server.serverSwitch;
|
|
|
|
|
@ -68,12 +64,9 @@ public class BlacklistCleaner_p {
|
|
|
|
|
Blacklist.class
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
|
|
|
|
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
|
|
|
|
final serverObjects prop = new serverObjects();
|
|
|
|
|
|
|
|
|
|
// initialize the list manager
|
|
|
|
|
ListManager.switchboard = (Switchboard) env;
|
|
|
|
|
ListManager.listsPath = new File(env.getDataPath(), env.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT));
|
|
|
|
|
String blacklistToUse = null;
|
|
|
|
|
|
|
|
|
|
prop.put(DISABLED+"checked", "1");
|
|
|
|
|
@ -272,26 +265,13 @@ public class BlacklistCleaner_p {
|
|
|
|
|
* @return Length of the list of entries to be removed.
|
|
|
|
|
*/
|
|
|
|
|
private static int removeEntries(final String blacklistToUse, final BlacklistType[] supportedBlacklistTypes, final String[] entries) {
|
|
|
|
|
// load blacklist data from file
|
|
|
|
|
final List<String> list = FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse));
|
|
|
|
|
|
|
|
|
|
boolean listChanged = false;
|
|
|
|
|
|
|
|
|
|
// delete the old entry from file
|
|
|
|
|
for (final String entry : entries) {
|
|
|
|
|
String s = entry;
|
|
|
|
|
|
|
|
|
|
if (list != null){
|
|
|
|
|
|
|
|
|
|
// get rid of escape characters which make it impossible to
|
|
|
|
|
// properly use contains()
|
|
|
|
|
if (s.contains("\\\\")) {
|
|
|
|
|
s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (list.contains(s)) {
|
|
|
|
|
listChanged = list.remove(s);
|
|
|
|
|
}
|
|
|
|
|
// get rid of escape characters which make it impossible to
|
|
|
|
|
// properly use contains()
|
|
|
|
|
if (s.contains("\\\\")) {
|
|
|
|
|
s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// remove the entry from the running blacklist engine
|
|
|
|
|
@ -300,7 +280,7 @@ public class BlacklistCleaner_p {
|
|
|
|
|
final String host = (s.indexOf('/',0) == -1) ? s : s.substring(0, s.indexOf('/',0));
|
|
|
|
|
final String path = (s.indexOf('/',0) == -1) ? ".*" : s.substring(s.indexOf('/',0) + 1);
|
|
|
|
|
try {
|
|
|
|
|
Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path);
|
|
|
|
|
Switchboard.urlBlacklist.remove(supportedBlacklistType, blacklistToUse, host, path);
|
|
|
|
|
} catch (final RuntimeException e) {
|
|
|
|
|
ConcurrentLog.severe("BLACKLIST-CLEANER", e.getMessage() + ": " + host + "/" + path);
|
|
|
|
|
}
|
|
|
|
|
@ -308,9 +288,6 @@ public class BlacklistCleaner_p {
|
|
|
|
|
}
|
|
|
|
|
SearchEventCache.cleanupEvents(true);
|
|
|
|
|
}
|
|
|
|
|
if (listChanged){
|
|
|
|
|
FileUtils.writeList(new File(ListManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
|
|
|
|
|
}
|
|
|
|
|
return entries.length;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -328,34 +305,23 @@ public class BlacklistCleaner_p {
|
|
|
|
|
final String[] oldEntry,
|
|
|
|
|
final String[] newEntry) {
|
|
|
|
|
removeEntries(blacklistToUse, supportedBlacklistTypes, oldEntry);
|
|
|
|
|
PrintWriter pw = null;
|
|
|
|
|
try {
|
|
|
|
|
pw = new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklistToUse), true));
|
|
|
|
|
String host, path;
|
|
|
|
|
for (final String n : newEntry) {
|
|
|
|
|
final int pos = n.indexOf('/',0);
|
|
|
|
|
if (pos < 0) {
|
|
|
|
|
host = n;
|
|
|
|
|
path = ".*";
|
|
|
|
|
} else {
|
|
|
|
|
host = n.substring(0, pos);
|
|
|
|
|
path = n.substring(pos + 1);
|
|
|
|
|
}
|
|
|
|
|
pw.println(host + "/" + path);
|
|
|
|
|
for (final BlacklistType s : supportedBlacklistTypes) {
|
|
|
|
|
if (ListManager.listSetContains(s + ".BlackLists",blacklistToUse)) {
|
|
|
|
|
Switchboard.urlBlacklist.add(
|
|
|
|
|
s,
|
|
|
|
|
host,
|
|
|
|
|
path);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
SearchEventCache.cleanupEvents(true);
|
|
|
|
|
}
|
|
|
|
|
pw.close();
|
|
|
|
|
} catch (final IOException e) {
|
|
|
|
|
ConcurrentLog.severe("BLACKLIST-CLEANER", "error on writing altered entries to blacklist", e);
|
|
|
|
|
}
|
|
|
|
|
String host, path;
|
|
|
|
|
for (final String n : newEntry) {
|
|
|
|
|
final int pos = n.indexOf('/',0);
|
|
|
|
|
if (pos < 0) {
|
|
|
|
|
host = n;
|
|
|
|
|
path = ".*";
|
|
|
|
|
} else {
|
|
|
|
|
host = n.substring(0, pos);
|
|
|
|
|
path = n.substring(pos + 1);
|
|
|
|
|
}
|
|
|
|
|
for (final BlacklistType s : supportedBlacklistTypes) {
|
|
|
|
|
if (ListManager.listSetContains(s + ".BlackLists",blacklistToUse)) {
|
|
|
|
|
Switchboard.urlBlacklist.add(s, blacklistToUse, host, path);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
SearchEventCache.cleanupEvents(true);
|
|
|
|
|
}
|
|
|
|
|
return newEntry.length;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
sixcooler
11 years ago