diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index 327a4d6d5..1e2695436 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -30,9 +30,7 @@ // if the shell's current path is HTROOT import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.net.MalformedURLException; import java.util.Arrays; import java.util.List; @@ -251,7 +249,7 @@ public class Blacklist_p { if (selectedBlacklistEntries.length > 0) { String temp = null; for (final String selectedBlacklistEntry : selectedBlacklistEntries) { - if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) { + if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header)) != null) { prop.put("LOCATION", temp); return prop; } @@ -273,7 +271,7 @@ public class Blacklist_p { // store this call as api call ListManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + blentry); - final String temp = addBlacklistEntry(blacklistToUse, blentry, header, BlacklistType.values()); + final String temp = addBlacklistEntry(blacklistToUse, blentry, header); if (temp != null) { prop.put("LOCATION", temp); return prop; @@ -300,12 +298,12 @@ public class Blacklist_p { !targetBlacklist.equals(blacklistToUse)) { String temp; for (final String selectedBlacklistEntry : selectedBlacklistEntries) { - if ((temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntry, header, BlacklistType.values())) != null) { + if ((temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntry, header)) != null) { prop.put("LOCATION", temp); return prop; } - if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) { + if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header)) != null) { prop.put("LOCATION", temp); return prop; @@ -342,12 +340,12 @@ public class Blacklist_p { if (!selectedBlacklistEntries[i].equals(editedBlacklistEntries[i])) { - if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, BlacklistType.values())) != null) { + if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header)) != null) { prop.put("LOCATION", temp); return prop; } - if ((temp = addBlacklistEntry(blacklistToUse, editedBlacklistEntries[i], header, BlacklistType.values())) != null) { + if ((temp = addBlacklistEntry(blacklistToUse, editedBlacklistEntries[i], header)) != null) { prop.put("LOCATION", temp); return prop; } @@ -519,49 +517,19 @@ public class Blacklist_p { return prop; } - - - /** - * This method adds a new entry to the chosen blacklist. - * @param blacklistToUse the name of the blacklist the entry is to be added to - * @param newEntry the entry that is to be added - * @param header - * @param supportedBlacklistTypes - * @return null if no error occurred, else a String to put into LOCATION - */ - private static String addBlacklistEntry( - final String blacklistToUse, - final String newEntry, - final RequestHeader header, - final BlacklistType[] supportedBlacklistTypes) { - - if (blacklistToUse == null || blacklistToUse.isEmpty()) { - return ""; - } - - if (newEntry == null || newEntry.isEmpty()) { - return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse; - } - - addBlacklistEntry(ListManager.listsPath, blacklistToUse, newEntry, supportedBlacklistTypes); - SearchEventCache.cleanupEvents(true); - return null; - } - - /** * This method deletes a blacklist entry. * @param blacklistToUse the name of the blacklist the entry is to be deleted from - * @param oldEntry the entry that is to be deleted + * @param entry the entry that is to be deleted * @param header * @param supportedBlacklistTypes - * @return null if no error occured, else a String to put into LOCATION + * @return null if no error occurred, else a String to put into LOCATION */ private static String deleteBlacklistEntry( final String blacklistToUse, - final String oldEntry, - final RequestHeader header, - final BlacklistType[] supportedBlacklistTypes) { + final String entry, + final RequestHeader header) { + String oldEntry = entry; if (blacklistToUse == null || blacklistToUse.isEmpty()) { return ""; @@ -571,69 +539,59 @@ public class Blacklist_p { return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse; } - deleteBlacklistEntry(ListManager.listsPath, blacklistToUse, oldEntry, supportedBlacklistTypes); - SearchEventCache.cleanupEvents(true); - return null; - } - - /** - * This method deletes a blacklist entry. - * @param blacklistToUse the name of the blacklist the entry is to be deleted from - * @param oldEntry the entry that is to be deleted - * @param supportedBlacklistTypes - */ - private static void deleteBlacklistEntry( - final File listsPath, - final String blacklistToUse, - String oldEntry, - final BlacklistType[] supportedBlacklistTypes) { - - // load blacklist data from file - final List list = FileUtils.getListArray(new File(listsPath, blacklistToUse)); - - // delete the old entry from file - if (list != null) { - for (final String entry : list) { - if (entry.equals(oldEntry)) { - list.remove(entry); - break; - } - } - FileUtils.writeList(new File(listsPath, blacklistToUse), list.toArray(new String[list.size()])); - } // remove the entry from the running blacklist engine int pos = oldEntry.indexOf('/',0); - if (pos < 0) { - // add default empty path pattern - pos = oldEntry.length(); - oldEntry = oldEntry + "/.*"; + String host = oldEntry.substring(0, pos); + String path = ""; + if (pos > 0) { + path = oldEntry.substring(pos + 1); + } + if (Switchboard.urlBlacklist.getFileName(BlacklistType.DHT) == blacklistToUse) { + Switchboard.urlBlacklist.remove(host, path); } - for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) { - if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) { - Switchboard.urlBlacklist.remove(supportedBlacklistType,oldEntry.substring(0, pos), oldEntry.substring(pos + 1)); + + for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { + if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) != blacklistToUse) { + continue; } + + Blacklist bl = new Blacklist(ListManager.listsPath); + bl.loadList(supportedBlacklistType, blacklistToUse, "/"); + + bl.remove(host, path); } + + SearchEventCache.cleanupEvents(true); + return null; } - - - + /** * This method adds a new entry to the chosen blacklist. * @param blacklistToUse the name of the blacklist the entry is to be added to * @param newEntry the entry that is to be added + * @param header * @param supportedBlacklistTypes + * @return null if no error occurred, else a String to put into LOCATION */ - private static void addBlacklistEntry( - final File listsPath, - final String blacklistToUse, - String newEntry, - final BlacklistType[] supportedBlacklistTypes) { + private static String addBlacklistEntry( + final String blacklistToUse, + final String entry, + final RequestHeader header) { + String newEntry = entry; + + if (blacklistToUse == null || blacklistToUse.isEmpty()) { + return ""; + } + + if (newEntry == null || newEntry.isEmpty()) { + return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse; + } // ignore empty entries if(newEntry == null || newEntry.isEmpty()) { Log.logWarning("Blacklist", "skipped adding an empty entry"); - return; + return ""; } if (newEntry.startsWith("http://") ){ @@ -645,8 +603,8 @@ public class Blacklist_p { if (newEntry.indexOf("*") < 0) { // user did not use any wild cards and just submitted a word - addBlacklistEntry0(listsPath, blacklistToUse, ".*" + newEntry + ".*/.*", supportedBlacklistTypes); - addBlacklistEntry0(listsPath, blacklistToUse, ".*.*/.*" + newEntry + ".*", supportedBlacklistTypes); + newEntry = ".*" + newEntry + ".*/.*"; + newEntry = ".*.*/.*" + newEntry + ".*"; } else { @@ -655,44 +613,29 @@ public class Blacklist_p { // add default empty path pattern newEntry = newEntry + "/.*"; } - - addBlacklistEntry0(listsPath, blacklistToUse, newEntry, supportedBlacklistTypes); } - } - private static void addBlacklistEntry0( - final File listsPath, - final String blacklistToUse, - String newEntry, - final BlacklistType[] supportedBlacklistTypes) { - - if (!Blacklist.blacklistFileContains(listsPath, blacklistToUse, newEntry)) { - // append the line to the file - PrintWriter pw = null; - try { - pw = new PrintWriter(new FileWriter(new File(listsPath, blacklistToUse), true)); - pw.println(newEntry); - pw.close(); - } catch (final IOException e) { - Log.logException(e); - } finally { - if (pw != null) { - try { - pw.close(); - } catch (final Exception e) { - Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage()); - } + int pos = newEntry.indexOf('/',0); + String host = newEntry.substring(0, pos); + String path = newEntry.substring(pos + 1); - } - } - - // add to blacklist - int pos = newEntry.indexOf('/',0); - for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) { - if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistToUse)) { - Switchboard.urlBlacklist.add(supportedBlacklistType, newEntry.substring(0, pos), newEntry.substring(pos + 1)); - } + if (Switchboard.urlBlacklist.getFileName(BlacklistType.DHT) == blacklistToUse) { + Switchboard.urlBlacklist.add(host, path); + } + + for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { + if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) != blacklistToUse) { + continue; } + + Blacklist bl = new Blacklist(ListManager.listsPath); + bl.loadList(supportedBlacklistType, blacklistToUse, "/"); + + bl.add(supportedBlacklistType, host, path); } + + SearchEventCache.cleanupEvents(true); + return null; } + } diff --git a/source/net/yacy/repository/Blacklist.java b/source/net/yacy/repository/Blacklist.java index d01310f7b..85f0d3ba5 100644 --- a/source/net/yacy/repository/Blacklist.java +++ b/source/net/yacy/repository/Blacklist.java @@ -28,15 +28,18 @@ package net.yacy.repository; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.PrintWriter; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; @@ -44,6 +47,7 @@ import java.util.regex.PatternSyntaxException; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.SpaceExceededException; +import net.yacy.data.ListManager; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.meta.URIMetadataRow; @@ -92,6 +96,7 @@ public class Blacklist { } private File blacklistRootPath = null; + private Map blacklistFiles = new TreeMap(); private final ConcurrentMap cachedUrlHashs; private final ConcurrentMap>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here private final ConcurrentMap>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here @@ -148,6 +153,14 @@ public class Blacklist { return this.cachedUrlHashs.get(blacklistType); } + public final String getFileName(BlacklistType type) { + return blacklistFiles.get(type); + } + + public final File getRootPath() { + return blacklistRootPath; + } + public final void clear() { for (final Map> entry : this.hostpaths_matchable.values()) { entry.clear(); @@ -158,6 +171,9 @@ public class Blacklist { for (final HandleSet entry : this.cachedUrlHashs.values()) { entry.clear(); } + blacklistFiles.clear(); + blacklistRootPath = null; + } public final int size() { @@ -177,7 +193,7 @@ public class Blacklist { public final void loadList(final BlacklistFile[] blFiles, final String sep) { for (final BlacklistFile blf : blFiles) { - loadList(blf.getType(), blf.getFileName(), sep); + loadList(blf, sep); } } @@ -188,6 +204,10 @@ public class Blacklist { * @param sep */ private void loadList(final BlacklistFile blFile, final String sep) { + if (!blacklistFiles.containsKey(blFile.getType())) { + blacklistFiles.put(blFile.getType(), blFile.getFileName()); + } + final Map> blacklistMapMatch = getBlacklistMap(blFile.getType(), true); final Map> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false); Set>> loadedBlacklist; @@ -250,6 +270,15 @@ public class Blacklist { getBlacklistMap(blacklistType, false).remove(host); } + /** + * Removes entry for all blacklist types. + */ + public final void remove(final String host, final String path) { + for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { + Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path); + } + } + public final void remove(final BlacklistType blacklistType, final String host, final String path) { final Map> blacklistMap = getBlacklistMap(blacklistType, true); @@ -269,9 +298,35 @@ public class Blacklist { blacklistMapNotMatch.remove(host); } } + + // load blacklist data from file + final List list = FileUtils.getListArray(new File(ListManager.listsPath, getFileName(blacklistType))); + + // delete the old entry from file + if (list != null) { + for (final String e : list) { + if (e.equals(host + "/" + path)) { + list.remove(e); + break; + } + } + FileUtils.writeList(new File(ListManager.listsPath, getFileName(blacklistType)), list.toArray(new String[list.size()])); + } + } + + /** + * Adds a new blacklist entry for all types. + */ + public final void add(final String host, final String path) { + for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { + add(supportedBlacklistType, host, path); + } } public final void add(final BlacklistType blacklistType, final String host, final String path) { + if (contains(blacklistType, host, path)) { + return; + } if (host == null) { throw new IllegalArgumentException("host may not be null"); } @@ -292,8 +347,31 @@ public class Blacklist { if (!(blacklistMap.containsKey(h) && ((hostList = blacklistMap.get(h)) != null))) { blacklistMap.put(h, (hostList = new HashSet())); } + + // Create add case insesitive regex + Pattern pattern = Pattern.compile("(?i)" + p); + + hostList.add(pattern); + + // append the line to the file + PrintWriter pw = null; + try { + pw = new PrintWriter(new FileWriter(new File(blacklistRootPath, getFileName(blacklistType)), true)); + pw.println(pattern); + pw.close(); + } catch (final IOException e) { + Log.logException(e); + } finally { + if (pw != null) { + try { + pw.close(); + } catch (final Exception e) { + Log.logWarning("Blacklist", "could not close stream to " + + getFileName(blacklistType) + "! " + e.getMessage()); + } - hostList.add(Pattern.compile("(?i)" + p)); // add case insesitive regex + } + } } public final int blacklistCacheSize() {