From 7d53ac86a396bd57fb4f34cd73bbae7896654867 Mon Sep 17 00:00:00 2001 From: sixcooler Date: Mon, 29 Jul 2013 19:09:28 +0200 Subject: [PATCH] fix for Blacklist (-Administration) --- htroot/BlacklistCleaner_p.java | 80 +++++---------- htroot/BlacklistImpExp_p.java | 6 -- htroot/BlacklistTest_p.java | 9 +- htroot/Blacklist_p.java | 27 +---- htroot/IndexControlRWIs_p.java | 115 +++++++++------------- htroot/api/blacklists.java | 2 - htroot/api/blacklists_p.java | 2 - htroot/sharedBlacklist_p.java | 18 +--- source/net/yacy/repository/Blacklist.java | 45 +++------ 9 files changed, 90 insertions(+), 214 deletions(-) diff --git a/htroot/BlacklistCleaner_p.java b/htroot/BlacklistCleaner_p.java index 4cadb48cc..2beb6efe3 100644 --- a/htroot/BlacklistCleaner_p.java +++ b/htroot/BlacklistCleaner_p.java @@ -30,9 +30,6 @@ // if the shell's current path is HTROOT import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -52,7 +49,6 @@ import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist.BlacklistError; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.SearchEventCache; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -68,12 +64,9 @@ public class BlacklistCleaner_p { Blacklist.class }; - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { final serverObjects prop = new serverObjects(); - // initialize the list manager - ListManager.switchboard = (Switchboard) env; - ListManager.listsPath = new File(env.getDataPath(), env.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT)); String blacklistToUse = null; prop.put(DISABLED+"checked", "1"); @@ -272,26 +265,13 @@ public class BlacklistCleaner_p { * @return Length of the list of entries to be removed. */ private static int removeEntries(final String blacklistToUse, final BlacklistType[] supportedBlacklistTypes, final String[] entries) { - // load blacklist data from file - final List list = FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse)); - - boolean listChanged = false; - - // delete the old entry from file for (final String entry : entries) { String s = entry; - if (list != null){ - - // get rid of escape characters which make it impossible to - // properly use contains() - if (s.contains("\\\\")) { - s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\")); - } - - if (list.contains(s)) { - listChanged = list.remove(s); - } + // get rid of escape characters which make it impossible to + // properly use contains() + if (s.contains("\\\\")) { + s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\")); } // remove the entry from the running blacklist engine @@ -300,7 +280,7 @@ public class BlacklistCleaner_p { final String host = (s.indexOf('/',0) == -1) ? s : s.substring(0, s.indexOf('/',0)); final String path = (s.indexOf('/',0) == -1) ? ".*" : s.substring(s.indexOf('/',0) + 1); try { - Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path); + Switchboard.urlBlacklist.remove(supportedBlacklistType, blacklistToUse, host, path); } catch (final RuntimeException e) { ConcurrentLog.severe("BLACKLIST-CLEANER", e.getMessage() + ": " + host + "/" + path); } @@ -308,9 +288,6 @@ public class BlacklistCleaner_p { } SearchEventCache.cleanupEvents(true); } - if (listChanged){ - FileUtils.writeList(new File(ListManager.listsPath, blacklistToUse), list.toArray(new String[list.size()])); - } return entries.length; } @@ -328,34 +305,23 @@ public class BlacklistCleaner_p { final String[] oldEntry, final String[] newEntry) { removeEntries(blacklistToUse, supportedBlacklistTypes, oldEntry); - PrintWriter pw = null; - try { - pw = new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklistToUse), true)); - String host, path; - for (final String n : newEntry) { - final int pos = n.indexOf('/',0); - if (pos < 0) { - host = n; - path = ".*"; - } else { - host = n.substring(0, pos); - path = n.substring(pos + 1); - } - pw.println(host + "/" + path); - for (final BlacklistType s : supportedBlacklistTypes) { - if (ListManager.listSetContains(s + ".BlackLists",blacklistToUse)) { - Switchboard.urlBlacklist.add( - s, - host, - path); - } - } - SearchEventCache.cleanupEvents(true); - } - pw.close(); - } catch (final IOException e) { - ConcurrentLog.severe("BLACKLIST-CLEANER", "error on writing altered entries to blacklist", e); - } + String host, path; + for (final String n : newEntry) { + final int pos = n.indexOf('/',0); + if (pos < 0) { + host = n; + path = ".*"; + } else { + host = n.substring(0, pos); + path = n.substring(pos + 1); + } + for (final BlacklistType s : supportedBlacklistTypes) { + if (ListManager.listSetContains(s + ".BlackLists",blacklistToUse)) { + Switchboard.urlBlacklist.add(s, blacklistToUse, host, path); + } + } + SearchEventCache.cleanupEvents(true); + } return newEntry.length; } } diff --git a/htroot/BlacklistImpExp_p.java b/htroot/BlacklistImpExp_p.java index 040a6833c..b832f4da0 100644 --- a/htroot/BlacklistImpExp_p.java +++ b/htroot/BlacklistImpExp_p.java @@ -29,7 +29,6 @@ // javac -classpath .:../classes Blacklist_p.java // if the shell's current path is HTROOT -import java.io.File; import java.util.Iterator; import java.util.List; import java.util.TreeMap; @@ -41,7 +40,6 @@ import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.Seed; import net.yacy.repository.Blacklist; import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -51,10 +49,6 @@ public class BlacklistImpExp_p { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; - // initialize the list manager - ListManager.switchboard = (Switchboard) env; - ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT)); - // loading all blacklist files located in the directory final List dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); diff --git a/htroot/BlacklistTest_p.java b/htroot/BlacklistTest_p.java index 198de03cb..3fe6a88b7 100644 --- a/htroot/BlacklistTest_p.java +++ b/htroot/BlacklistTest_p.java @@ -29,26 +29,19 @@ // javac -classpath .:../classes Blacklist_p.java // if the shell's current path is HTROOT -import java.io.File; import java.net.MalformedURLException; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.data.ListManager; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; public class BlacklistTest_p { - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { - - // initialize the list manager - ListManager.switchboard = (Switchboard) env; - ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT)); + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { final serverObjects prop = new serverObjects(); prop.putHTML("blacklistEngine", Blacklist.getEngineInfo()); diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index a746142a0..ed86da3a0 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -45,7 +45,6 @@ import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.SearchEventCache; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -58,11 +57,7 @@ public class Blacklist_p { private final static String BLACKLIST_SHARED = "BlackLists.Shared"; - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - - // initialize the list manager - ListManager.switchboard = (Switchboard) env; - ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT)); + public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { // load all blacklist files located in the directory List dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); @@ -550,14 +545,8 @@ public class Blacklist_p { } for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { - if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) != blacklistToUse) { - Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path); - } - else { - Blacklist bl = new Blacklist(ListManager.listsPath); - bl.loadList(supportedBlacklistType, blacklistToUse, "/"); - - bl.remove(host, path); + if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) { + Switchboard.urlBlacklist.remove(supportedBlacklistType, blacklistToUse, host, path); } } @@ -619,14 +608,8 @@ public class Blacklist_p { String path = newEntry.substring(pos + 1); for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { - if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) == blacklistToUse) { - Switchboard.urlBlacklist.add(supportedBlacklistType, host, path); - } - else { - Blacklist bl = new Blacklist(ListManager.listsPath); - bl.loadList(supportedBlacklistType, blacklistToUse, "/"); - - bl.add(supportedBlacklistType, host, path); + if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) { + Switchboard.urlBlacklist.add(supportedBlacklistType, blacklistToUse, host, path); } } diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index b85c6c0be..90f556251 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -25,10 +25,7 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -367,72 +364,58 @@ public class IndexControlRWIs_p { URIMetadataRow.rowdef.objectOrder, urlb.size()); if ( post.containsKey("blacklisturls") ) { - PrintWriter pw; - try { - final String[] supportedBlacklistTypes = - env.getConfig("BlackLists.types", "").split(","); - pw = - new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklist), true)); - DigestURI url; - for ( final byte[] b : urlb ) { - try { - urlHashes.put(b); - } catch (final SpaceExceededException e ) { - ConcurrentLog.logException(e); - } - url = segment.fulltext().getURL(b); - segment.fulltext().remove(b); - if ( url != null ) { - pw.println(url.getHost() + "/" + url.getFile()); - for ( final String supportedBlacklistType : supportedBlacklistTypes ) { - if ( ListManager.listSetContains( - supportedBlacklistType + ".BlackLists", - blacklist) ) { - Switchboard.urlBlacklist.add( - BlacklistType.valueOf(supportedBlacklistType), - url.getHost(), - url.getFile()); - } - } - SearchEventCache.cleanupEvents(true); - } - } - pw.close(); - } catch (final IOException e ) { - } + final String[] supportedBlacklistTypes = + env.getConfig("BlackLists.types", "").split(","); + DigestURI url; + for ( final byte[] b : urlb ) { + try { + urlHashes.put(b); + } catch (final SpaceExceededException e ) { + ConcurrentLog.logException(e); + } + url = segment.fulltext().getURL(b); + segment.fulltext().remove(b); + if ( url != null ) { + for ( final String supportedBlacklistType : supportedBlacklistTypes ) { + if ( ListManager.listSetContains( + supportedBlacklistType + ".BlackLists", + blacklist) ) { + Switchboard.urlBlacklist.add( + BlacklistType.valueOf(supportedBlacklistType), + blacklist, + url.getHost(), + url.getFile()); + } + } + SearchEventCache.cleanupEvents(true); + } + } } if ( post.containsKey("blacklistdomains") ) { - PrintWriter pw; - try { - pw = - new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklist), true)); - DigestURI url; - for ( final byte[] b : urlb ) { - try { - urlHashes.put(b); - } catch (final SpaceExceededException e ) { - ConcurrentLog.logException(e); - } - url = segment.fulltext().getURL(b); - segment.fulltext().remove(b); - if ( url != null ) { - pw.println(url.getHost() + "/.*"); - for ( final BlacklistType supportedBlacklistType : BlacklistType.values() ) { - if ( ListManager.listSetContains( - supportedBlacklistType + ".BlackLists", - blacklist) ) { - Switchboard.urlBlacklist.add( - supportedBlacklistType, - url.getHost(), - ".*"); - } - } - } - } - pw.close(); - } catch (final IOException e ) { - } + DigestURI url; + for ( final byte[] b : urlb ) { + try { + urlHashes.put(b); + } catch (final SpaceExceededException e ) { + ConcurrentLog.logException(e); + } + url = segment.fulltext().getURL(b); + segment.fulltext().remove(b); + if ( url != null ) { + for ( final BlacklistType supportedBlacklistType : BlacklistType.values() ) { + if ( ListManager.listSetContains( + supportedBlacklistType + ".BlackLists", + blacklist) ) { + Switchboard.urlBlacklist.add( + supportedBlacklistType, + blacklist, + url.getHost(), + ".*"); + } + } + } + } } try { segment.termIndex().remove(keyhash, urlHashes); diff --git a/htroot/api/blacklists.java b/htroot/api/blacklists.java index 2ebba0c02..75232cd02 100644 --- a/htroot/api/blacklists.java +++ b/htroot/api/blacklists.java @@ -5,7 +5,6 @@ import java.util.List; import net.yacy.cora.protocol.RequestHeader; import net.yacy.data.ListManager; import net.yacy.kelondro.util.FileUtils; -import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -14,7 +13,6 @@ public class blacklists { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { final serverObjects prop = new serverObjects(); - ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT)); final List dirlist = FileUtils.getDirListing(ListManager.listsPath); int blacklistCount = 0; diff --git a/htroot/api/blacklists_p.java b/htroot/api/blacklists_p.java index 2ef23f65b..cba337ffd 100644 --- a/htroot/api/blacklists_p.java +++ b/htroot/api/blacklists_p.java @@ -7,7 +7,6 @@ import net.yacy.data.ListManager; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist.BlacklistType; -import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -17,7 +16,6 @@ public class blacklists_p { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { final serverObjects prop = new serverObjects(); - ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT)); final List dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); int blacklistCount=0; diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index 643cab4b5..f040e763e 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -30,9 +30,7 @@ //if the shell's current path is HTROOT import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.io.StringReader; import java.util.Arrays; import java.util.HashSet; @@ -53,7 +51,6 @@ import net.yacy.peers.Seed; import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.SearchEventCache; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -89,10 +86,6 @@ public class sharedBlacklist_p { if (post != null) { - // initialize the list manager - ListManager.switchboard = (Switchboard) env; - ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT)); - // loading all blacklist files located in the directory final List dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); @@ -209,11 +202,7 @@ public class sharedBlacklist_p { prop.put("page", "1"); //result page prop.put("status", STATUS_ENTRIES_ADDED); //list of added Entries - PrintWriter pw = null; try { - // open the blacklist file - pw = new PrintWriter(new FileWriter(new File(ListManager.listsPath, selectedBlacklistName), true)); - // loop through the received entry list final int num = post.getInt("num", 0); for(int i = 0; i < num; i++){ @@ -233,13 +222,10 @@ public class sharedBlacklist_p { newItem = newItem + "/.*"; } - // append the item to the file - pw.println(newItem); - if (Switchboard.urlBlacklist != null) { for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",selectedBlacklistName)) { - Switchboard.urlBlacklist.add(supportedBlacklistType,newItem.substring(0, pos), newItem.substring(pos + 1)); + Switchboard.urlBlacklist.add(supportedBlacklistType,selectedBlacklistName,newItem.substring(0, pos), newItem.substring(pos + 1)); } } SearchEventCache.cleanupEvents(true); @@ -249,8 +235,6 @@ public class sharedBlacklist_p { } catch (final Exception e) { prop.put("status", "1"); prop.putHTML("status_error", e.getLocalizedMessage()); - } finally { - if (pw != null) try { pw.close(); } catch (final Exception e){ /* */} } /* unable to use prop.putHTML() or prop.putXML() here because they diff --git a/source/net/yacy/repository/Blacklist.java b/source/net/yacy/repository/Blacklist.java index a7042252c..1129d2e1b 100644 --- a/source/net/yacy/repository/Blacklist.java +++ b/source/net/yacy/repository/Blacklist.java @@ -132,7 +132,7 @@ public class Blacklist { ConcurrentLog.fine("Blacklist", "All blacklists has been shutdown."); } - public final void setRootPath(final File rootPath) { + private final void setRootPath(final File rootPath) { if (rootPath == null) { throw new NullPointerException("The blacklist root path must not be null."); } @@ -154,10 +154,6 @@ public class Blacklist { return this.cachedUrlHashs.get(blacklistType); } - public final String getFileName(BlacklistType type) { - return blacklistFiles.get(type); - } - public final File getRootPath() { return blacklistRootPath; } @@ -173,8 +169,6 @@ public class Blacklist { entry.clear(); } blacklistFiles.clear(); - blacklistRootPath = null; - } public final int size() { @@ -271,16 +265,7 @@ public class Blacklist { getBlacklistMap(blacklistType, false).remove(host); } - /** - * Removes entry for all blacklist types. - */ - public final void remove(final String host, final String path) { - for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { - Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path); - } - } - - public final void remove(final BlacklistType blacklistType, final String host, final String path) { + public final void remove(final BlacklistType blacklistType, final String blacklistToUse, final String host, final String path) { final Map> blacklistMap = getBlacklistMap(blacklistType, true); Set hostList = blacklistMap.get(host); @@ -301,7 +286,7 @@ public class Blacklist { } // load blacklist data from file - final List list = FileUtils.getListArray(new File(ListManager.listsPath, getFileName(blacklistType))); + final List list = FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse)); // delete the old entry from file if (list != null) { @@ -311,20 +296,11 @@ public class Blacklist { break; } } - FileUtils.writeList(new File(ListManager.listsPath, getFileName(blacklistType)), list.toArray(new String[list.size()])); + FileUtils.writeList(new File(ListManager.listsPath, blacklistToUse), list.toArray(new String[list.size()])); } } - /** - * Adds a new blacklist entry for all types. - */ - public final void add(final String host, final String path) { - for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { - add(supportedBlacklistType, host, path); - } - } - - public final void add(final BlacklistType blacklistType, final String host, final String path) { + public final void add(final BlacklistType blacklistType, final String blacklistToUse, final String host, final String path) { if (contains(blacklistType, host, path)) { return; } @@ -355,12 +331,13 @@ public class Blacklist { // Append the line to the file. PrintWriter pw = null; - try { + try { + final String newEntry = h + "/" + pattern; if (!blacklistFileContains(blacklistRootPath, - getFileName(blacklistType), pattern.toString())) { + blacklistToUse, newEntry)) { pw = new PrintWriter(new FileWriter(new File(blacklistRootPath, - getFileName(blacklistType)), true)); - pw.println(pattern); + blacklistToUse), true)); + pw.println(newEntry); pw.close(); } } catch (final IOException e) { @@ -371,7 +348,7 @@ public class Blacklist { pw.close(); } catch (final Exception e) { ConcurrentLog.warn("Blacklist", "could not close stream to " + - getFileName(blacklistType) + "! " + e.getMessage()); + blacklistToUse + "! " + e.getMessage()); } } }