Merge remote-tracking branch 'nutomics/blacklist_structure'

htroot/Blacklist_p.java

@@ -30,9 +30,7 @@
 // if the shell's current path is HTROOT
 
 import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.io.PrintWriter;
 import java.net.MalformedURLException;
 import java.util.Arrays;
 import java.util.List;
@@ -251,7 +249,7 @@ public class Blacklist_p {
                 if (selectedBlacklistEntries.length > 0) {
                     String temp = null;
                     for (final String selectedBlacklistEntry : selectedBlacklistEntries) {
-                        if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) {
+                        if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header)) != null) {
                             prop.put("LOCATION", temp);
                             return prop;
                         }
@@ -273,7 +271,7 @@ public class Blacklist_p {
                 // store this call as api call
                 ListManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + blentry);
 
-                final String temp = addBlacklistEntry(blacklistToUse, blentry, header, BlacklistType.values());
+                final String temp = addBlacklistEntry(blacklistToUse, blentry, header);
                 if (temp != null) {
                     prop.put("LOCATION", temp);
                     return prop;
@@ -300,12 +298,12 @@ public class Blacklist_p {
                         !targetBlacklist.equals(blacklistToUse)) {
                     String temp;
                     for (final String selectedBlacklistEntry : selectedBlacklistEntries) {
-                        if ((temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntry, header, BlacklistType.values())) != null) {
+                        if ((temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntry, header)) != null) {
                             prop.put("LOCATION", temp);
                             return prop;
                         }
 
-                        if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) {
+                        if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header)) != null) {
                             prop.put("LOCATION", temp);
                             return prop;
 
@@ -342,12 +340,12 @@ public class Blacklist_p {
 
                         if (!selectedBlacklistEntries[i].equals(editedBlacklistEntries[i])) {
 
-                            if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, BlacklistType.values())) != null) {
+                            if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header)) != null) {
                                 prop.put("LOCATION", temp);
                                 return prop;
                             }
 
-                            if ((temp = addBlacklistEntry(blacklistToUse, editedBlacklistEntries[i], header, BlacklistType.values())) != null) {
+                            if ((temp = addBlacklistEntry(blacklistToUse, editedBlacklistEntries[i], header)) != null) {
                                 prop.put("LOCATION", temp);
                                 return prop;
                             }
@@ -519,49 +517,19 @@ public class Blacklist_p {
         return prop;
     }
 
-
-
-    /**
-     * This method adds a new entry to the chosen blacklist.
-     * @param blacklistToUse the name of the blacklist the entry is to be added to
-     * @param newEntry the entry that is to be added
-     * @param header
-     * @param supportedBlacklistTypes
-     * @return null if no error occurred, else a String to put into LOCATION
-     */
-    private static String addBlacklistEntry(
-            final String blacklistToUse,
-            final String newEntry,
-            final RequestHeader header,
-            final BlacklistType[] supportedBlacklistTypes) {
-
-        if (blacklistToUse == null || blacklistToUse.isEmpty()) {
-            return "";
-        }
-
-        if (newEntry == null || newEntry.isEmpty()) {
-            return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
-        }
-
-        addBlacklistEntry(ListManager.listsPath, blacklistToUse, newEntry, supportedBlacklistTypes);
-        SearchEventCache.cleanupEvents(true);
-        return null;
-    }
-
-
     /**
      * This method deletes a blacklist entry.
      * @param blacklistToUse the name of the blacklist the entry is to be deleted from
-     * @param oldEntry the entry that is to be deleted
+     * @param entry the entry that is to be deleted
      * @param header
      * @param supportedBlacklistTypes
-     * @return null if no error occured, else a String to put into LOCATION
+     * @return null if no error occurred, else a String to put into LOCATION
      */
     private static String deleteBlacklistEntry(
             final String blacklistToUse,
-            final String oldEntry,
+            final String entry,
-            final RequestHeader header,
+            final RequestHeader header) {
-            final BlacklistType[] supportedBlacklistTypes) {
+    	String oldEntry = entry;
 
         if (blacklistToUse == null || blacklistToUse.isEmpty()) {
             return "";
@@ -571,69 +539,59 @@ public class Blacklist_p {
             return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
         }
 
-        deleteBlacklistEntry(ListManager.listsPath, blacklistToUse, oldEntry, supportedBlacklistTypes);
-        SearchEventCache.cleanupEvents(true);
-        return null;
-    }
-
-    /**
-     * This method deletes a blacklist entry.
-     * @param blacklistToUse the name of the blacklist the entry is to be deleted from
-     * @param oldEntry the entry that is to be deleted
-     * @param supportedBlacklistTypes
-     */
-    private static void deleteBlacklistEntry(
-            final File listsPath,
-            final String blacklistToUse,
-            String oldEntry,
-            final BlacklistType[] supportedBlacklistTypes) {
-
-        // load blacklist data from file
-        final List<String> list = FileUtils.getListArray(new File(listsPath, blacklistToUse));
-
-        // delete the old entry from file
-        if (list != null) {
-            for (final String entry : list) {
-                if (entry.equals(oldEntry)) {
-                    list.remove(entry);
-                    break;
-                }
-            }
-            FileUtils.writeList(new File(listsPath, blacklistToUse), list.toArray(new String[list.size()]));
-        }
 
         // remove the entry from the running blacklist engine
         int pos = oldEntry.indexOf('/',0);
-        if (pos < 0) {
+        String host = oldEntry.substring(0, pos);
-            // add default empty path pattern
+        String path = "";
-            pos = oldEntry.length();
+        if (pos > 0) {
-            oldEntry = oldEntry + "/.*";
+            path = oldEntry.substring(pos + 1);
         }
-        for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) {
+        if (Switchboard.urlBlacklist.getFileName(BlacklistType.DHT) == blacklistToUse) {
-            if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) {
+        	Switchboard.urlBlacklist.remove(host, path);
-                Switchboard.urlBlacklist.remove(supportedBlacklistType,oldEntry.substring(0, pos), oldEntry.substring(pos + 1));
-            }
         }
-    }
         
+        for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
+            if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) != blacklistToUse) {
+            	continue;
+            }
+            
+            Blacklist bl = new Blacklist(ListManager.listsPath);
+        	bl.loadList(supportedBlacklistType, blacklistToUse, "/");
         	
+        	bl.remove(host, path);
+        }
+        
+        SearchEventCache.cleanupEvents(true);
+        return null;
+    }
     
     /**
      * This method adds a new entry to the chosen blacklist.
      * @param blacklistToUse the name of the blacklist the entry is to be added to
      * @param newEntry the entry that is to be added
+     * @param header
      * @param supportedBlacklistTypes
+     * @return null if no error occurred, else a String to put into LOCATION
      */
-    private static void addBlacklistEntry(
+	private static String addBlacklistEntry(
-            final File listsPath,
+	        final String blacklistToUse,
-            final String blacklistToUse,
+	        final String entry,
-            String newEntry,
+	        final RequestHeader header) {
-            final BlacklistType[] supportedBlacklistTypes) {
+    	String newEntry = entry;
+
+        if (blacklistToUse == null || blacklistToUse.isEmpty()) {
+            return "";
+        }
+
+        if (newEntry == null || newEntry.isEmpty()) {
+            return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
+        }
 
         // ignore empty entries
         if(newEntry == null || newEntry.isEmpty()) {
             Log.logWarning("Blacklist", "skipped adding an empty entry");
-            return;
+            return "";
         }
 
         if (newEntry.startsWith("http://") ){
@@ -645,8 +603,8 @@ public class Blacklist_p {
         if (newEntry.indexOf("*") < 0) {
             // user did not use any wild cards and just submitted a word
 
-            addBlacklistEntry0(listsPath, blacklistToUse, ".*" + newEntry + ".*/.*", supportedBlacklistTypes);
+            newEntry = ".*" + newEntry + ".*/.*";
-            addBlacklistEntry0(listsPath, blacklistToUse, ".*.*/.*" + newEntry + ".*", supportedBlacklistTypes);
+            newEntry =  ".*.*/.*" + newEntry + ".*";
 
         } else {
 
@@ -655,44 +613,29 @@ public class Blacklist_p {
                 // add default empty path pattern
                 newEntry = newEntry + "/.*";
             }
-
-            addBlacklistEntry0(listsPath, blacklistToUse, newEntry, supportedBlacklistTypes);
         }
-    }
 
-    private static void addBlacklistEntry0(
+        int pos = newEntry.indexOf('/',0);
-            final File listsPath,
+        String host = newEntry.substring(0, pos);
-            final String blacklistToUse,
+        String path = newEntry.substring(pos + 1);
-            String newEntry,
-            final BlacklistType[] supportedBlacklistTypes) {
-
-        if (!Blacklist.blacklistFileContains(listsPath, blacklistToUse, newEntry)) {
-            // append the line to the file
-            PrintWriter pw = null;
-            try {
-                pw = new PrintWriter(new FileWriter(new File(listsPath, blacklistToUse), true));
-                pw.println(newEntry);
-                pw.close();
-            } catch (final IOException e) {
-                Log.logException(e);
-            } finally {
-                if (pw != null) {
-                    try {
-                        pw.close();
-                    } catch (final Exception e) {
-                        Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage());
-                    }
 
-                }
+        if (Switchboard.urlBlacklist.getFileName(BlacklistType.DHT) == blacklistToUse) {
-            }
+        	Switchboard.urlBlacklist.add(host, path);	
+        }
         
-            // add to blacklist
+        for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
-            int pos = newEntry.indexOf('/',0);
+            if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) != blacklistToUse) {
-            for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) {
+            	continue;
-                if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistToUse)) {
-                    Switchboard.urlBlacklist.add(supportedBlacklistType, newEntry.substring(0, pos), newEntry.substring(pos + 1));
-                }
             }
+            
+	    	Blacklist bl = new Blacklist(ListManager.listsPath);
+	    	bl.loadList(supportedBlacklistType, blacklistToUse, "/");
+	    	
+	    	bl.add(supportedBlacklistType, host, path);    	
         }
+        
+        SearchEventCache.cleanupEvents(true);
+        return null;
     }
+    
 }

source/net/yacy/repository/Blacklist.java

@@ -28,15 +28,18 @@ package net.yacy.repository;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
+import java.io.PrintWriter;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.regex.Pattern;
@@ -44,6 +47,7 @@ import java.util.regex.PatternSyntaxException;
 
 import net.yacy.cora.storage.HandleSet;
 import net.yacy.cora.util.SpaceExceededException;
+import net.yacy.data.ListManager;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.data.meta.URIMetadataNode;
 import net.yacy.kelondro.data.meta.URIMetadataRow;
@@ -92,6 +96,7 @@ public class Blacklist {
     }
 
     private File blacklistRootPath = null;
+    private Map<BlacklistType, String> blacklistFiles = new TreeMap<BlacklistType, String>();
     private final ConcurrentMap<BlacklistType, HandleSet> cachedUrlHashs;
     private final ConcurrentMap<BlacklistType, Map<String, Set<Pattern>>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
     private final ConcurrentMap<BlacklistType, Map<String, Set<Pattern>>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
@@ -148,6 +153,14 @@ public class Blacklist {
         return this.cachedUrlHashs.get(blacklistType);
     }
 
+    public final String getFileName(BlacklistType type) {
+    	return blacklistFiles.get(type);
+    }
+    
+    public final File getRootPath() {
+    	return blacklistRootPath;
+    }
+    
     public final void clear() {
         for (final Map<String, Set<Pattern>> entry : this.hostpaths_matchable.values()) {
             entry.clear();
@@ -158,6 +171,9 @@ public class Blacklist {
         for (final HandleSet entry : this.cachedUrlHashs.values()) {
             entry.clear();
         }
+        blacklistFiles.clear();
+        blacklistRootPath = null;
+        
     }
 
     public final int size() {
@@ -177,7 +193,7 @@ public class Blacklist {
 
     public final void loadList(final BlacklistFile[] blFiles, final String sep) {
         for (final BlacklistFile blf : blFiles) {
-            loadList(blf.getType(), blf.getFileName(), sep);
+            loadList(blf, sep);
         }
     }
 
@@ -188,6 +204,10 @@ public class Blacklist {
      * @param sep
      */
     private void loadList(final BlacklistFile blFile, final String sep) {
+    	if (!blacklistFiles.containsKey(blFile.getType())) {
+    		blacklistFiles.put(blFile.getType(), blFile.getFileName());
+    	}
+    	
         final Map<String, Set<Pattern>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true);
         final Map<String, Set<Pattern>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false);
         Set<Map.Entry<String, List<String>>> loadedBlacklist;
@@ -250,6 +270,15 @@ public class Blacklist {
         getBlacklistMap(blacklistType, false).remove(host);
     }
 
+    /**
+     * Removes entry for all blacklist types.
+     */
+    public final void remove(final String host, final String path) {
+        for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
+            Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path);
+        }    	
+    }
+    
     public final void remove(final BlacklistType blacklistType, final String host, final String path) {
 
         final Map<String, Set<Pattern>> blacklistMap = getBlacklistMap(blacklistType, true);
@@ -269,9 +298,35 @@ public class Blacklist {
                 blacklistMapNotMatch.remove(host);
             }
         }
+
+        // load blacklist data from file
+        final List<String> list = FileUtils.getListArray(new File(ListManager.listsPath, getFileName(blacklistType)));
+        
+        // delete the old entry from file
+        if (list != null) {
+            for (final String e : list) {
+                if (e.equals(host + "/" + path)) {
+                    list.remove(e);
+                    break;
+                }
+            }
+            FileUtils.writeList(new File(ListManager.listsPath, getFileName(blacklistType)), list.toArray(new String[list.size()]));
+        }
+    }
+    
+    /**
+     * Adds a new blacklist entry for all types.
+     */
+    public final void add(final String host, final String path) {
+        for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
+	    	add(supportedBlacklistType, host, path);	    	
+        }
     }
 
     public final void add(final BlacklistType blacklistType, final String host, final String path) {
+    	if (contains(blacklistType, host, path)) {
+    		return;
+    	}
         if (host == null) {
             throw new IllegalArgumentException("host may not be null");
         }
@@ -293,7 +348,30 @@ public class Blacklist {
             blacklistMap.put(h, (hostList = new HashSet<Pattern>()));
         }
         
-        hostList.add(Pattern.compile("(?i)" + p)); // add case insesitive regex
+        // Create add case insesitive regex
+        Pattern pattern = Pattern.compile("(?i)" + p); 
+        
+        hostList.add(pattern); 
+        
+        // append the line to the file
+        PrintWriter pw = null;
+        try {
+            pw = new PrintWriter(new FileWriter(new File(blacklistRootPath, getFileName(blacklistType)), true));
+            pw.println(pattern);
+            pw.close();
+        } catch (final IOException e) {
+            Log.logException(e);
+        } finally {
+            if (pw != null) {
+                try {
+                    pw.close();
+                } catch (final Exception e) {
+                    Log.logWarning("Blacklist", "could not close stream to " + 
+                    		getFileName(blacklistType) + "! " + e.getMessage());
+                }
+
+            }
+        }
     }
 
     public final int blacklistCacheSize() {