*) added new method "contains()" to Blacklist interface

*) implemented contains() in class AbstractBlacklist
*) used new method in Blacklist_p to prevent double entries in blacklists

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5832 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Blacklist_p.java

@@ -518,6 +518,8 @@ public class Blacklist_p {
 
         if (newEntry.startsWith("http://") ){
             newEntry = newEntry.substring(7);
+        } else if (newEntry.startsWith("https://")) {
+            newEntry = newEntry.substring(8);
         }
 
         int pos = newEntry.indexOf("/");
@@ -527,6 +529,8 @@ public class Blacklist_p {
             newEntry = newEntry + "/.*";
         }
 
+        if (supportedBlacklistTypes.length > 0 &&
+                !plasmaSwitchboard.urlBlacklist.contains(supportedBlacklistTypes[0], newEntry.substring(0, pos), newEntry.substring(pos + 1))) {
             // append the line to the file
             PrintWriter pw = null;
             try {
@@ -536,13 +540,21 @@ public class Blacklist_p {
             } catch (final IOException e) {
                 e.printStackTrace();
             } finally {
-            if (pw != null) try { pw.close(); } catch (final Exception e){ Log.logWarning("Blacklist", "could not close stream to "+ blacklistToUse +"! "+ e.getMessage());}
+                if (pw != null) {
+                    try {
+                        pw.close();
+                    } catch (final Exception e) {
+                        Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage());
+                    }
+
+                }
             }
 
             // add to blacklist
-        for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
-            if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
-                plasmaSwitchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes],newEntry.substring(0, pos), newEntry.substring(pos + 1));
+            for (int blTypes = 0; blTypes < supportedBlacklistTypes.length; blTypes++) {
+                if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) {
+                    plasmaSwitchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes], newEntry.substring(0, pos), newEntry.substring(pos + 1));
+                }
             }
         }
 

source/de/anomic/data/AbstractBlacklist.java

@@ -244,6 +244,23 @@ public abstract class AbstractBlacklist implements Blacklist {
         return urlHashCache.contains(urlHash);
     }
 
+    public boolean contains(final String blacklistType, String host, String path) {
+        boolean ret = false;
+
+        if (blacklistType != null && host != null && path != null) {
+            HashMap<String, ArrayList<String>> blacklistMap;
+            blacklistMap = (isMatchable(host)) ? getBlacklistMap(blacklistType,true) : getBlacklistMap(blacklistType,false);
+
+            // avoid PatternSyntaxException e
+            if(!isMatchable(host) && host.startsWith("*"))
+                host = "." + host;
+
+            ArrayList<String> hostList = blacklistMap.get(host.toLowerCase());
+            if (hostList != null) ret = hostList.contains(path);
+        }
+        return ret;
+    }
+
     public boolean isListed(final String blacklistType, final yacyURL url) {
 
         final Set<String> urlHashCache = getCacheUrlHashsSet(blacklistType);        

source/de/anomic/data/Blacklist.java

@@ -87,6 +87,8 @@ public interface Blacklist {
     public void loadList(blacklistFile[] blFiles, String sep);
 
 
+    public boolean contains(String blacklistType, String host, String path);
+
     public boolean hashInBlacklistedCache(String blacklistType, String urlHash);
     
     public boolean isListed(String blacklistType, yacyURL url);