Merged changes from master.

source/net/yacy/http/servlets/YaCyDefaultServlet.java

@@ -290,7 +290,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
 
             if (!hasClass && (resource == null || !resource.exists()) && !pathInContext.contains("..")) {
                 // try to get this in the alternative htDocsPath
-                resource = Resource.newResource(new File(HTTPDFileHandler.htDocsPath, pathInContext));
+                resource = Resource.newResource(new File(_htDocsPath, pathInContext));
             }
             
             if (ConcurrentLog.isFine("FILEHANDLER")) {
@@ -1033,8 +1033,15 @@ public class YaCyDefaultServlet extends HttpServlet  {
 			}
 		}
 	}
-    
-    private static String appendPath(String proplist, String path) {
+
+    /**
+     * Append a path string to comma separated string of pathes if not already
+     * contained in the proplist string
+     * @param proplist comma separated string of pathes
+     * @param path path to be appended
+     * @return comma separated string of pathes including param path
+     */
+    private String appendPath(String proplist, String path) {
         if (proplist.length() == 0) return path;
         if (proplist.contains(path)) return proplist;
         return proplist + "," + path;

source/net/yacy/repository/Blacklist.java

@@ -33,6 +33,7 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.PrintWriter;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -256,17 +257,27 @@ public class Blacklist {
         loadList(blFile, sep);
     }
 
-    public final void removeAll(final BlacklistType blacklistType, final String host) {
-        getBlacklistMap(blacklistType, true).remove(host);
-        getBlacklistMap(blacklistType, false).remove(host);
-    }
-
+    /**
+     * remove the host/path from internal blacklist maps for given blacklistType
+     * !! and removes the entry from source blacklist file !!
+     * @param blacklistType
+     * @param blacklistToUse
+     * @param host
+     * @param path
+     */
     public final void remove(final BlacklistType blacklistType, final String blacklistToUse, final String host, final String path) {
 
         final Map<String, Set<Pattern>> blacklistMap = getBlacklistMap(blacklistType, true);
         Set<Pattern> hostList = blacklistMap.get(host);
         if (hostList != null) {
-            hostList.remove(path);
+            // remove pattern from list (by comparing patternstring with path, remove(path) will not match path)
+            for (Pattern hp : hostList) {
+                String hpxs = hp.pattern();
+                if (hpxs.equals(path)) {
+                    hostList.remove(hp);
+                    break;
+                }
+            }
             if (hostList.isEmpty()) {
                 blacklistMap.remove(host);
             }
@@ -275,12 +286,21 @@ public class Blacklist {
         final Map<String, Set<Pattern>> blacklistMapNotMatch = getBlacklistMap(blacklistType, false);
         hostList = blacklistMapNotMatch.get(host);
         if (hostList != null) {
-            hostList.remove(path);
+            // remove pattern from list
+            for (Pattern hp : hostList) {
+                String hpxs = hp.pattern();
+                if (hpxs.equals(path)) {
+                    hostList.remove(hp);
+                    break;
+                }
+            }
             if (hostList.isEmpty()) {
                 blacklistMapNotMatch.remove(host);
             }
         }
 
+        //TODO: check if delete from blacklist is desired, on reload entry will not be available in any blacklist
+        //      even if remove (above) from internal maps (at runtime) is only done for given blacklistType
         // load blacklist data from file
         final List<String> list = FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse));
         
@@ -297,9 +317,9 @@ public class Blacklist {
     }
     
     /**
-     * 
+     * Adds entry to a given blacklist internal data and updates the source file
      * @param blacklistType
-     * @param blacklistToUse
+     * @param blacklistToUse source file
      * @param host
      * @param path
      * @throws PunycodeException
@@ -362,7 +382,7 @@ public class Blacklist {
     }
 
     /**
-     * appends aN entry to the backlist source file.
+     * appends aN entry to the backlist source file and updates internal blacklist maps.
      * 
      * @param blacklistSourcefile name of the blacklist file (LISTS/*.black)
      * @param host host or host pattern
@@ -387,8 +407,21 @@ public class Blacklist {
         
         if (!p.isEmpty() && p.charAt(0) == '*') {
             p = "." + p;
-        }
+        }        
         Pattern pattern = Pattern.compile(p, Pattern.CASE_INSENSITIVE); 
+        
+        // update (put) pattern to internal blacklist maps (for which source is active)
+        for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
+            if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistSourcefile)) {
+                final Map<String, Set<Pattern>> blacklistMap = getBlacklistMap(supportedBlacklistType, isMatchable(host));
+                Set<Pattern> hostList;
+                if (!(blacklistMap.containsKey(h) && ((hostList = blacklistMap.get(h)) != null))) {
+                    blacklistMap.put(h, (hostList = new HashSet<Pattern>()));
+                }
+                hostList.add(pattern);
+            }
+        }
+
         // Append the line to the file.
         PrintWriter pw = null;
         try {
@@ -433,6 +466,14 @@ public class Blacklist {
         return s != null && s.has(urlHash);
     }
 
+    /**
+     * Check blacklist to contain given host & path pattern.
+     * To check if a url matches a blacklist pattern, use isListed()
+     * @param blacklistType
+     * @param host
+     * @param path
+     * @return
+     */
     public final boolean contains(final BlacklistType blacklistType, final String host, final String path) {
         boolean ret = false;
 
@@ -444,7 +485,13 @@ public class Blacklist {
 
             final Set<Pattern> hostList = blacklistMap.get(h);
             if (hostList != null) {
-                ret = hostList.contains(path);
+                for (Pattern hp : hostList) {
+                    String hpxs = hp.pattern();
+                    if (hpxs.equals(path)) {
+                        ret = true;
+                        break;
+                    }
+                }
             }
         }
         return ret;

test/java/net/yacy/repository/BlacklistTest.java

@@ -0,0 +1,43 @@
+package net.yacy.repository;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import net.yacy.cora.document.id.Punycode;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class BlacklistTest {
+
+    /**
+     * Simulates contains method, of class Blacklist as proof for pattern.toString
+     * needed and works
+     */
+    @Test
+    public void testContains() throws Punycode.PunycodeException {
+        String path = ".*"; // simplest test pattern
+
+        Pattern pattern = Pattern.compile(path, Pattern.CASE_INSENSITIVE);
+
+        // pattern list as in Blacklist class
+        // ConcurrentMap<BlacklistType, Map<String, Set<Pattern>>> hostpaths_matchable;
+        // simulate last part, path pattern set
+        Set<Pattern> hostList = new HashSet<Pattern>();
+        hostList.add(pattern);
+
+        // proof assumption pattern(path) != path
+        boolean ret = hostList.contains(path);
+        assertFalse("match blacklist pattern " + path, ret);
+
+        // proof pattern.toString match works
+        for (Pattern hp : hostList) {
+            String hpxs = hp.pattern();
+            if (hpxs.equals(path)) {
+                ret = true;
+                break;
+            }
+        }
+        assertTrue("match blacklist pattern " + path, ret);
+    }
+
+}