*) blacklist cleaner supports usage of regular expressions now

*) refacored BlacklistCleaner_p.java for better readability
*) moved check of validity of patterns to the Balcklist implementation since patterns might be valid in one implementation, but not in another
*) added method to check validity to Blacklist interface
*) fixed some minor issues like typos or wrong whitespaces
*) set subversion properties for a whole bunch of files

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6359 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/BlacklistCleaner_p.html

@@ -17,6 +17,10 @@
           <select name="listNames" size="1">#{blacklists}# 
             <option value="#[name]#"#(selected)#:: selected="selected"#(/selected)#>#[name]#</option>#{/blacklists}#
           </select>
+            <br /><br />
+            <input type="checkbox" name="allowRegex" #(checked)#:: checked="checked"#(/checked)# />
+                   Allow regular expressions in host part of blacklist entries.
+            <br /><br />
           <input type="submit" name="list" value="Check" />
         </div>::
         <p class="error">The blacklist-cleaner only works for the following blacklist-engines up to now:</p>
@@ -40,11 +44,13 @@
           <dt>
             <label for="select#[entry]#" class="error">
               #(error)#Two wildcards in host-part
+              ::<!-- no error, should never be visible -->
               ::Either subdomain <u>or</u> wildcard
               ::Path is invalid Regex
               ::Wildcard not on begin or end
               ::Host contains illegal chars
               ::Double
+              ::Host is invalid Regex
               #(/error)#
             </label>
             <input type="checkbox" name="select#[entry]#" id="select#[entry]#" checked="checked" />

htroot/BlacklistCleaner_p.java

@@ -37,13 +37,11 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
 
 import de.anomic.data.AbstractBlacklist;
 import de.anomic.data.Blacklist;
@@ -54,6 +52,7 @@ import de.anomic.search.Switchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.yacy.logging.Log;
+import java.util.Set;
 
 public class BlacklistCleaner_p {
     
@@ -62,13 +61,6 @@ public class BlacklistCleaner_p {
     private static final String BLACKLISTS = "blacklists_";
     private static final String ENTRIES = "entries_";
         
-    private static final int ERR_TWO_WILDCARDS_IN_HOST = 0;
-    private static final int ERR_SUBDOMAIN_XOR_WILDCARD = 1;
-    private static final int ERR_PATH_REGEX = 2;
-    private static final int ERR_WILDCARD_BEGIN_OR_END = 3;
-    private static final int ERR_HOST_WRONG_CHARS = 4;
-    private static final int ERR_DOUBLE_OCCURANCE = 5;
-    
     private final static String BLACKLIST_FILENAME_FILTER = "^.*\\.black$";
     
     public static final Class<?>[] supportedBLEngines = {
@@ -87,16 +79,19 @@ public class BlacklistCleaner_p {
         final String supportedBlacklistTypesStr = AbstractBlacklist.BLACKLIST_TYPES_STRING;
         final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(","); 
 
-        if (post == null) {
+        prop.put(DISABLED+"checked", "1");
-            prop.put("results", "0");
+
-            putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
+        if (post != null) {
-            return prop;
+
-        }
+            final boolean allowRegex = post.get("allowRegex", "off").equalsIgnoreCase("on") ? true: false;
+            prop.put(DISABLED+"checked", (allowRegex) ? "1" : "0");
 
             if (post.containsKey("listNames")) {
                 blacklistToUse = post.get("listNames");
-            if (blacklistToUse.length() == 0 || !listManager.listSetContains("listManager.listsPath", blacklistToUse))
+                if (blacklistToUse.length() == 0 || !listManager.listSetContains("listManager.listsPath", blacklistToUse)) {
                     prop.put("results", "2");
+
+                }
             }
 
             putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
@@ -106,34 +101,45 @@ public class BlacklistCleaner_p {
 
                 if (post.containsKey("delete")) {
                     prop.put(RESULTS + "modified", "1");
-                prop.put(RESULTS + "modified_delCount", removeEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select", true, true)));
+                    prop.put(RESULTS + "modified_delCount", removeEntries(blacklistToUse, supportedBlacklistTypes, getKeysByPrefix(post, "select", true)));
                 } else if (post.containsKey("alter")) {
                     prop.put(RESULTS + "modified", "2");
-                prop.put(RESULTS + "modified_alterCount", alterEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select", true, false), getByPrefix(post, "entry", false, false)));
+                    prop.put(RESULTS + "modified_alterCount", alterEntries(blacklistToUse, supportedBlacklistTypes, getKeysByPrefix(post, "select", false), getValuesByPrefix(post, "entry", false)));
                 }
 
                 // list illegal entries
-            final HashMap<String, Integer> ies = getIllegalEntries(blacklistToUse, Switchboard.urlBlacklist);
+                final Map<String, Integer> illegalEntries = getIllegalEntries(blacklistToUse, Switchboard.urlBlacklist, allowRegex);
                 prop.put(RESULTS + "blList", blacklistToUse);
-            prop.put(RESULTS + "entries", ies.size());
+                prop.put(RESULTS + "entries", illegalEntries.size());
                 prop.putHTML(RESULTS + "blEngine", Switchboard.urlBlacklist.getEngineInfo());
-            prop.put(RESULTS + "disabled", (ies.size() == 0) ? "1" : "0");
+                prop.put(RESULTS + "disabled", (illegalEntries.size() == 0) ? "1" : "0");
-            if (ies.size() > 0) {
+                if (illegalEntries.size() > 0) {
-                prop.put(RESULTS + DISABLED + "entries", ies.size());
+                    prop.put(RESULTS + DISABLED + "entries", illegalEntries.size());
                     int i = 0;
-                String s;
+                    String key;
-                for (Entry<String, Integer> entry: ies.entrySet()) {
+                    for (Entry<String, Integer> entry : illegalEntries.entrySet()) {
-                    s = entry.getKey();
+                        key = entry.getKey();
                         prop.put(RESULTS + DISABLED + ENTRIES + i + "_error", entry.getValue().longValue());
-                    prop.putHTML(RESULTS + DISABLED + ENTRIES + i + "_entry", s);
+                        prop.putHTML(RESULTS + DISABLED + ENTRIES + i + "_entry", key);
                         i++;
                     }
                 }
             }
+        } else {
+            prop.put("results", "0");
+            putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
+        }
         
         return prop;
     }
 
+    /**
+     * Adds a list of blacklist to the server objects properties which are used to
+     * display the blacklist in the HTML page belonging to this servlet.
+     * @param prop Server objects properties object.
+     * @param lists List of blacklists.
+     * @param selected Element in list of blacklists which will be preselected in HTML.
+     */
     private static void putBlacklists(final serverObjects prop, final List<String> lists, final String selected) {
         boolean supported = false;
         for (int i=0; i < supportedBLEngines.length && !supported; i++) {
@@ -162,6 +168,41 @@ public class BlacklistCleaner_p {
         }
     }
 
+    /**
+     * Retrieves all keys with a certain prefix from the data which has been sent and returns them as an array. This
+     * method is only a wrapper for {@link getByPrefix(de.anomic.server.serverObjects, java.lang.String, boolean, boolean)}
+     * which has been created to make it easier to understand the code.
+     * @param post All POST values.
+     * @param prefix Prefix by which the input is filtered.
+     * @param filterDoubles Set true if only unique results shall be returned, else false.
+     * @return Keys which have been posted.
+     */
+    private static String[] getKeysByPrefix(final serverObjects post, final String prefix, final boolean filterDoubles) {
+        return getByPrefix(post, prefix, true, filterDoubles);
+    }
+
+    /**
+     * Retrieves all values with a certain prefix from the data which has been sent and returns them as an array. This
+     * method is only a wrapper for {@link getByPrefix(de.anomic.server.serverObjects, java.lang.String, boolean, boolean)}.
+     * @param post All POST values.
+     * @param prefix Prefix by which the input is filtered.
+     * @param filterDoubles Set true if only unique results shall be returned, else false.
+     * @return Values which have been posted.
+     */
+    private static String[] getValuesByPrefix(final serverObjects post, final String prefix, final boolean filterDoubles) {
+        return getByPrefix(post, prefix, false, filterDoubles);
+    }
+
+    /**
+     * Method which does all the work for {@link getKeysByPrefix(de.anomic.server.serverObjects, java.lang.String prefix, boolean)}
+     * and {@link getValuesByPrefix(de.anomic.server.serverObjects, java.lang.String prefix, boolean)} which
+     * have been crested to make it easier to understand the code.
+     * @param post
+     * @param prefix
+     * @param useKeys
+     * @param useHashSet
+     * @return
+     */
     private static String[] getByPrefix(final serverObjects post, final String prefix, final boolean useKeys, final boolean useHashSet) {
         Collection<String> r;
         if (useHashSet) {
@@ -170,19 +211,14 @@ public class BlacklistCleaner_p {
             r = new ArrayList<String>();
         }
 
-        String s;
         if (useKeys) {
-            final Iterator<String> it =  post.keySet().iterator();
+            for (String entry : post.keySet()) {
-            while (it.hasNext()) {
+                if (entry.indexOf(prefix) == 0) {
-                if ((s = it.next()).indexOf(prefix) == 0) {
+                    r.add(entry.substring(prefix.length()));
-                    r.add(s.substring(prefix.length()));
                 }
             }
         } else {
-            final Iterator<Map.Entry<String, String>> it = post.entrySet().iterator();
+            for (Map.Entry<String, String> entry : post.entrySet()) {
-            Map.Entry<String, String> entry;
-            while (it.hasNext()) {
-                entry = it.next();
                 if (entry.getKey().indexOf(prefix) == 0) {
                     r.add(entry.getValue());
                 }
@@ -192,78 +228,55 @@ public class BlacklistCleaner_p {
         return r.toArray(new String[r.size()]);
     }
 
-    private static HashMap<String, Integer>/* entry, error-code */ getIllegalEntries(final String blacklistToUse, final Blacklist blEngine) {
+    /**
-        final HashMap<String, Integer> r = new HashMap<String, Integer>();
+     * Finds illegal entries in black list.
-        final HashSet<String> ok = new HashSet<String>();
+     * @param blacklistToUse The blacklist to be checked.
+     * @param blEngine The blacklist engine which is used to check
+     * @param allowRegex Set to true to allow regular expressions in host part of blacklist entry.
+     * @return A map which contains all entries whoch have been identified as being
+     * illegal by the blacklistEngine with the entry as key and an error code as
+     * value.
+     */
+    private static Map<String, Integer> getIllegalEntries(final String blacklistToUse, final Blacklist blEngine, final boolean allowRegex) {
+        final Map<String, Integer> illegalEntries = new HashMap<String, Integer>();
+        final Set<String> legalEntries = new HashSet<String>();
         
-        final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
+        final List<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
-        final Iterator<String> it = list.iterator();
+        final Map<String, String> properties= new HashMap<String, String>();
-        String s, host, path;
+        properties.put("allowRegex", String.valueOf(allowRegex));
                 
-        if (blEngine instanceof DefaultBlacklist) {
+        if (blEngine instanceof AbstractBlacklist) {
-            int slashPos;
-            while (it.hasNext()) {
-                s = (it.next()).trim();
 
-                // check for double-occurance
+            int err = 0;
-                if (ok.contains(s)) {
-                    r.put(s, Integer.valueOf(ERR_DOUBLE_OCCURANCE));
-                    continue;
-                }
-                ok.add(s);
 
-                if ((slashPos = s.indexOf("/")) == -1) {
+            for (String element : list) {
-                    host = s;
+                element = element.trim();
-                    path = ".*";
-                } else {
-                    host = s.substring(0, slashPos);
-                    path = s.substring(slashPos + 1);
-                }
-                
-                final int i = host.indexOf("*");
-                
-                // check whether host begins illegally
-                if (!host.matches("([A-Za-z0-9_-]+|\\*)(\\.([A-Za-z0-9_-]+|\\*))*")) {
-                    if (i == 0 && host.length() > 1 && host.charAt(1) != '.') {
-                        r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD));
-                        continue;
-                    }
-                    r.put(s, Integer.valueOf(ERR_HOST_WRONG_CHARS));
-                    continue;
-                }
                 
-                // in host-part only full sub-domains may be wildcards
+                // check for double-occurance
-                if (host.length() > 0 && i > -1) {
+                if (legalEntries.contains(element)) {
-                    if (!(i == 0 || i == host.length() - 1)) {
+                    illegalEntries.put(element, Integer.valueOf(AbstractBlacklist.ERR_DOUBLE_OCCURANCE));
-                        r.put(s, Integer.valueOf(ERR_WILDCARD_BEGIN_OR_END));
                     continue;
                 }
+                legalEntries.add(element);
 
-                    if (i == host.length() - 1 && host.length() > 1 && host.charAt(i - 1) != '.') {
+                err = blEngine.checkError(element, properties);
-                        r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD));
-                        continue;
-                    }
-                }
 
-                // check for double-occurences of "*" in host
+                if (err > 0) {
-                if (host.indexOf("*", i + 1) > -1) {
+                    illegalEntries.put(element, err);
-                    r.put(s, Integer.valueOf(ERR_TWO_WILDCARDS_IN_HOST));
-                    continue;
-                }
-                
-                // check for errors on regex-compiling path
-                try {
-                    Pattern.compile(path);
-                } catch (final PatternSyntaxException e) {
-                    r.put(s, Integer.valueOf(ERR_PATH_REGEX));
-                    continue;
                 }
             }
         }
 
-        return r;
+        return illegalEntries;
     }
 
+    /**
+     * Removes existing entries from a blacklist.
+     * @param blacklistToUse The blacklist which contains the
+     * @param supportedBlacklistTypes Types of blacklists which the entry is to changed in.
+     * @param entries Array of entries to be deleted.
+     * @return Length of the list of entries to be removed.
+     */
     private static int removeEntries(final String blacklistToUse, final String[] supportedBlacklistTypes, final String[] entries) {
         // load blacklist data from file
         final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
@@ -310,24 +323,32 @@ public class BlacklistCleaner_p {
         return entries.length;
     }
 
+    /**
+     * Changes existing entry in a blacklist.
+     * @param blacklistToUse The blacklist which contains the entry.
+     * @param supportedBlacklistTypes Types of blacklists which the entry is to changed in.
+     * @param oldEntry Entry to be changed.
+     * @param newEntry Changed entry.
+     * @return The length of the new entry.
+     */
     private static int alterEntries(
             final String blacklistToUse,
             final String[] supportedBlacklistTypes,
-            final String[] oldE,
+            final String[] oldEntry,
-            final String[] newE) {
+            final String[] newEntry) {
-        removeEntries(blacklistToUse, supportedBlacklistTypes, oldE);
+        removeEntries(blacklistToUse, supportedBlacklistTypes, oldEntry);
         PrintWriter pw = null;
         try {
             pw = new PrintWriter(new FileWriter(new File(listManager.listsPath, blacklistToUse), true));
             String host, path;
-            for (int i=0, pos; i<newE.length; i++) {
+            for (int i=0, pos; i<newEntry.length; i++) {
-                pos = newE[i].indexOf("/");
+                pos = newEntry[i].indexOf("/");
                 if (pos < 0) {
-                    host = newE[i];
+                    host = newEntry[i];
                     path = ".*";
                 } else {
-                    host = newE[i].substring(0, pos);
+                    host = newEntry[i].substring(0, pos);
-                    path = newE[i].substring(pos + 1);
+                    path = newEntry[i].substring(pos + 1);
                 }
                 pw.println(host + "/" + path);
                 for (int blTypes = 0; blTypes < supportedBlacklistTypes.length; blTypes++) {
@@ -343,6 +364,6 @@ public class BlacklistCleaner_p {
         } catch (final IOException e) {
             Log.logSevere("BLACKLIST-CLEANER", "error on writing altered entries to blacklist", e);
         }
-        return newE.length;
+        return newEntry.length;
     }
 }

htroot/Blacklist_p.html

@@ -74,7 +74,7 @@
           <input type="text" name="newEntry" size="50" />
           <input type="submit" name="addBlacklistEntry" value="Add URL pattern" />
         </div>
-        <p>The right '*', after the '/', can replaced by a <a href="http://java.sun.com/j2se/1.5.0/docs/api/java/util/regex/Pattern.html">regex</a>.</p>
+        <p>The right '*', after the '/', can be replaced by a <a href="http://java.sun.com/j2se/1.5.0/docs/api/java/util/regex/Pattern.html">regex</a>.</p>
         <ul>
           <li>domain.net/fullpath</li>
           <li>domain.net/*</li>

htroot/Blacklist_p.java

@@ -43,7 +43,6 @@ import de.anomic.data.AbstractBlacklist;
 import de.anomic.data.Blacklist;
 import de.anomic.data.listManager;
 import de.anomic.http.metadata.RequestHeader;
-import de.anomic.search.SearchEventCache;
 import de.anomic.search.Switchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
@@ -65,9 +64,6 @@ public class Blacklist_p {
         listManager.switchboard = (Switchboard) env;
         listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
         
-        // clean up all search events in case that a (new) blacklist entry denies previously returned results
-        SearchEventCache.cleanupEvents(true);
-        
         // getting the list of supported blacklist types
         final String supportedBlacklistTypesStr = AbstractBlacklist.BLACKLIST_TYPES_STRING;
         final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");

htroot/IndexImportWikimedia_p.java

@@ -4,9 +4,9 @@
 // first published 04.05.2009 on http://yacy.net
 // Frankfurt, Germany
 //
-// $LastChangedDate: 2009-04-16 17:29:00 +0200 (Do, 16 Apr 2009) $
+// $LastChangedDate$
-// $LastChangedRevision: 5812 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by

htroot/IndexImport_p.java

@@ -7,9 +7,9 @@
 //
 //This file is contributed by Martin Thelian
 //
-// $LastChangedDate: 2005-10-17 17:46:12 +0200 (Mo, 17 Okt 2005) $
+// $LastChangedDate$
-// $LastChangedRevision: 947 $
+// $LastChangedRevision$
-// $LastChangedBy: borg-0300 $
+// $LastChangedBy$
 //
 //This program is free software; you can redistribute it and/or modify
 //it under the terms of the GNU General Public License as published by

htroot/LogStatistics_p.java

@@ -8,9 +8,9 @@
 //
 // This File is contributed by Franz Brausze
 //
-// $LastChangedDate: 2007-01-17 12:00:00 +0100 (Di, 17 Jan 2007) $
+// $LastChangedDate$
-// $LastChangedRevision: 3216 $
+// $LastChangedRevision$
-// $LastChangedBy: karlchenofhell $
+// $LastChangedBy$
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by

htroot/PerformanceGraph.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

htroot/PerformanceSearch_p.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

htroot/Ranking_p.java

@@ -5,9 +5,9 @@
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2007-07-19 22:11:48 +0000 (Do, 19 Jul 2007) $
+// $LastChangedDate$
-// $LastChangedRevision: 3995 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

htroot/SearchEventPicture.java

@@ -6,9 +6,9 @@
 // Frankfurt, Germany, 2005
 // Created 24.10.2005
 //
-// $LastChangedDate: 2005-10-23 19:50:27 +0200 (Sun, 23 Oct 2005) $
+// $LastChangedDate$
-// $LastChangedRevision: 976 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by

htroot/Supporter.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

htroot/Surftips.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

htroot/Threaddump_p.java

@@ -7,9 +7,9 @@
 //
 // This File is contributed by Alexander Fieger
 //
-// $LastChangedDate: 2008-01-22 12:51:43 +0100 (Di, 22 Jan 2008) $
+// $LastChangedDate$
-// $LastChangedRevision: 4374 $
+// $LastChangedRevision$
-// $LastChangedBy: low012 $
+// $LastChangedBy$
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by

source/de/anomic/data/AbstractBlacklist.java

@@ -1,13 +1,13 @@
-// indexAbstractReference.java
+// AbstractBlacklist.java
 // first published on http://www.yacy.net
 // (C) 2007 by Bjoern Krombholz
 // last major change: 12. August 2006 (theli) ?
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2009-01-30 23:44:20 +0100 (Fr, 30 Jan 2009) $
+// $LastChangedDate$
-// $LastChangedRevision: 5543 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 
@@ -41,10 +41,19 @@ import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
 import de.anomic.kelondro.util.SetTools;
+import de.anomic.search.SearchEventCache;
 import de.anomic.yacy.yacyURL;
 
 public abstract class AbstractBlacklist implements Blacklist {
 
+    public static final int ERR_TWO_WILDCARDS_IN_HOST = 1;
+    public static final int ERR_SUBDOMAIN_XOR_WILDCARD = 2;
+    public static final int ERR_PATH_REGEX = 3;
+    public static final int ERR_WILDCARD_BEGIN_OR_END = 4;
+    public static final int ERR_HOST_WRONG_CHARS = 5;
+    public static final int ERR_DOUBLE_OCCURANCE = 6;
+    public static final int ERR_HOST_REGEX = 7;
+
     protected static final HashSet<String> BLACKLIST_TYPES = new HashSet<String>(Arrays.asList(new String[]{
             Blacklist.BLACKLIST_CRAWLER,
             Blacklist.BLACKLIST_PROXY,
@@ -117,6 +126,9 @@ public abstract class AbstractBlacklist implements Blacklist {
         for(final Set<String> entry: this.cachedUrlHashs.values()) {
             entry.clear();
         }
+
+        // clean up all search events in case that an old blacklist entry denied previously returned results, but does not anymore
+        SearchEventCache.cleanupEvents(true);
     }
 
     public int size() {
@@ -178,6 +190,8 @@ public abstract class AbstractBlacklist implements Blacklist {
                     }
                 }
             }
+            // clean up all search events in case that a (new) blacklist entry denies previously returned results
+            SearchEventCache.cleanupEvents(true);
         }
     }
     
@@ -191,6 +205,9 @@ public abstract class AbstractBlacklist implements Blacklist {
     public void removeAll(final String blacklistType, final String host) {
         getBlacklistMap(blacklistType,true).remove(host);
         getBlacklistMap(blacklistType,false).remove(host);
+
+        // clean up all search events in case that an old blacklist entry denied previously returned results, but does not anymore
+        SearchEventCache.cleanupEvents(true);
     }
     
     public void remove(final String blacklistType, final String host, final String path) {
@@ -209,6 +226,9 @@ public abstract class AbstractBlacklist implements Blacklist {
             if (hostList.size() == 0)
                 blacklistMapNotMatch.remove(host);
         }
+
+        // clean up all search events in case that an old blacklist entry denied previously returned results, but does not anymore
+        SearchEventCache.cleanupEvents(true);
     }
 
     public void add(final String blacklistType, String host, String path) {
@@ -227,6 +247,9 @@ public abstract class AbstractBlacklist implements Blacklist {
         ArrayList<String> hostList = blacklistMap.get(host.toLowerCase());
         if (hostList == null) blacklistMap.put(host.toLowerCase(), (hostList = new ArrayList<String>()));
         hostList.add(path);
+
+        // clean up all search events in case that a (new) blacklist entry denies previously returned results
+        SearchEventCache.cleanupEvents(true);
     }
 
     public int blacklistCacheSize() {
@@ -273,6 +296,7 @@ public abstract class AbstractBlacklist implements Blacklist {
         }        
         return true;  
     }
+    
     public static boolean isMatchable (final String host) {
         try {
             if(Pattern.matches("^[a-z0-9.-]*$", host)) // simple Domain (yacy.net or www.yacy.net)

source/de/anomic/data/Blacklist.java

@@ -1,12 +1,12 @@
-// indexReferenceBlacklist.java
+// Blacklist.java
 // (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
 // first published 26.03.2008 on http://yacy.net
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 
@@ -29,6 +29,7 @@ package de.anomic.data;
 import java.io.File;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.Map;
 
 import de.anomic.yacy.yacyURL;
 
@@ -95,4 +96,6 @@ public interface Blacklist {
     
     public boolean isListed(String blacklistType, String hostlow, String path);
 
+    public int checkError(String entry, Map<String, String> properties);
+
 }

source/de/anomic/data/Coordinates.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

source/de/anomic/data/DefaultBlacklist.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2008-08-20 09:54:56 +0200 (Mi, 20 Aug 2008) $
+// $LastChangedDate$
-// $LastChangedRevision: 5063 $
+// $LastChangedRevision$
-// $LastChangedBy: danielr $
+// $LastChangedBy$
 //
 // LICENSE
 // 
@@ -29,6 +29,7 @@ package de.anomic.data;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
@@ -117,4 +118,77 @@ public class DefaultBlacklist extends AbstractBlacklist implements Blacklist {
         }
         return matched;
     }
+
+    public int checkError(String element, Map<String, String> properties) {
+
+        boolean allowRegex = true;
+        int slashPos;
+        String host, path;
+
+        if (properties != null) {
+            allowRegex = properties.get("allowRegex").equalsIgnoreCase("true") ? true : false;
+        }
+
+        if ((slashPos = element.indexOf("/")) == -1) {
+            host = element;
+            path = ".*";
+        } else {
+            host = element.substring(0, slashPos);
+            path = element.substring(slashPos + 1);
+        }
+
+        if (!allowRegex || !isValidRegex(host)) {
+            final int i = host.indexOf("*");
+
+            // check whether host begins illegally
+            if (!host.matches("([A-Za-z0-9_-]+|\\*)(\\.([A-Za-z0-9_-]+|\\*))*")) {
+                if (i == 0 && host.length() > 1 && host.charAt(1) != '.') {
+                    return ERR_SUBDOMAIN_XOR_WILDCARD;
+                }
+                return ERR_HOST_WRONG_CHARS;
+            }
+
+            // in host-part only full sub-domains may be wildcards
+            if (host.length() > 0 && i > -1) {
+                if (!(i == 0 || i == host.length() - 1)) {
+                    return  ERR_WILDCARD_BEGIN_OR_END;
+                }
+
+                if (i == host.length() - 1 && host.length() > 1 && host.charAt(i - 1) != '.') {
+                    return ERR_SUBDOMAIN_XOR_WILDCARD;
+                }
+            }
+
+            // check for double-occurences of "*" in host
+            if (host.indexOf("*", i + 1) > -1) {
+                return ERR_TWO_WILDCARDS_IN_HOST;
+            }
+        } else if (allowRegex && !isValidRegex(host)) {
+            return ERR_HOST_REGEX;
+        }
+
+        // check for errors on regex-compiling path
+        if (!isValidRegex(path) && !path.equals("*")) {
+            return ERR_PATH_REGEX;
+        }
+
+        return 0;
+    }
+
+    /**
+     * Checks if a given expression is a valid regular expression.
+     * @param expression The expression to be checked.
+     * @return True if the expression is a valid regular expression, else false.
+     */
+    private static boolean isValidRegex(String expression) {
+        boolean ret = true;
+        try {
+            Pattern.compile(expression);
+        } catch (final PatternSyntaxException e) {
+
+            ret = false;
+        }
+        return ret;
+    }
+
 }

source/de/anomic/data/DidYouMeanLibrary.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

source/de/anomic/data/LibraryProvider.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

source/de/anomic/data/Location.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

source/de/anomic/data/OpenGeoDB.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

source/de/anomic/data/URLAnalysis.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2009-01-02 12:38:20 +0100 (Fr, 02 Jan 2009) $
+// $LastChangedDate$
-// $LastChangedRevision: 5432 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

source/de/anomic/data/URLLicense.java

@@ -4,9 +4,9 @@
 //
 // This is a part of YaCy, a peer-to-peer based web search engine
 //
-// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedDate$
-// $LastChangedRevision: 1986 $
+// $LastChangedRevision$
-// $LastChangedBy: orbiter $
+// $LastChangedBy$
 //
 // LICENSE
 // 

source/de/anomic/data/diff.java

@@ -8,9 +8,9 @@
 //
 // This file is contributed by Franz Brausze
 //
-// $LastChangedDate: $
+// $LastChangedDate$
-// $LastChangedRevision: $
+// $LastChangedRevision$
-// $LastChangedBy: $
+// $LastChangedBy$
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by