code-split

allow cgi indexing

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5839 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/crawler/CrawlStacker.java

@@ -207,7 +207,7 @@ public final class CrawlStacker {
         }
 
         // deny cgi
-        if (entry.url().isCGI())  {
+        if (entry.url().isIndividual())  {
             if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' is CGI URL. " +
                              "Stack processing time: " + (System.currentTimeMillis() - startTime) + "ms");
             return "cgi url not allowed";

source/de/anomic/yacy/yacyURL.java

@@ -668,10 +668,14 @@ public class yacyURL implements Serializable {
 
     public final boolean isCGI() {
         final String ls = unescape(path.toLowerCase());
-        int pos;
         return ls.indexOf(".cgi") >= 0 ||
-               ls.indexOf(".exe") >= 0 ||
+               ls.indexOf(".exe") >= 0;
-
+    }
+    
+    public final boolean isIndividual() {
+        final String ls = unescape(path.toLowerCase());
+        int pos;
+        return
                ((pos = ls.indexOf("sid")) > 0 &&
                 (ls.charAt(--pos) == '?' || ls.charAt(pos) == '&' || ls.charAt(pos) == ';') &&
                 (pos += 5) < ls.length() &&