* fix problem in robotparser, see http://forum.yacy-websuche.de/viewtopic.php?f=5&t=1421&p=9742

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5137 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/crawler/robotsParser.java

@@ -108,7 +108,8 @@ public final class robotsParser {
         
         try {
             while ((line = reader.readLine()) != null) {
-                line = line.trim();
+                // replacing all tabs with spaces
+                line = line.replaceAll("\t"," ").replaceAll(":"," ").trim();
                 lineUpper = line.toUpperCase();
                 
                 if (line.length() == 0) {
@@ -137,9 +138,6 @@ public final class robotsParser {
                     pos = line.indexOf(ROBOTS_COMMENT);
                     if (pos != -1) line = line.substring(0,pos).trim();
                     
-                    // replacing all tabs with spaces
-                    line = line.replaceAll("\t"," ").replaceAll(":"," ");
-                    
                     // getting out the robots name
                     pos = line.indexOf(" ");
                     if (pos != -1) {
@@ -149,9 +147,6 @@ public final class robotsParser {
                         if (isRule4YaCyAgent) rule4YaCyFound = true;
                     }
                 } else if (lineUpper.startsWith(ROBOTS_CRAWL_DELAY)) {
-                    // replacing all tabs with spaces
-                    line = line.replaceAll("\t"," ").replaceAll(":"," ");
-                    
                     pos = line.indexOf(" ");
                     if (pos != -1) {
                     	try {
@@ -174,9 +169,6 @@ public final class robotsParser {
                         // cutting of tailing *
                         if (line.endsWith("*")) line = line.substring(0,line.length()-1);
                         
-                        // replacing all tabs with spaces
-                        line = line.replaceAll("\t"," ").replaceAll(":"," ");
-                        
                         // getting the path
                         pos = line.indexOf(" ");
                         if (pos != -1) {