*) Undoing robots parser policy changes from svn rev. 1421

- crawling is not allowed if server returned a 403 statuscode (according to rfc)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1864 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/data/robotsParser.java

@@ -348,7 +348,7 @@ public final class robotsParser{
                         "\nRedirecting request to: " + redirectionUrl);
                 return downloadRobotsTxt(redirectionUrl,redirectionCount,entry);
                 
-            } else if (res.status.startsWith("401")/* || res.status.startsWith("403") */) {
+            } else if (res.status.startsWith("401") || res.status.startsWith("403")) {
                 accessCompletelyRestricted = true;
                 serverLog.logFinest("ROBOTS","Access to Robots.txt not allowed on URL '" + robotsURL + "'.");
             } else {