From 915812f597b1f05c694a2e3d30d5ecac2ee0aec7 Mon Sep 17 00:00:00 2001
From: theli <theli@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Thu, 9 Mar 2006 10:17:37 +0000
Subject: [PATCH] *) Undoing robots parser policy changes from svn rev. 1421   
 - crawling is not allowed if server returned a 403 statuscode (according to
 rfc)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1864 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 source/de/anomic/data/robotsParser.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java
index 1a8a9d541..a92fbaf1d 100644
--- a/source/de/anomic/data/robotsParser.java
+++ b/source/de/anomic/data/robotsParser.java
@@ -348,7 +348,7 @@ public final class robotsParser{
                         "\nRedirecting request to: " + redirectionUrl);
                 return downloadRobotsTxt(redirectionUrl,redirectionCount,entry);
                 
-            } else if (res.status.startsWith("401")/* || res.status.startsWith("403") */) {
+            } else if (res.status.startsWith("401") || res.status.startsWith("403")) {
                 accessCompletelyRestricted = true;
                 serverLog.logFinest("ROBOTS","Access to Robots.txt not allowed on URL '" + robotsURL + "'.");
             } else {