From ebc39a7b9a110b29861412c6b7c716e16ac45a51 Mon Sep 17 00:00:00 2001
From: allo <allo@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Mon, 5 Sep 2005 10:34:26 +0000
Subject: [PATCH] minor fixes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@659 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 source/de/anomic/data/robotsParser.java | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java
index d3536940e..94c9e195b 100644
--- a/source/de/anomic/data/robotsParser.java
+++ b/source/de/anomic/data/robotsParser.java
@@ -104,9 +104,7 @@ public class robotsParser{
 			line=lines[i];
 			if(line.startsWith("Disallow:")){
 				line=line.substring(9);
-				if(line.startsWith(" ")){
-					line=line.substring(1);
-				}
+				line=line.trim()
 				deny.add(line);
 			}
 		}
@@ -120,7 +118,7 @@ public class robotsParser{
 			Iterator it=deny.iterator();
 			allowed=true;
 			while(it.hasNext()){
-				if(url.indexOf((String)it.next()) >= 0){
+				if(url.startsWith((String)it.next()){
 					allowed=false;
 					break;
 				}