minor fixes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@659 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/data/robotsParser.java

@@ -104,9 +104,7 @@ public class robotsParser{
 			line=lines[i];
 			if(line.startsWith("Disallow:")){
 				line=line.substring(9);
-				if(line.startsWith(" ")){
-					line=line.substring(1);
-				}
+				line=line.trim()
 				deny.add(line);
 			}
 		}
@@ -120,7 +118,7 @@ public class robotsParser{
 			Iterator it=deny.iterator();
 			allowed=true;
 			while(it.hasNext()){
-				if(url.indexOf((String)it.next()) >= 0){
+				if(url.startsWith((String)it.next()){
 					allowed=false;
 					break;
 				}