From bea6c1313946d3157ec51387c9c1b0ef3242a972 Mon Sep 17 00:00:00 2001
From: f1ori <f1ori@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Fri, 12 Sep 2008 09:06:38 +0000
Subject: [PATCH] * with r5137 robotParser didn't work at all -> fix

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5139 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 source/de/anomic/crawler/robotsParser.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/de/anomic/crawler/robotsParser.java b/source/de/anomic/crawler/robotsParser.java
index bb87e01f2..97b9648f0 100644
--- a/source/de/anomic/crawler/robotsParser.java
+++ b/source/de/anomic/crawler/robotsParser.java
@@ -109,7 +109,7 @@ public final class robotsParser {
         try {
             while ((line = reader.readLine()) != null) {
                 // replacing all tabs with spaces
-                line = line.replaceAll("\t"," ").replaceAll(":"," ").trim();
+                line = line.replaceAll("\t"," ").trim();
                 lineUpper = line.toUpperCase();
                 
                 if (line.length() == 0) {