diff --git a/source/de/anomic/crawler/robotsParser.java b/source/de/anomic/crawler/robotsParser.java index bb87e01f2..97b9648f0 100644 --- a/source/de/anomic/crawler/robotsParser.java +++ b/source/de/anomic/crawler/robotsParser.java @@ -109,7 +109,7 @@ public final class robotsParser { try { while ((line = reader.readLine()) != null) { // replacing all tabs with spaces - line = line.replaceAll("\t"," ").replaceAll(":"," ").trim(); + line = line.replaceAll("\t"," ").trim(); lineUpper = line.toUpperCase(); if (line.length() == 0) {