diff --git a/source/de/anomic/crawler/RobotsTxtParser.java b/source/de/anomic/crawler/RobotsTxtParser.java index fa0a16964..986edfd3b 100644 --- a/source/de/anomic/crawler/RobotsTxtParser.java +++ b/source/de/anomic/crawler/RobotsTxtParser.java @@ -78,13 +78,17 @@ public final class RobotsTxtParser { private final Set myNames; // a list of own name lists private String agentName; // the name of the agent that was used to return the result - protected RobotsTxtParser(final byte[] robotsTxt, final Set myNames) { + protected RobotsTxtParser(final Set myNames) { this.allowList = new ArrayList(0); this.denyList = new ArrayList(0); this.sitemap = ""; this.crawlDelayMillis = 0; this.myNames = myNames; this.agentName = null; + } + + protected RobotsTxtParser(final Set myNames, final byte[] robotsTxt) { + this(myNames); if (robotsTxt != null && robotsTxt.length != 0) { final ByteArrayInputStream bin = new ByteArrayInputStream(robotsTxt); final BufferedReader reader = new BufferedReader(new InputStreamReader(bin));