From 0aafb83edc7ac3bf9e3b0aa838f7ed5c3efb5830 Mon Sep 17 00:00:00 2001 From: theli Date: Wed, 7 Sep 2005 13:18:34 +0000 Subject: [PATCH] *) Bugfix for robots.txt isDisallowed Check. Setting path to "/" if it is null or empty. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@677 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/plasma/plasmaCrawlRobotsTxt.java | 4 ++-- source/de/anomic/plasma/plasmaCrawlWorker.java | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java index 1cad929d7..3f7c7bbaf 100644 --- a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java +++ b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java @@ -196,8 +196,8 @@ public class plasmaCrawlRobotsTxt { } public boolean isDisallowed(String path) { - if ((path == null) || (path.length() == 0)) return false; - if ((this.mem == null) || (this.disallowPathList.size() == 0)) return false; + if ((this.mem == null) || (this.disallowPathList.size() == 0)) return false; + if ((path == null) || (path.length() == 0)) path = "/"; Iterator pathIter = this.disallowPathList.iterator(); while (pathIter.hasNext()) { diff --git a/source/de/anomic/plasma/plasmaCrawlWorker.java b/source/de/anomic/plasma/plasmaCrawlWorker.java index 1f1a97341..74b985416 100644 --- a/source/de/anomic/plasma/plasmaCrawlWorker.java +++ b/source/de/anomic/plasma/plasmaCrawlWorker.java @@ -4,7 +4,9 @@ //(C) by Michael Peter Christen; mc@anomic.de //first published on http://www.anomic.de //Frankfurt, Germany, 2004 -//last major change: 21.04.2005 by Martin Thelian +// +//last major change: $LastChangedDate$ by $LastChangedBy$ +//Revision: $LastChangedRevision$ // //This program is free software; you can redistribute it and/or modify //it under the terms of the GNU General Public License as published by