From 34c075c1c71b3e5413f2a84e9d67d6e2aa57dbe9 Mon Sep 17 00:00:00 2001
From: theli <theli@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Tue, 14 Mar 2006 08:54:44 +0000
Subject: [PATCH] testcommit with subversive

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1886 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 source/de/anomic/data/robotsParser.java | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java
index b09992fc3..43c1b930e 100644
--- a/source/de/anomic/data/robotsParser.java
+++ b/source/de/anomic/data/robotsParser.java
@@ -65,13 +65,20 @@ import de.anomic.server.logging.serverLog;
 /*
  * A class for Parsing robots.txt files.
  * It only parses the Deny Part, yet.
- *  *
+ *  
+ * Robots RFC
  * http://www.robotstxt.org/wc/norobots-rfc.html
  * 
  * TODO:
  *      - On the request attempt resulted in temporary failure a robot
  *      should defer visits to the site until such time as the resource
  *      can be retrieved.
+ *      
+ *      - Extended Standard for Robot Exclusion 
+ *        See: http://www.conman.org/people/spc/robots2.html
+ *        
+ *      - Robot Exclusion Standard Revisited 
+ *        See: http://www.kollar.com/robots.html
  */
 public final class robotsParser{