testcommit with subversive

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1886 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 19 years ago
parent f588c0724f
commit 34c075c1c7

@ -65,13 +65,20 @@ import de.anomic.server.logging.serverLog;
/*
* A class for Parsing robots.txt files.
* It only parses the Deny Part, yet.
* *
*
* Robots RFC
* http://www.robotstxt.org/wc/norobots-rfc.html
*
* TODO:
* - On the request attempt resulted in temporary failure a robot
* should defer visits to the site until such time as the resource
* can be retrieved.
*
* - Extended Standard for Robot Exclusion
* See: http://www.conman.org/people/spc/robots2.html
*
* - Robot Exclusion Standard Revisited
* See: http://www.kollar.com/robots.html
*/
public final class robotsParser{

Loading…
Cancel
Save