Robots.txt is now reused after one day. See forum-topic:

http://forum.yacy-websuche.de/viewtopic.php?f=5&t=1669&p=13565#p13565 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5772 6c8d7289-2bf4-0310-a012-ef5d649a1542
16 years ago · 1f37cc6107
parent f21a8c9e9c
commit 1f37cc6107
1 changed files with 1 additions and 1 deletions
--- a/source/de/anomic/crawler/RobotsTxt.java
+++ b/source/de/anomic/crawler/RobotsTxt.java
@ -150,7 +150,7 @@ public class RobotsTxt {
                }
                if (robotsTxt4Host != null &&
                    robotsTxt4Host.getLoadedDate() != null &&
-                    System.currentTimeMillis() - robotsTxt4Host.getLoadedDate().getTime() <= 7*24*60*60*1000) {
+                    System.currentTimeMillis() - robotsTxt4Host.getLoadedDate().getTime() <= 1*24*60*60*1000) {
                    return robotsTxt4Host;
                }