From f243e338cfc791abb9d468b5b300d103be5b046d Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 6 Dec 2007 21:53:17 +0000 Subject: [PATCH] implemented online caution also for local and remote search git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4252 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/index.java | 5 ++++- htroot/yacy/search.java | 2 ++ htroot/yacysearch.java | 5 ++++- source/de/anomic/plasma/plasmaSwitchboard.java | 17 ++++++++++------- yacy.init | 6 ++++-- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/htroot/index.java b/htroot/index.java index 9dc9082ec..9cea0fe18 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -147,7 +147,10 @@ public class index { prop.put("contentdomCheckVideo", (contentdom == plasmaSearchQuery.CONTENTDOM_VIDEO) ? "1" : "0"); prop.put("contentdomCheckImage", (contentdom == plasmaSearchQuery.CONTENTDOM_IMAGE) ? "1" : "0"); prop.put("contentdomCheckApp", (contentdom == plasmaSearchQuery.CONTENTDOM_APP) ? "1" : "0"); - + + // online caution timing + sb.localSearchLastAccess = System.currentTimeMillis(); + return prop; } } diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index e30052f03..e1755f4a0 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -59,6 +59,8 @@ public final class search { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // return variable that accumulates replacements final plasmaSwitchboard sb = (plasmaSwitchboard) env; + sb.remoteSearchLastAccess = System.currentTimeMillis(); + serverObjects prop = new serverObjects(); if ((post == null) || (env == null)) return prop; if (!yacyNetwork.authentifyRequest(post, env)) return prop; diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 92f4f193a..0524cfcaf 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -73,7 +73,8 @@ public class yacysearch { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { final plasmaSwitchboard sb = (plasmaSwitchboard) env; - + sb.localSearchLastAccess = System.currentTimeMillis(); + boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header, false); boolean authenticated = sb.adminAuthenticated(header) >= 2; @@ -406,6 +407,8 @@ public class yacysearch { // for RSS: don't HTML encode some elements prop.putHTML("rss_query", querystring, true); prop.put("rss_queryenc", yacyURL.escape(querystring.replace(' ', '+'))); + + sb.localSearchLastAccess = System.currentTimeMillis(); // return rewrite properties return prop; diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 2e4fdd1d2..4f2a71d93 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -220,7 +220,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public HashMap outgoingCookies, incomingCookies; public kelondroMapTable facilityDB; public plasmaParser parser; - public long proxyLastAccess; + public long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess; public yacyCore yc; public HashMap indexingTasksInProcess; public userDB userDB; @@ -562,7 +562,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser *

public static final String PROXY_ONLINE_CAUTION_DELAY = "onlineCautionDelay"

*

Name of the setting how long indexing should pause after the last time the proxy was used in milliseconds

*/ - public static final String PROXY_ONLINE_CAUTION_DELAY = "onlineCautionDelay"; + public static final String PROXY_ONLINE_CAUTION_DELAY = "crawlPause.proxy"; + public static final String LOCALSEACH_ONLINE_CAUTION_DELAY = "crawlPause.localsearch"; + public static final String REMOTESEARCH_ONLINE_CAUTION_DELAY = "crawlPause.remotesearch"; /** *

public static final String PROXY_PREFETCH_DEPTH = "proxyPrefetchDepth"

*

Name of the setting how deep URLs fetched by proxy usage shall be followed

@@ -971,6 +973,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // setting timestamp of last proxy access this.proxyLastAccess = System.currentTimeMillis() - 60000; + this.localSearchLastAccess = System.currentTimeMillis() - 60000; + this.remoteSearchLastAccess = System.currentTimeMillis() - 60000; this.webStructure = new plasmaWebStructure(log, rankingPath, "LOCAL/010_cr/", getConfig("CRDist0Path", plasmaRankingDistribution.CR_OWN), new File(plasmaPath, "webStructure.map")); // configuring list path @@ -1507,11 +1511,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser } public boolean onlineCaution() { - try { - return System.currentTimeMillis() - proxyLastAccess < Integer.parseInt(getConfig(PROXY_ONLINE_CAUTION_DELAY, "30000")); - } catch (NumberFormatException e) { - return false; - } + return + (System.currentTimeMillis() - this.proxyLastAccess < Integer.parseInt(getConfig(PROXY_ONLINE_CAUTION_DELAY, "30000"))) || + (System.currentTimeMillis() - this.localSearchLastAccess < Integer.parseInt(getConfig(LOCALSEACH_ONLINE_CAUTION_DELAY, "30000"))) || + (System.currentTimeMillis() - this.remoteSearchLastAccess < Integer.parseInt(getConfig(REMOTESEARCH_ONLINE_CAUTION_DELAY, "30000"))); } private static String ppRamString(long bytes) { diff --git a/yacy.init b/yacy.init index 6b536fb45..d467c23db 100644 --- a/yacy.init +++ b/yacy.init @@ -714,8 +714,10 @@ msgForwardingEnabled=false msgForwardingCmd=/usr/sbin/sendmail msgForwardingTo=root@localhost -#onlineCautionDelay: delay time after proxy usage before crawling is resumed -onlineCautionDelay=10000 +#crawlPause: delay time after specific functions before crawling is resumed +crawlPause.proxy=15000 +crawlPause.localsearch=9000 +crawlPause.remotesearch=3000 # Some configuration values for the crawler crawler.clientTimeout=9000