diff --git a/build.properties b/build.properties index d7ef7f420..316860373 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.5 javacTarget=1.5 # Release Configuration -releaseVersion=0.566 +releaseVersion=0.567 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz diff --git a/htroot/PerformanceQueues_p.html b/htroot/PerformanceQueues_p.html index 460209729..588fb2007 100644 --- a/htroot/PerformanceQueues_p.html +++ b/htroot/PerformanceQueues_p.html @@ -205,14 +205,20 @@
Proxy Performance Settings

- This is the time that the crawler idles when the proxy is accessed. The delay is extended by this time - each time the proxy is accessed afterwards. This shall improve performance of the proxy throughput. - (current delta is #[onlineCautionDelayCurrent]# milliseconds since last proxy access.) + This is the time that the crawler idles when the proxy is accessed, or a local or remote search is done. + The delay is extended by this time each time the proxy is accessed afterwards. + This shall improve performance of the affected process (proxy or search). + (current delta is #[crawlPauseProxyCurrent]#/#[crawlPauseLocalsearchCurrent]#/#[crawlPauseRemotesearchCurrent]# + seconds since last proxy/local-search/remote-search access.)

-
:
-
-
+
:
+
+
:
+
+
:
+
+
Changes take effect immediately
diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 05311f353..ff843c691 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -227,9 +227,10 @@ public class PerformanceQueues_p { switchboard.setConfig("javastart_priority",post.get("YaCyPriority","0")); } - if ((post != null) && (post.containsKey("proxyControlSubmit"))) { - int onlineCautionDelay = post.getInt("onlineCautionDelay", 30000); - switchboard.setConfig("onlineCautionDelay", Integer.toString(onlineCautionDelay)); + if ((post != null) && (post.containsKey("onlineCautionSubmit"))) { + switchboard.setConfig(plasmaSwitchboard.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000))); + switchboard.setConfig(plasmaSwitchboard.LOCALSEACH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseLocalsearch", 30000))); + switchboard.setConfig(plasmaSwitchboard.REMOTESEARCH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseRemotesearch", 30000))); } // table cache settings @@ -248,8 +249,12 @@ public class PerformanceQueues_p { prop.put("wordCacheMaxCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 20000)); prop.put("wordCacheInitCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_INIT_COUNT, 30000)); prop.put("wordFlushSize", switchboard.getConfigLong("wordFlushSize", 2000)); - prop.put("onlineCautionDelay", switchboard.getConfigLong("onlineCautionDelay", 30000)); - prop.putNum("onlineCautionDelayCurrent", System.currentTimeMillis() - switchboard.proxyLastAccess); + prop.put("crawlPauseProxy", switchboard.getConfigLong(plasmaSwitchboard.PROXY_ONLINE_CAUTION_DELAY, 30000)); + prop.put("crawlPauseLocalsearch", switchboard.getConfigLong(plasmaSwitchboard.LOCALSEACH_ONLINE_CAUTION_DELAY, 30000)); + prop.put("crawlPauseRemotesearch", switchboard.getConfigLong(plasmaSwitchboard.REMOTESEARCH_ONLINE_CAUTION_DELAY, 30000)); + prop.putNum("crawlPauseProxyCurrent", (System.currentTimeMillis() - switchboard.proxyLastAccess) / 1000); + prop.putNum("crawlPauseLocalsearchCurrent", (System.currentTimeMillis() - switchboard.localSearchLastAccess) / 1000); + prop.putNum("crawlPauseRemotesearchCurrent", (System.currentTimeMillis() - switchboard.remoteSearchLastAccess) / 1000); // table thread pool settings prop.put("pool_0_name","Crawler Pool"); diff --git a/htroot/index.java b/htroot/index.java index 6cb15cb8f..f6c323e6b 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -68,26 +68,8 @@ public class index { final boolean indexDistributeGranted = sb.getConfigBool(plasmaSwitchboard.INDEX_DIST_ALLOW, true); final boolean indexReceiveGranted = sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, true); - global = global && indexDistributeGranted && indexReceiveGranted; -/* - final String referer = (String) header.get(httpHeader.REFERER); - if (referer != null) { - yacyURL url; - try { - url = new yacyURL(referer, null); - } catch (MalformedURLException e) { - url = null; - } - if ((url != null) && (!url.isLocal())) { - final HashMap referrerprop = new HashMap(); - referrerprop.put("count", "1"); - referrerprop.put("clientip", header.get(httpHeader.CONNECTION_PROP_CLIENTIP)); - referrerprop.put("useragent", header.get(httpHeader.USER_AGENT)); - referrerprop.put("date", (new serverDate()).toShortString(false)); - if (sb.facilityDB != null) try {sb.facilityDB.update("backlinks", referer, referrerprop);} catch (IOException e) {} - } - } -*/ + //global = global && indexDistributeGranted && indexReceiveGranted; + // search domain int contentdom = plasmaSearchQuery.CONTENTDOM_TEXT; String cds = (post == null) ? "text" : post.get("contentdom", "text"); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index b774f4daf..55cc16af9 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -91,22 +91,6 @@ public class yacysearch { boolean rss = (post == null) ? false : post.get("rss", "false").equals("true"); if ((post == null) || (env == null) || (querystring.length() == 0) || (!searchAllowed)) { - /* - // save referrer - final String referer = (String) header.get("Referer"); - if (referer != null) { - yacyURL url; - try { url = new yacyURL(referer, null); } catch (MalformedURLException e) { url = null; } - if ((url != null) && (!url.isLocal())) { - final HashMap referrerprop = new HashMap(); - referrerprop.put("count", "1"); - referrerprop.put("clientip", header.get("CLIENTIP")); - referrerprop.put("useragent", header.get("User-Agent")); - referrerprop.put("date", (new serverDate()).toShortString(false)); - if (sb.facilityDB != null) try { sb.facilityDB.update("backlinks", referer, referrerprop); } catch (IOException e) {} - } - } - */ // we create empty entries for template strings final serverObjects prop = new serverObjects(); prop.put("searchagain", "0"); @@ -169,13 +153,13 @@ public class yacysearch { } // SEARCH - final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true"); - final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true"); - final boolean offline = yacyCore.seedDB.mySeed().isVirgin(); + //final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true"); + //final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true"); + //final boolean offline = yacyCore.seedDB.mySeed().isVirgin(); final boolean clustersearch = sb.isRobinsonMode() && (sb.getConfig("cluster.mode", "").equals("privatecluster") || sb.getConfig("cluster.mode", "").equals("publiccluster")); - if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; } + //if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; } if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes // find search domain diff --git a/source/de/anomic/server/serverSystem.java b/source/de/anomic/server/serverSystem.java index 9f8afbc0c..f9d4529f1 100644 --- a/source/de/anomic/server/serverSystem.java +++ b/source/de/anomic/server/serverSystem.java @@ -50,6 +50,8 @@ import java.util.Hashtable; import java.util.Properties; import java.util.Vector; +import de.anomic.server.logging.serverLog; + public final class serverSystem { // constants for system identification @@ -330,6 +332,7 @@ public final class serverSystem { try { Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor(); } catch (InterruptedException e) { + serverLog.logSevere("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e); throw new IOException(e.getMessage()); } } diff --git a/yacy.init b/yacy.init index 47deece72..ac58b2a98 100644 --- a/yacy.init +++ b/yacy.init @@ -725,11 +725,11 @@ crawler.http.acceptEncoding=gzip crawler.http.acceptLanguage=en-us,en;q=0.5 crawler.http.acceptCharset=ISO-8859-1,utf-8;q=0.7,*;q=0.7 crawler.http.maxFileSize=262144 -crawler.http.maxFileSize__pro=1048576 +crawler.http.maxFileSize__pro=262144 # ftp crawler specific settings; size in bytes crawler.ftp.maxFileSize=262144 -crawler.ftp.maxFileSize__pro=1048576 +crawler.ftp.maxFileSize__pro=262144 # maximum number of crawler threads crawler.MaxActiveThreads = 30 @@ -903,27 +903,6 @@ tableTypeForPreNURL=0 publicSurftips = true publicSearchpage = true -# a Java Properties file containig a list of SOAP services that should deployed -# additionally to the default services. E.g. -# soap.serviceDeploymentList = DATA/SETTINGS/myServices.properties -# -# One entry in the property file should must have the following format: -# servicename=fullClassname -# -# e.g. -# test=org.myservices.test -# -# Servicename is the name that shoud be used to access the service, e.g. -# if the service name is "test" then the service can be reached using -# http://localhost:8080/soap/test -# -# The WSDL document that belongs to the deployed service could be reached -# using -# http://localhost:8080/soap/test?wsdl -# -soap.serviceDeploymentList = - - # Wiki access rights # the built-in wiki system allows by default only that the administrator is allowed to make changes # this can be changed. There are three options: