boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fix for online caution delay settings, see

Browse Source 
http://forum.yacy-websuche.de/viewtopic.php?f=6&t=738&p=4723#p4723
- removed remote search limitation for non-dht-peers according to discussion in
  http://forum.yacy-websuche.de/viewtopic.php?f=15&t=793&hilit=&p=5277#p5277

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4438 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent f35a3794e0
commit 3c7b94c119
7 changed files with 34 additions and 75 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5
# Release Configuration
releaseVersion=0.566
releaseVersion=0.567
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

18
htroot/PerformanceQueues_p.html
Unescape View File

@ -205,14 +205,20 @@
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<fieldset><legend>Proxy Performance Settings</legend>
<p>
This is the time that the crawler idles when the proxy is accessed. The delay is extended by this time
each time the proxy is accessed afterwards. This shall improve performance of the proxy throughput.
(current delta is #[onlineCautionDelayCurrent]# milliseconds since last proxy access.)
This is the time that the crawler idles when the proxy is accessed, or a local or remote search is done.
The delay is extended by this time each time the proxy is accessed afterwards.
This shall improve performance of the affected process (proxy or search).
(current delta is #[crawlPauseProxyCurrent]#/#[crawlPauseLocalsearchCurrent]#/#[crawlPauseRemotesearchCurrent]#
seconds since last proxy/local-search/remote-search access.)
</p>
<dl>
<dt><label for="onlineCautionDelay">Online Caution Delay (milliseconds)</label>:</dt>
<dd><input id="onlineCautionDelay" name="onlineCautionDelay" type="text" size="20" maxlength="100" value="#[onlineCautionDelay]#" /></dd>
<dt><input type="submit" name="proxyControlSubmit" value="Enter New Parameters" /></dt>
<dt><label for="crawlPauseProxy">Online Caution Delay for Proxy (milliseconds)</label>:</dt>
<dd><input id="crawlPauseProxy" name="crawlPauseProxy" type="text" size="20" maxlength="100" value="#[crawlPauseProxy]#" /></dd>
<dt><label for="crawlPauseLocalsearch">Online Caution Delay for Local Search (milliseconds)</label>:</dt>
<dd><input id="crawlPauseLocalsearch" name="crawlPauseLocalsearch" type="text" size="20" maxlength="100" value="#[crawlPauseLocalsearch]#" /></dd>
<dt><label for="crawlPauseRemotesearch">Online Caution Delay for Remote Search (milliseconds)</label>:</dt>
<dd><input id="crawlPauseRemotesearch" name="crawlPauseRemotesearch" type="text" size="20" maxlength="100" value="#[crawlPauseRemotesearch]#" /></dd>
<dt><input type="submit" name="onlineCautionSubmit" value="Enter New Parameters" /></dt>
<dd>Changes take effect immediately</dd>
</dl>
</fieldset>

15
htroot/PerformanceQueues_p.java
Unescape View File

@ -227,9 +227,10 @@ public class PerformanceQueues_p {
switchboard.setConfig("javastart_priority",post.get("YaCyPriority","0"));
}
if ((post != null) && (post.containsKey("proxyControlSubmit"))) {
int onlineCautionDelay = post.getInt("onlineCautionDelay", 30000);
switchboard.setConfig("onlineCautionDelay", Integer.toString(onlineCautionDelay));
if ((post != null) && (post.containsKey("onlineCautionSubmit"))) {
switchboard.setConfig(plasmaSwitchboard.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000)));
switchboard.setConfig(plasmaSwitchboard.LOCALSEACH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseLocalsearch", 30000)));
switchboard.setConfig(plasmaSwitchboard.REMOTESEARCH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseRemotesearch", 30000)));
}
// table cache settings
@ -248,8 +249,12 @@ public class PerformanceQueues_p {
prop.put("wordCacheMaxCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 20000));
prop.put("wordCacheInitCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_INIT_COUNT, 30000));
prop.put("wordFlushSize", switchboard.getConfigLong("wordFlushSize", 2000));
prop.put("onlineCautionDelay", switchboard.getConfigLong("onlineCautionDelay", 30000));
prop.putNum("onlineCautionDelayCurrent", System.currentTimeMillis() - switchboard.proxyLastAccess);
prop.put("crawlPauseProxy", switchboard.getConfigLong(plasmaSwitchboard.PROXY_ONLINE_CAUTION_DELAY, 30000));
prop.put("crawlPauseLocalsearch", switchboard.getConfigLong(plasmaSwitchboard.LOCALSEACH_ONLINE_CAUTION_DELAY, 30000));
prop.put("crawlPauseRemotesearch", switchboard.getConfigLong(plasmaSwitchboard.REMOTESEARCH_ONLINE_CAUTION_DELAY, 30000));
prop.putNum("crawlPauseProxyCurrent", (System.currentTimeMillis() - switchboard.proxyLastAccess) / 1000);
prop.putNum("crawlPauseLocalsearchCurrent", (System.currentTimeMillis() - switchboard.localSearchLastAccess) / 1000);
prop.putNum("crawlPauseRemotesearchCurrent", (System.currentTimeMillis() - switchboard.remoteSearchLastAccess) / 1000);
// table thread pool settings
prop.put("pool_0_name","Crawler Pool");

22
htroot/index.java
Unescape View File

@ -68,26 +68,8 @@ public class index {
final boolean indexDistributeGranted = sb.getConfigBool(plasmaSwitchboard.INDEX_DIST_ALLOW, true);
final boolean indexReceiveGranted = sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, true);
global = global && indexDistributeGranted && indexReceiveGranted;
/*
final String referer = (String) header.get(httpHeader.REFERER);
if (referer != null) {
yacyURL url;
try {
url = new yacyURL(referer, null);
} catch (MalformedURLException e) {
url = null;
}
if ((url != null) && (!url.isLocal())) {
final HashMap referrerprop = new HashMap();
referrerprop.put("count", "1");
referrerprop.put("clientip", header.get(httpHeader.CONNECTION_PROP_CLIENTIP));
referrerprop.put("useragent", header.get(httpHeader.USER_AGENT));
referrerprop.put("date", (new serverDate()).toShortString(false));
if (sb.facilityDB != null) try {sb.facilityDB.update("backlinks", referer, referrerprop);} catch (IOException e) {}
}
}
*/
//global = global && indexDistributeGranted && indexReceiveGranted;
// search domain
int contentdom = plasmaSearchQuery.CONTENTDOM_TEXT;
String cds = (post == null) ? "text" : post.get("contentdom", "text");

24
htroot/yacysearch.java
Unescape View File

@ -91,22 +91,6 @@ public class yacysearch {
boolean rss = (post == null) ? false : post.get("rss", "false").equals("true");
if ((post == null) || (env == null) || (querystring.length() == 0) || (!searchAllowed)) {
/*
// save referrer
final String referer = (String) header.get("Referer");
if (referer != null) {
yacyURL url;
try { url = new yacyURL(referer, null); } catch (MalformedURLException e) { url = null; }
if ((url != null) && (!url.isLocal())) {
final HashMap referrerprop = new HashMap();
referrerprop.put("count", "1");
referrerprop.put("clientip", header.get("CLIENTIP"));
referrerprop.put("useragent", header.get("User-Agent"));
referrerprop.put("date", (new serverDate()).toShortString(false));
if (sb.facilityDB != null) try { sb.facilityDB.update("backlinks", referer, referrerprop); } catch (IOException e) {}
}
}
*/
// we create empty entries for template strings
final serverObjects prop = new serverObjects();
prop.put("searchagain", "0");
@ -169,13 +153,13 @@ public class yacysearch {
}
// SEARCH
final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true");
final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true");
final boolean offline = yacyCore.seedDB.mySeed().isVirgin();
//final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true");
//final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true");
//final boolean offline = yacyCore.seedDB.mySeed().isVirgin();
final boolean clustersearch = sb.isRobinsonMode() &&
(sb.getConfig("cluster.mode", "").equals("privatecluster") ||
sb.getConfig("cluster.mode", "").equals("publiccluster"));
if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; }
//if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; }
if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes
// find search domain

3
source/de/anomic/server/serverSystem.java
Unescape View File

@ -50,6 +50,8 @@ import java.util.Hashtable;
import java.util.Properties;
import java.util.Vector;
import de.anomic.server.logging.serverLog;
public final class serverSystem {
// constants for system identification
@ -330,6 +332,7 @@ public final class serverSystem {
try {
Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (InterruptedException e) {
serverLog.logSevere("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);
throw new IOException(e.getMessage());
}
}

25
yacy.init
Unescape View File

@ -725,11 +725,11 @@ crawler.http.acceptEncoding=gzip
crawler.http.acceptLanguage=en-us,en;q=0.5
crawler.http.acceptCharset=ISO-8859-1,utf-8;q=0.7,*;q=0.7
crawler.http.maxFileSize=262144
crawler.http.maxFileSize__pro=1048576
crawler.http.maxFileSize__pro=262144
# ftp crawler specific settings; size in bytes
crawler.ftp.maxFileSize=262144
crawler.ftp.maxFileSize__pro=1048576
crawler.ftp.maxFileSize__pro=262144
# maximum number of crawler threads
crawler.MaxActiveThreads = 30
@ -903,27 +903,6 @@ tableTypeForPreNURL=0
publicSurftips = true
publicSearchpage = true
# a Java Properties file containig a list of SOAP services that should deployed
# additionally to the default services. E.g.
# soap.serviceDeploymentList = DATA/SETTINGS/myServices.properties
#
# One entry in the property file should must have the following format:
# servicename=fullClassname
#
# e.g.
# test=org.myservices.test
#
# Servicename is the name that shoud be used to access the service, e.g.
# if the service name is "test" then the service can be reached using
# http://localhost:8080/soap/test
#
# The WSDL document that belongs to the deployed service could be reached
# using
# http://localhost:8080/soap/test?wsdl
#
soap.serviceDeploymentList =
# Wiki access rights
# the built-in wiki system allows by default only that the administrator is allowed to make changes
# this can be changed. There are three options:

Write Preview
Loading…
Cancel
Save