boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixes for bad long computation

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 6bb07afcc3
commit d0ec8018f5
5 changed files with 56 additions and 57 deletions
Show Stats Download Patch File Download Diff File

12
htroot/PerformanceMemory_p.java
Unescape View File

@ -124,7 +124,7 @@ public class PerformanceMemory_p {
c++;
}
prop.put("EcoList", c);
prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d));
prop.putNum("EcoIndexTotalMem", totalmem / (1024d * 1024d));
// write object cache table
final Iterator<Map.Entry<String, RAMIndex>> oi = RAMIndex.objects();
@ -153,7 +153,7 @@ public class PerformanceMemory_p {
c++;
}
prop.put("indexcache", c);
prop.putNum("indexcacheTotalMem", totalhitmem / (1024 * 1024d));
prop.putNum("indexcacheTotalMem", totalhitmem / (1024d * 1024d));
// write object cache table
i = Cache.filenames();
@ -195,10 +195,10 @@ public class PerformanceMemory_p {
c++;
}
prop.put("ObjectList", c);
prop.putNum("objectCacheStopGrow", Cache.getMemStopGrow() / (1024 * 1024d));
prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024 * 1024d));
prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024 * 1024d));
prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024 * 1024d));
prop.putNum("objectCacheStopGrow", Cache.getMemStopGrow() / (1024d * 1024d));
prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024d * 1024d));
prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024d * 1024d));
prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024d * 1024d));
// other caching structures
prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());

84
htroot/PerformanceQueues_p.java
Unescape View File

@ -55,13 +55,13 @@ public class PerformanceQueues_p {
performanceProfiles.put("defaults/yacy.init", "default (crawl)");
performanceProfiles.put("defaults/performance_dht.profile", "prefer DHT");
}
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
File defaultSettingsFile = new File(sb.getAppPath(), "defaults/yacy.init");
// get segment
Segment indexSegment = null;
if (post != null && post.containsKey("segment")) {
@ -73,7 +73,7 @@ public class PerformanceQueues_p {
// take default segment
indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
}
if(post != null) {
if(post.containsKey("defaultFile")){
// TODO check file-path!
@ -108,10 +108,10 @@ public class PerformanceQueues_p {
Iterator<String> threads = sb.threadNames();
String threadName;
BusyThread thread;
final boolean xml = (header.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml");
prop.setLocalized(!xml);
// calculate totals
long blocktime_total = 0, sleeptime_total = 0, exectime_total = 0;
while (threads.hasNext()) {
@ -120,11 +120,11 @@ public class PerformanceQueues_p {
blocktime_total += thread.getBlockTime();
sleeptime_total += thread.getSleepTime();
exectime_total += thread.getExecTime();
}
}
if (blocktime_total == 0) blocktime_total = 1;
if (sleeptime_total == 0) sleeptime_total = 1;
if (exectime_total == 0) exectime_total = 1;
// set templates for latest news from the threads
long blocktime, sleeptime, exectime;
long idlesleep, busysleep, memuse, memprereq;
@ -141,11 +141,11 @@ public class PerformanceQueues_p {
sb.setConfig("performanceProfile", post.get("defaultFile", "defaults/yacy.init"));
sb.setConfig("performanceSpeed", post.getInt("profileSpeed", 100));
}
while (threads.hasNext()) {
threadName = threads.next();
thread = sb.getThread(threadName);
// set values to templates
prop.put("table_" + c + "_threadname", threadName);
@ -159,7 +159,7 @@ public class PerformanceQueues_p {
prop.putHTML("table_" + c + "_longdescr", thread.getLongDescription());
queuesize = thread.getJobCount();
prop.put("table_" + c + "_queuesize", (queuesize == Integer.MAX_VALUE) ? "unlimited" : Formatter.number(queuesize, !xml));
blocktime = thread.getBlockTime();
sleeptime = thread.getSleepTime();
exectime = thread.getExecTime();
@ -180,7 +180,7 @@ public class PerformanceQueues_p {
prop.putNum("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? -1 : sleeptime / (idleCycles + busyCycles));
prop.putNum("table_" + c + "_execpercycle", (busyCycles == 0) ? -1 : exectime / busyCycles);
prop.putNum("table_" + c + "_memusepercycle", (busyCycles == 0) ? -1 : memuse / busyCycles / 1024);
// load with old values
idlesleep = sb.getConfigLong(threadName + "_idlesleep" , 1000);
busysleep = sb.getConfigLong(threadName + "_busysleep", 100);
@ -189,13 +189,13 @@ public class PerformanceQueues_p {
// load with new values
idlesleep = post.getLong(threadName + "_idlesleep", idlesleep);
busysleep = post.getLong(threadName + "_busysleep", busysleep);
memprereq = post.getLong(threadName + "_memprereq", memprereq) * 1024;
memprereq = post.getLong(threadName + "_memprereq", memprereq) * 1024l;
if (memprereq == 0) memprereq = sb.getConfigLong(threadName + "_memprereq", 0);
// check values to prevent short-cut loops
if (idlesleep < 1000) idlesleep = 1000;
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
idlesleep = sb.getConfigLong(threadName + "_idlesleep", idlesleep);
busysleep = sb.getConfigLong(threadName + "_busysleep", busysleep);
@ -228,7 +228,7 @@ public class PerformanceQueues_p {
c++;
}
prop.put("table", c);
// performance profiles
c = 0;
final String usedfile = sb.getConfig("performanceProfile", "defaults/yacy.init");
@ -239,7 +239,7 @@ public class PerformanceQueues_p {
c++;
}
prop.put("profile", c);
c = 0;
final int[] speedValues = {200,150,100,50,25,10};
final int usedspeed = sb.getConfigInt("performanceSpeed", 100);
@ -250,27 +250,27 @@ public class PerformanceQueues_p {
c++;
}
prop.put("speed", c);
if ((post != null) && (post.containsKey("cacheSizeSubmit"))) {
final int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 20000);
sb.setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount));
indexSegment.termIndex().setBufferMaxWordCount(wordCacheMaxCount);
}
if ((post != null) && (post.containsKey("poolConfig"))) {
/*
* configuring the crawler pool
/*
* configuring the crawler pool
*/
// get the current crawler pool configuration
int maxBusy = post.getInt("Crawler Pool_maxActive", 8);
// storing the new values into configfile
sb.setConfig(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX,maxBusy);
//switchboard.setConfig("crawler.MinIdleThreads",minIdle);
/*
* configuring the http pool
/*
* configuring the http pool
*/
final WorkflowThread httpd = sb.getThread("10_httpd");
try {
@ -279,23 +279,23 @@ public class PerformanceQueues_p {
maxBusy = 8;
}
((serverCore)httpd).setMaxSessionCount(maxBusy);
((serverCore)httpd).setMaxSessionCount(maxBusy);
// storing the new values into configfile
sb.setConfig("httpdMaxBusySessions",maxBusy);
}
}
if ((post != null) && (post.containsKey("PrioritySubmit"))) {
sb.setConfig("javastart_priority",post.get("YaCyPriority","0"));
}
if ((post != null) && (post.containsKey("onlineCautionSubmit"))) {
sb.setConfig(SwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000)));
sb.setConfig(SwitchboardConstants.LOCALSEACH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseLocalsearch", 30000)));
sb.setConfig(SwitchboardConstants.REMOTESEARCH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseRemotesearch", 30000)));
}
if ((post != null) && (post.containsKey("minimumDeltaSubmit"))) {
final long minimumLocalDelta = post.getLong("minimumLocalDelta", sb.crawlQueues.noticeURL.getMinimumLocalDelta());
final long minimumGlobalDelta = post.getLong("minimumGlobalDelta", sb.crawlQueues.noticeURL.getMinimumGlobalDelta());
@ -303,13 +303,13 @@ public class PerformanceQueues_p {
sb.setConfig("minimumGlobalDelta", minimumGlobalDelta);
sb.crawlQueues.noticeURL.setMinimumDelta(minimumLocalDelta, minimumGlobalDelta);
}
// delta settings
prop.put("minimumLocalDelta", sb.crawlQueues.noticeURL.getMinimumLocalDelta());
prop.put("minimumGlobalDelta", sb.crawlQueues.noticeURL.getMinimumGlobalDelta());
// table cache settings
prop.putNum("urlCacheSize", indexSegment.urlMetadata().writeCacheSize());
prop.putNum("urlCacheSize", indexSegment.urlMetadata().writeCacheSize());
prop.putNum("wordCacheSize", indexSegment.termIndex().getBufferSize());
prop.putNum("wordCacheSizeKBytes", indexSegment.termIndex().getBufferSizeBytes()/1024);
prop.putNum("maxURLinCache", indexSegment.termIndex().getBufferMaxReferences());
@ -323,30 +323,30 @@ public class PerformanceQueues_p {
prop.putNum("crawlPauseProxyCurrent", (System.currentTimeMillis() - sb.proxyLastAccess) / 1000);
prop.putNum("crawlPauseLocalsearchCurrent", (System.currentTimeMillis() - sb.localSearchLastAccess) / 1000);
prop.putNum("crawlPauseRemotesearchCurrent", (System.currentTimeMillis() - sb.remoteSearchLastAccess) / 1000);
// table thread pool settings
prop.put("pool_0_name","Crawler Pool");
prop.put("pool_0_maxActive", sb.getConfigLong("crawler.MaxActiveThreads", 0));
prop.put("pool_0_numActive",sb.crawlQueues.workerSize());
final WorkflowThread httpd = sb.getThread("10_httpd");
prop.put("pool_1_name", "httpd Session Pool");
prop.put("pool_1_maxActive", ((serverCore)httpd).getMaxSessionCount());
prop.put("pool_1_numActive", ((serverCore)httpd).getJobCount());
prop.put("pool", "2");
final long curr_prio = sb.getConfigLong("javastart_priority",0);
prop.put("priority_normal",(curr_prio == 0) ? "1" : "0");
prop.put("priority_below",(curr_prio == 10) ? "1" : "0");
prop.put("priority_low",(curr_prio == 20) ? "1" : "0");
// parse initialization memory settings
final String Xmx = sb.getConfig("javastart_Xmx", "Xmx500m").substring(3);
prop.put("Xmx", Xmx.substring(0, Xmx.length() - 1));
final String Xms = sb.getConfig("javastart_Xms", "Xms500m").substring(3);
prop.put("Xms", Xms.substring(0, Xms.length() - 1));
final long diskFree = sb.getConfigLong(SwitchboardConstants.DISK_FREE, 3000L);
final long diskFreeHardlimit = sb.getConfigLong(SwitchboardConstants.DISK_FREE_HARDLIMIT, 1000L);
final long memoryAcceptDHT = sb.getConfigLong(SwitchboardConstants.MEMORY_ACCEPTDHT, 50000L);
@ -355,11 +355,11 @@ public class PerformanceQueues_p {
prop.put("diskFreeHardlimit", diskFreeHardlimit);
prop.put("memoryAcceptDHT", memoryAcceptDHT);
if(observerTrigger) prop.put("observerTrigger", "1");
// return rewrite values for templates
return prop;
}
private static String d(final String a, final String b) {
return (a == null) ? b : a;
}

13
htroot/ProxyIndexingMonitor_p.java
Unescape View File

@ -1,4 +1,4 @@
// ProxyIndexingMonitor_p.java
// ProxyIndexingMonitor_p.java
// ---------------------------
// part of the AnomicHTTPD caching proxy
// (C) by Michael Peter Christen; mc@yacy.net
@ -33,7 +33,6 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import de.anomic.crawler.Cache;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -66,7 +65,7 @@ public class ProxyIndexingMonitor_p {
final boolean proxyYaCyOnly = post.containsKey("proxyYacyOnly");
env.setConfig(SwitchboardConstants.PROXY_YACY_ONLY, (proxyYaCyOnly) ? true : false);
int newProxyPrefetchDepth = post.getInt("proxyPrefetchDepth", 0);
if (newProxyPrefetchDepth < 0) newProxyPrefetchDepth = 0;
if (newProxyPrefetchDepth < 0) newProxyPrefetchDepth = 0;
if (newProxyPrefetchDepth > 20) newProxyPrefetchDepth = 20; // self protection ?
env.setConfig("proxyPrefetchDepth", Integer.toString(newProxyPrefetchDepth));
final boolean proxyStoreHTCache = post.containsKey("proxyStoreHTCache");
@ -77,7 +76,7 @@ public class ProxyIndexingMonitor_p {
env.setConfig("proxyIndexingLocalText", proxyIndexingLocalText ? true : false);
final boolean proxyIndexingLocalMedia = post.containsKey("proxyIndexingLocalMedia");
env.setConfig("proxyIndexingLocalMedia", proxyIndexingLocalMedia ? true : false);
// added proxyCache, proxyCacheSize - Borg-0300
// proxyCache - check and create the directory
oldProxyCachePath = env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT);
@ -90,12 +89,12 @@ public class ProxyIndexingMonitor_p {
final File cache = env.getDataPath(SwitchboardConstants.HTCACHE_PATH, oldProxyCachePath);
if (!cache.isDirectory() && !cache.isFile()) cache.mkdirs();
// proxyCacheSize
// proxyCacheSize
oldProxyCacheSize = env.getConfigLong(SwitchboardConstants.PROXY_CACHE_SIZE, 64L);
newProxyCacheSize = post.getLong(SwitchboardConstants.PROXY_CACHE_SIZE, 64L);
if (newProxyCacheSize < 4) { newProxyCacheSize = 4; }
env.setConfig(SwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize);
Cache.setMaxCacheSize(newProxyCacheSize * 1024 * 1024);
Cache.setMaxCacheSize(newProxyCacheSize * 1024L * 1024L);
// implant these settings also into the crawling profile for the proxy
if (sb.crawler.defaultProxyProfile == null) {
@ -108,7 +107,7 @@ public class ProxyIndexingMonitor_p {
sb.crawler.defaultProxyProfile.put("indexText", proxyIndexingLocalText);
sb.crawler.defaultProxyProfile.put("indexMedia", proxyIndexingLocalMedia);
sb.crawler.putActive(sb.crawler.defaultProxyProfile.handle().getBytes(), sb.crawler.defaultProxyProfile);
prop.put("info", "2");//new proxyPrefetchdepth
prop.put("info_message", newProxyPrefetchDepth);
prop.put("info_caching", proxyStoreHTCache ? "1" : "0");

2
htroot/YBRFetch_p.java
Unescape View File

@ -28,7 +28,7 @@ public class YBRFetch_p
final servletProperties prop = new servletProperties();
final Switchboard sb = (Switchboard) env;
if ( post == null || !post.containsKey("ghrt4") || MemoryControl.available() < 1024 * 1024 * 1024 ) {
if ( post == null || !post.containsKey("ghrt4") || MemoryControl.available() < 1024L * 1024L * 1024L ) {
return prop;
}
final File hostIndexFile = new File(sb.queuesRoot, "hostIndex.blob");

2
source/net/yacy/search/Switchboard.java
Unescape View File

@ -556,7 +556,7 @@ public final class Switchboard extends serverSwitch
getDataPath(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT);
this.log.logInfo("HTCACHE Path = " + this.htCachePath.getAbsolutePath());
final long maxCacheSize =
1024 * 1024 * Long.parseLong(getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte
1024L * 1024L * Long.parseLong(getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte
Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize);
// create the surrogates directories

Write Preview
Loading…
Cancel
Save