From d0ec8018f5b6183c95f2ac3102be20f8a65a3056 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Wed, 6 Jun 2012 14:13:31 +0200 Subject: [PATCH] fixes for bad long computation --- htroot/PerformanceMemory_p.java | 12 ++-- htroot/PerformanceQueues_p.java | 84 ++++++++++++------------- htroot/ProxyIndexingMonitor_p.java | 13 ++-- htroot/YBRFetch_p.java | 2 +- source/net/yacy/search/Switchboard.java | 2 +- 5 files changed, 56 insertions(+), 57 deletions(-) diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index dccb39f51..4a1b5942f 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -124,7 +124,7 @@ public class PerformanceMemory_p { c++; } prop.put("EcoList", c); - prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d)); + prop.putNum("EcoIndexTotalMem", totalmem / (1024d * 1024d)); // write object cache table final Iterator> oi = RAMIndex.objects(); @@ -153,7 +153,7 @@ public class PerformanceMemory_p { c++; } prop.put("indexcache", c); - prop.putNum("indexcacheTotalMem", totalhitmem / (1024 * 1024d)); + prop.putNum("indexcacheTotalMem", totalhitmem / (1024d * 1024d)); // write object cache table i = Cache.filenames(); @@ -195,10 +195,10 @@ public class PerformanceMemory_p { c++; } prop.put("ObjectList", c); - prop.putNum("objectCacheStopGrow", Cache.getMemStopGrow() / (1024 * 1024d)); - prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024 * 1024d)); - prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024 * 1024d)); - prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024 * 1024d)); + prop.putNum("objectCacheStopGrow", Cache.getMemStopGrow() / (1024d * 1024d)); + prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024d * 1024d)); + prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024d * 1024d)); + prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024d * 1024d)); // other caching structures prop.putNum("namecacheHit.size", Domains.nameCacheHitSize()); diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index f8d2fe607..aa71f767b 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -55,13 +55,13 @@ public class PerformanceQueues_p { performanceProfiles.put("defaults/yacy.init", "default (crawl)"); performanceProfiles.put("defaults/performance_dht.profile", "prefer DHT"); } - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); File defaultSettingsFile = new File(sb.getAppPath(), "defaults/yacy.init"); - + // get segment Segment indexSegment = null; if (post != null && post.containsKey("segment")) { @@ -73,7 +73,7 @@ public class PerformanceQueues_p { // take default segment indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC); } - + if(post != null) { if(post.containsKey("defaultFile")){ // TODO check file-path! @@ -108,10 +108,10 @@ public class PerformanceQueues_p { Iterator threads = sb.threadNames(); String threadName; BusyThread thread; - + final boolean xml = (header.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml"); prop.setLocalized(!xml); - + // calculate totals long blocktime_total = 0, sleeptime_total = 0, exectime_total = 0; while (threads.hasNext()) { @@ -120,11 +120,11 @@ public class PerformanceQueues_p { blocktime_total += thread.getBlockTime(); sleeptime_total += thread.getSleepTime(); exectime_total += thread.getExecTime(); - } + } if (blocktime_total == 0) blocktime_total = 1; if (sleeptime_total == 0) sleeptime_total = 1; if (exectime_total == 0) exectime_total = 1; - + // set templates for latest news from the threads long blocktime, sleeptime, exectime; long idlesleep, busysleep, memuse, memprereq; @@ -141,11 +141,11 @@ public class PerformanceQueues_p { sb.setConfig("performanceProfile", post.get("defaultFile", "defaults/yacy.init")); sb.setConfig("performanceSpeed", post.getInt("profileSpeed", 100)); } - + while (threads.hasNext()) { threadName = threads.next(); thread = sb.getThread(threadName); - + // set values to templates prop.put("table_" + c + "_threadname", threadName); @@ -159,7 +159,7 @@ public class PerformanceQueues_p { prop.putHTML("table_" + c + "_longdescr", thread.getLongDescription()); queuesize = thread.getJobCount(); prop.put("table_" + c + "_queuesize", (queuesize == Integer.MAX_VALUE) ? "unlimited" : Formatter.number(queuesize, !xml)); - + blocktime = thread.getBlockTime(); sleeptime = thread.getSleepTime(); exectime = thread.getExecTime(); @@ -180,7 +180,7 @@ public class PerformanceQueues_p { prop.putNum("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? -1 : sleeptime / (idleCycles + busyCycles)); prop.putNum("table_" + c + "_execpercycle", (busyCycles == 0) ? -1 : exectime / busyCycles); prop.putNum("table_" + c + "_memusepercycle", (busyCycles == 0) ? -1 : memuse / busyCycles / 1024); - + // load with old values idlesleep = sb.getConfigLong(threadName + "_idlesleep" , 1000); busysleep = sb.getConfigLong(threadName + "_busysleep", 100); @@ -189,13 +189,13 @@ public class PerformanceQueues_p { // load with new values idlesleep = post.getLong(threadName + "_idlesleep", idlesleep); busysleep = post.getLong(threadName + "_busysleep", busysleep); - memprereq = post.getLong(threadName + "_memprereq", memprereq) * 1024; + memprereq = post.getLong(threadName + "_memprereq", memprereq) * 1024l; if (memprereq == 0) memprereq = sb.getConfigLong(threadName + "_memprereq", 0); - + // check values to prevent short-cut loops if (idlesleep < 1000) idlesleep = 1000; if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; } - + sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq); idlesleep = sb.getConfigLong(threadName + "_idlesleep", idlesleep); busysleep = sb.getConfigLong(threadName + "_busysleep", busysleep); @@ -228,7 +228,7 @@ public class PerformanceQueues_p { c++; } prop.put("table", c); - + // performance profiles c = 0; final String usedfile = sb.getConfig("performanceProfile", "defaults/yacy.init"); @@ -239,7 +239,7 @@ public class PerformanceQueues_p { c++; } prop.put("profile", c); - + c = 0; final int[] speedValues = {200,150,100,50,25,10}; final int usedspeed = sb.getConfigInt("performanceSpeed", 100); @@ -250,27 +250,27 @@ public class PerformanceQueues_p { c++; } prop.put("speed", c); - + if ((post != null) && (post.containsKey("cacheSizeSubmit"))) { final int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 20000); sb.setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount)); indexSegment.termIndex().setBufferMaxWordCount(wordCacheMaxCount); } - + if ((post != null) && (post.containsKey("poolConfig"))) { - - /* - * configuring the crawler pool + + /* + * configuring the crawler pool */ // get the current crawler pool configuration int maxBusy = post.getInt("Crawler Pool_maxActive", 8); - + // storing the new values into configfile sb.setConfig(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX,maxBusy); //switchboard.setConfig("crawler.MinIdleThreads",minIdle); - - /* - * configuring the http pool + + /* + * configuring the http pool */ final WorkflowThread httpd = sb.getThread("10_httpd"); try { @@ -279,23 +279,23 @@ public class PerformanceQueues_p { maxBusy = 8; } - ((serverCore)httpd).setMaxSessionCount(maxBusy); - + ((serverCore)httpd).setMaxSessionCount(maxBusy); + // storing the new values into configfile sb.setConfig("httpdMaxBusySessions",maxBusy); - } - + } + if ((post != null) && (post.containsKey("PrioritySubmit"))) { sb.setConfig("javastart_priority",post.get("YaCyPriority","0")); } - + if ((post != null) && (post.containsKey("onlineCautionSubmit"))) { sb.setConfig(SwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000))); sb.setConfig(SwitchboardConstants.LOCALSEACH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseLocalsearch", 30000))); sb.setConfig(SwitchboardConstants.REMOTESEARCH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseRemotesearch", 30000))); } - + if ((post != null) && (post.containsKey("minimumDeltaSubmit"))) { final long minimumLocalDelta = post.getLong("minimumLocalDelta", sb.crawlQueues.noticeURL.getMinimumLocalDelta()); final long minimumGlobalDelta = post.getLong("minimumGlobalDelta", sb.crawlQueues.noticeURL.getMinimumGlobalDelta()); @@ -303,13 +303,13 @@ public class PerformanceQueues_p { sb.setConfig("minimumGlobalDelta", minimumGlobalDelta); sb.crawlQueues.noticeURL.setMinimumDelta(minimumLocalDelta, minimumGlobalDelta); } - + // delta settings prop.put("minimumLocalDelta", sb.crawlQueues.noticeURL.getMinimumLocalDelta()); prop.put("minimumGlobalDelta", sb.crawlQueues.noticeURL.getMinimumGlobalDelta()); - + // table cache settings - prop.putNum("urlCacheSize", indexSegment.urlMetadata().writeCacheSize()); + prop.putNum("urlCacheSize", indexSegment.urlMetadata().writeCacheSize()); prop.putNum("wordCacheSize", indexSegment.termIndex().getBufferSize()); prop.putNum("wordCacheSizeKBytes", indexSegment.termIndex().getBufferSizeBytes()/1024); prop.putNum("maxURLinCache", indexSegment.termIndex().getBufferMaxReferences()); @@ -323,30 +323,30 @@ public class PerformanceQueues_p { prop.putNum("crawlPauseProxyCurrent", (System.currentTimeMillis() - sb.proxyLastAccess) / 1000); prop.putNum("crawlPauseLocalsearchCurrent", (System.currentTimeMillis() - sb.localSearchLastAccess) / 1000); prop.putNum("crawlPauseRemotesearchCurrent", (System.currentTimeMillis() - sb.remoteSearchLastAccess) / 1000); - + // table thread pool settings prop.put("pool_0_name","Crawler Pool"); prop.put("pool_0_maxActive", sb.getConfigLong("crawler.MaxActiveThreads", 0)); prop.put("pool_0_numActive",sb.crawlQueues.workerSize()); - + final WorkflowThread httpd = sb.getThread("10_httpd"); prop.put("pool_1_name", "httpd Session Pool"); prop.put("pool_1_maxActive", ((serverCore)httpd).getMaxSessionCount()); prop.put("pool_1_numActive", ((serverCore)httpd).getJobCount()); - + prop.put("pool", "2"); - + final long curr_prio = sb.getConfigLong("javastart_priority",0); prop.put("priority_normal",(curr_prio == 0) ? "1" : "0"); prop.put("priority_below",(curr_prio == 10) ? "1" : "0"); prop.put("priority_low",(curr_prio == 20) ? "1" : "0"); - + // parse initialization memory settings final String Xmx = sb.getConfig("javastart_Xmx", "Xmx500m").substring(3); prop.put("Xmx", Xmx.substring(0, Xmx.length() - 1)); final String Xms = sb.getConfig("javastart_Xms", "Xms500m").substring(3); prop.put("Xms", Xms.substring(0, Xms.length() - 1)); - + final long diskFree = sb.getConfigLong(SwitchboardConstants.DISK_FREE, 3000L); final long diskFreeHardlimit = sb.getConfigLong(SwitchboardConstants.DISK_FREE_HARDLIMIT, 1000L); final long memoryAcceptDHT = sb.getConfigLong(SwitchboardConstants.MEMORY_ACCEPTDHT, 50000L); @@ -355,11 +355,11 @@ public class PerformanceQueues_p { prop.put("diskFreeHardlimit", diskFreeHardlimit); prop.put("memoryAcceptDHT", memoryAcceptDHT); if(observerTrigger) prop.put("observerTrigger", "1"); - + // return rewrite values for templates return prop; } - + private static String d(final String a, final String b) { return (a == null) ? b : a; } diff --git a/htroot/ProxyIndexingMonitor_p.java b/htroot/ProxyIndexingMonitor_p.java index 98553fd23..6bef3fa91 100644 --- a/htroot/ProxyIndexingMonitor_p.java +++ b/htroot/ProxyIndexingMonitor_p.java @@ -1,4 +1,4 @@ -// ProxyIndexingMonitor_p.java +// ProxyIndexingMonitor_p.java // --------------------------- // part of the AnomicHTTPD caching proxy // (C) by Michael Peter Christen; mc@yacy.net @@ -33,7 +33,6 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; - import de.anomic.crawler.Cache; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -66,7 +65,7 @@ public class ProxyIndexingMonitor_p { final boolean proxyYaCyOnly = post.containsKey("proxyYacyOnly"); env.setConfig(SwitchboardConstants.PROXY_YACY_ONLY, (proxyYaCyOnly) ? true : false); int newProxyPrefetchDepth = post.getInt("proxyPrefetchDepth", 0); - if (newProxyPrefetchDepth < 0) newProxyPrefetchDepth = 0; + if (newProxyPrefetchDepth < 0) newProxyPrefetchDepth = 0; if (newProxyPrefetchDepth > 20) newProxyPrefetchDepth = 20; // self protection ? env.setConfig("proxyPrefetchDepth", Integer.toString(newProxyPrefetchDepth)); final boolean proxyStoreHTCache = post.containsKey("proxyStoreHTCache"); @@ -77,7 +76,7 @@ public class ProxyIndexingMonitor_p { env.setConfig("proxyIndexingLocalText", proxyIndexingLocalText ? true : false); final boolean proxyIndexingLocalMedia = post.containsKey("proxyIndexingLocalMedia"); env.setConfig("proxyIndexingLocalMedia", proxyIndexingLocalMedia ? true : false); - + // added proxyCache, proxyCacheSize - Borg-0300 // proxyCache - check and create the directory oldProxyCachePath = env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT); @@ -90,12 +89,12 @@ public class ProxyIndexingMonitor_p { final File cache = env.getDataPath(SwitchboardConstants.HTCACHE_PATH, oldProxyCachePath); if (!cache.isDirectory() && !cache.isFile()) cache.mkdirs(); - // proxyCacheSize + // proxyCacheSize oldProxyCacheSize = env.getConfigLong(SwitchboardConstants.PROXY_CACHE_SIZE, 64L); newProxyCacheSize = post.getLong(SwitchboardConstants.PROXY_CACHE_SIZE, 64L); if (newProxyCacheSize < 4) { newProxyCacheSize = 4; } env.setConfig(SwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize); - Cache.setMaxCacheSize(newProxyCacheSize * 1024 * 1024); + Cache.setMaxCacheSize(newProxyCacheSize * 1024L * 1024L); // implant these settings also into the crawling profile for the proxy if (sb.crawler.defaultProxyProfile == null) { @@ -108,7 +107,7 @@ public class ProxyIndexingMonitor_p { sb.crawler.defaultProxyProfile.put("indexText", proxyIndexingLocalText); sb.crawler.defaultProxyProfile.put("indexMedia", proxyIndexingLocalMedia); sb.crawler.putActive(sb.crawler.defaultProxyProfile.handle().getBytes(), sb.crawler.defaultProxyProfile); - + prop.put("info", "2");//new proxyPrefetchdepth prop.put("info_message", newProxyPrefetchDepth); prop.put("info_caching", proxyStoreHTCache ? "1" : "0"); diff --git a/htroot/YBRFetch_p.java b/htroot/YBRFetch_p.java index 4f97ea207..3dc7625b3 100644 --- a/htroot/YBRFetch_p.java +++ b/htroot/YBRFetch_p.java @@ -28,7 +28,7 @@ public class YBRFetch_p final servletProperties prop = new servletProperties(); final Switchboard sb = (Switchboard) env; - if ( post == null || !post.containsKey("ghrt4") || MemoryControl.available() < 1024 * 1024 * 1024 ) { + if ( post == null || !post.containsKey("ghrt4") || MemoryControl.available() < 1024L * 1024L * 1024L ) { return prop; } final File hostIndexFile = new File(sb.queuesRoot, "hostIndex.blob"); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 4f1957e9a..2b3bad5ab 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -556,7 +556,7 @@ public final class Switchboard extends serverSwitch getDataPath(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT); this.log.logInfo("HTCACHE Path = " + this.htCachePath.getAbsolutePath()); final long maxCacheSize = - 1024 * 1024 * Long.parseLong(getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte + 1024L * 1024L * Long.parseLong(getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize); // create the surrogates directories