From 2f732e32a2e6b39ab4b8c96a0d4dd283712d1db4 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 21 Sep 2005 12:21:01 +0000 Subject: [PATCH] enhancements to memory menue git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@762 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/PerformanceMemory_p.html | 93 ++++++++++++++--------- htroot/PerformanceMemory_p.java | 78 ++++++++++--------- source/de/anomic/server/serverSwitch.java | 1 + source/yacy.java | 19 +++-- 4 files changed, 107 insertions(+), 84 deletions(-) diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html index 9993822e5..cb18b10ea 100644 --- a/htroot/PerformanceMemory_p.html +++ b/htroot/PerformanceMemory_p.html @@ -12,32 +12,44 @@

-

Available Memory:
+
Memory Usage:
- - - - + + + + + + - - - - - + + + - - - + + + + + - - - - + + + + + + + + + + + + + +
TypeAfter StartupAfter InitializationsNow 
Type		After Startup
 		After Initializations
before GC		After Initializations
after GC		Now
 		Description
 
Free#[memoryFreeAfterStartup]# bytes#[memoryFreeAfterInit]# bytes#[memoryFreeNow]# bytesMax#[memoryMax]# MBytemaximum memory that the JVM will attempt to use
Total#[memoryTotalAfterStartup]# bytes#[memoryTotalAfterInit]# bytes#[memoryTotalNow]# bytes#[memoryTotalAfterStartup]# KByte#[memoryTotalAfterInitBGC]# KByte#[memoryTotalAfterInitAGC]# KByte#[memoryTotalNow]# MBytetotal memory taken from the OS
Max#[memoryMaxAfterStartup]# bytes#[memoryMaxAfterInit]# bytes#[memoryMaxNow]# bytesFree#[memoryFreeAfterStartup]# KByte#[memoryFreeAfterInitBGC]# KByte#[memoryFreeAfterInitAGC]# KByte#[memoryFreeNow]# MBytefree memory in the JVM within total amount
Used#[memoryUsedAfterStartup]# KByte#[memoryUsedAfterInitBGC]# KByte#[memoryUsedAfterInitAGC]# KByte#[memoryUsedNow]# MByteused memory in the JVM within total amount

@@ -47,18 +59,23 @@ - - - - - - - - - - - - + + + + + + + + + + + + + + + + + @@ -70,7 +87,7 @@ - + - + - + - + - + - + - + @@ -184,7 +201,7 @@ Increasing this cache may speed up many functions, but we need to test this to s - + - +
Database
Chunk Size
(bytes)		#Needed Slots
(= DB Size)		#Empty Slots
 		#Used Slots
High Priority		#Used Slots
Medium Priority		#Used Slots
Low Priority		Used
Size		Assigned
Max-Size		Recommended
Max-Size		Optimum
Max-Size		Description
 		Database
 		Chunk Size
(bytes)		#SlotsMemory Occupation (KBytes)Description
 
Needed
(= DB Size)		Empty
 		Used
High Prio		Used
Medium Prio		Used
Low Prio		Used NowAssigned MaxGood MaxBest Max
#[slmedRWI]# #[sllowRWI]# #[usedRWI]# bytes #[goodRWI]# #[bestRWI]# The Assortment Cluster stores most of the page indexes. @@ -87,7 +104,7 @@ cache will speed up crawls with a depth > 3. #[slmedHTTP]# #[sllowHTTP]# #[usedHTTP]# bytes #[goodHTTP]# #[bestHTTP]# The Response Header database stores the HTTP heades that other servers send when YaCy retrieves web pages @@ -104,7 +121,7 @@ Increasing this cache will be most important for a fast proxy mode. #[slmedLURL]# #[sllowLURL]# #[usedLURL]# bytes #[goodLURL]# #[bestLURL]# This is the database that holds the hash/url - relation and properties regarding the url like load date and server date. @@ -120,7 +137,7 @@ This cache is very important for a fast search process. Increasing the cache siz #[slmedNURL]# #[sllowNURL]# #[usedNURL]# bytes #[goodNURL]# #[bestNURL]# A noticed URL is one that was discovered during crawling but was not loaded yet. @@ -136,7 +153,7 @@ Increasing the cache size will result in faster double-check during URL recognit #[slmedEURL]# #[sllowEURL]# #[usedEURL]# bytes #[goodEURL]# #[bestEURL]# URLs that cannot be loaded are stored in this database. It is also used for double-checked during crawling. @@ -152,7 +169,7 @@ Increasing the cache size will most probably speed up crawling slightly, but not #[slmedDHT]# #[sllowDHT]# #[usedDHT]# bytes #[goodDHT]# #[bestDHT]# This is simply the cache for the seed-dbs (active, passive, potential). @@ -169,7 +186,7 @@ Increasing this cache may speed up many functions, but we need to test this to s #[slmedMessage]# #[sllowMessage]# #[usedMessage]# bytes #[goodMessage]# #[bestMessage]# The Message cache for peer-to-peer messages. Less important.#[slmedWiki]# #[sllowWiki]# #[usedWiki]# bytes #[goodWiki]# #[bestWiki]# The YaCy-Wiki uses a database to store its pages. @@ -201,7 +218,7 @@ Increasing this cache may speed up access to the wiki pages. #[slmedNews]# #[sllowNews]# #[usedNews]# bytes #[goodNews]# #[bestNews]# The News-DB stores property-lists for news that are included in seeds. diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index f29c70315..69703febd 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -59,6 +59,9 @@ import de.anomic.yacy.yacySeedDB; public class PerformanceMemory_p { + private static final int KB = 1024; + private static final int MB = 1024 * KB; + private static int[] slt; private static int req,chk,usd,bst,god; @@ -70,91 +73,94 @@ public class PerformanceMemory_p { Map defaultSettings = ((post == null) || (!(post.containsKey("submitdefault")))) ? null : serverFileUtils.loadHashMap(defaultSettingsFile); if ((post != null) && (post.containsKey("submitcache"))) { - sb.setConfig("ramCacheRWI", post.get("ramCacheRWI", "0")); - sb.setConfig("ramCacheHTTP", post.get("ramCacheHTTP", "0")); - sb.setConfig("ramCacheLURL", post.get("ramCacheLURL", "0")); - sb.setConfig("ramCacheNURL", post.get("ramCacheNURL", "0")); - sb.setConfig("ramCacheEURL", post.get("ramCacheEURL", "0")); - sb.setConfig("ramCacheDHT", post.get("ramCacheDHT", "0")); - sb.setConfig("ramCacheMessage", post.get("ramCacheMessage", "0")); - sb.setConfig("ramCacheWiki", post.get("ramCacheWiki", "0")); - sb.setConfig("ramCacheNews", post.get("ramCacheNews", "0")); + sb.setConfig("ramCacheRWI", Long.parseLong(post.get("ramCacheRWI", "0")) * KB); + sb.setConfig("ramCacheHTTP", Long.parseLong(post.get("ramCacheHTTP", "0")) * KB); + sb.setConfig("ramCacheLURL", Long.parseLong(post.get("ramCacheLURL", "0")) * KB); + sb.setConfig("ramCacheNURL", Long.parseLong(post.get("ramCacheNURL", "0")) * KB); + sb.setConfig("ramCacheEURL", Long.parseLong(post.get("ramCacheEURL", "0")) * KB); + sb.setConfig("ramCacheDHT", Long.parseLong(post.get("ramCacheDHT", "0")) * KB); + sb.setConfig("ramCacheMessage", Long.parseLong(post.get("ramCacheMessage", "0")) * KB); + sb.setConfig("ramCacheWiki", Long.parseLong(post.get("ramCacheWiki", "0")) * KB); + sb.setConfig("ramCacheNews", Long.parseLong(post.get("ramCacheNews", "0")) * KB); } - System.gc(); long memoryFreeNow = Runtime.getRuntime().freeMemory(); - long memoryFreeAfterInit = Long.parseLong(sb.getConfig("memoryFreeAfterInit", "0")); + long memoryFreeAfterInitBGC = Long.parseLong(sb.getConfig("memoryFreeAfterInitBGC", "0")); + long memoryFreeAfterInitAGC = Long.parseLong(sb.getConfig("memoryFreeAfterInitAGC", "0")); long memoryFreeAfterStartup = Long.parseLong(sb.getConfig("memoryFreeAfterStartup", "0")); long memoryTotalNow = Runtime.getRuntime().totalMemory(); - long memoryTotalAfterInit = Long.parseLong(sb.getConfig("memoryTotalAfterInit", "0")); + long memoryTotalAfterInitBGC = Long.parseLong(sb.getConfig("memoryTotalAfterInitBGC", "0")); + long memoryTotalAfterInitAGC = Long.parseLong(sb.getConfig("memoryTotalAfterInitAGC", "0")); long memoryTotalAfterStartup = Long.parseLong(sb.getConfig("memoryTotalAfterStartup", "0")); - long memoryMaxNow = Runtime.getRuntime().maxMemory(); - long memoryMaxAfterInit = Long.parseLong(sb.getConfig("memoryMaxAfterInit", "0")); - long memoryMaxAfterStartup = Long.parseLong(sb.getConfig("memoryMaxAfterStartup", "0")); + long memoryMax = Runtime.getRuntime().maxMemory(); - prop.put("memoryFreeNow", memoryFreeNow); - prop.put("memoryFreeAfterInit", memoryFreeAfterInit); - prop.put("memoryFreeAfterStartup", memoryFreeAfterStartup); - prop.put("memoryTotalNow", memoryTotalNow); - prop.put("memoryTotalAfterInit", memoryTotalAfterInit); - prop.put("memoryTotalAfterStartup", memoryTotalAfterStartup); - prop.put("memoryMaxNow", memoryMaxNow); - prop.put("memoryMaxAfterInit", memoryMaxAfterInit); - prop.put("memoryMaxAfterStartup", memoryMaxAfterStartup); + prop.put("memoryUsedAfterStartup", (memoryTotalAfterStartup - memoryFreeAfterStartup) / KB); + prop.put("memoryUsedAfterInitBGC", (memoryTotalAfterInitBGC - memoryFreeAfterInitBGC) / KB); + prop.put("memoryUsedAfterInitAGC", (memoryTotalAfterInitAGC - memoryFreeAfterInitAGC) / KB); + prop.put("memoryUsedNow", (memoryTotalNow - memoryFreeNow) / MB); + prop.put("memoryFreeAfterStartup", memoryFreeAfterStartup / KB); + prop.put("memoryFreeAfterInitBGC", memoryFreeAfterInitBGC / KB); + prop.put("memoryFreeAfterInitAGC", memoryFreeAfterInitAGC / KB); + prop.put("memoryFreeNow", memoryFreeNow / MB); + prop.put("memoryTotalAfterStartup", memoryTotalAfterStartup / KB); + prop.put("memoryTotalAfterInitBGC", memoryTotalAfterInitBGC / KB); + prop.put("memoryTotalAfterInitAGC", memoryTotalAfterInitAGC / KB); + prop.put("memoryTotalNow", memoryTotalNow / MB); + prop.put("memoryMax", memoryMax / MB); req = switchboard.wordIndex.size(); chk = switchboard.wordIndex.assortmentsCacheChunkSizeAvg(); slt = switchboard.wordIndex.assortmentsCacheFillStatusCml(); calc(); putprop(prop, "RWI"); - prop.put("ramCacheRWI", sb.getConfig("ramCacheRWI", "0")); + prop.put("ramCacheRWI", Long.parseLong(sb.getConfig("ramCacheRWI", "0")) / KB); req = switchboard.cacheManager.dbSize(); chk = switchboard.cacheManager.dbCacheChunkSize(); slt = switchboard.cacheManager.dbCacheFillStatus(); calc(); putprop(prop, "HTTP"); - prop.put("ramCacheHTTP", sb.getConfig("ramCacheHTTP", "0")); + prop.put("ramCacheHTTP", Long.parseLong(sb.getConfig("ramCacheHTTP", "0")) / KB); req = switchboard.urlPool.loadedURL.urlHashCache.size(); chk = switchboard.urlPool.loadedURL.urlHashCache.cacheChunkSize(); slt = switchboard.urlPool.loadedURL.urlHashCache.cacheFillStatus(); calc(); putprop(prop, "LURL"); - prop.put("ramCacheLURL", sb.getConfig("ramCacheLURL", "0")); + prop.put("ramCacheLURL", Long.parseLong(sb.getConfig("ramCacheLURL", "0")) / KB); req = switchboard.urlPool.noticeURL.urlHashCache.size(); chk = switchboard.urlPool.noticeURL.urlHashCache.cacheChunkSize(); slt = switchboard.urlPool.noticeURL.urlHashCache.cacheFillStatus(); calc(); putprop(prop, "NURL"); - prop.put("ramCacheNURL", sb.getConfig("ramCacheNURL", "0")); + prop.put("ramCacheNURL", Long.parseLong(sb.getConfig("ramCacheNURL", "0")) / KB); req = switchboard.urlPool.errorURL.urlHashCache.size(); chk = switchboard.urlPool.errorURL.urlHashCache.cacheChunkSize(); slt = switchboard.urlPool.errorURL.urlHashCache.cacheFillStatus(); calc(); putprop(prop, "EURL"); - prop.put("ramCacheEURL", sb.getConfig("ramCacheEURL", "0")); + prop.put("ramCacheEURL", Long.parseLong(sb.getConfig("ramCacheEURL", "0")) / KB); req = yacyCore.seedDB.sizeConnected() + yacyCore.seedDB.sizeDisconnected() + yacyCore.seedDB.sizePotential(); chk = yacyCore.seedDB.dbCacheChunkSize(); slt = yacyCore.seedDB.dbCacheFillStatus(); calc(); putprop(prop, "DHT"); - prop.put("ramCacheDHT", sb.getConfig("ramCacheDHT", "0")); + prop.put("ramCacheDHT", Long.parseLong(sb.getConfig("ramCacheDHT", "0")) / KB); req = switchboard.messageDB.size(); chk = switchboard.messageDB.dbCacheChunkSize(); slt = switchboard.messageDB.dbCacheFillStatus(); calc(); putprop(prop, "Message"); - prop.put("ramCacheMessage", sb.getConfig("ramCacheMessage", "0")); + prop.put("ramCacheMessage", Long.parseLong(sb.getConfig("ramCacheMessage", "0")) / KB); req = switchboard.wikiDB.sizeOfTwo(); chk = switchboard.wikiDB.dbCacheChunkSize(); slt = switchboard.wikiDB.dbCacheFillStatus(); calc(); putprop(prop, "Wiki"); - prop.put("ramCacheWiki", sb.getConfig("ramCacheWiki", "0")); + prop.put("ramCacheWiki", Long.parseLong(sb.getConfig("ramCacheWiki", "0")) / KB); req = yacyCore.newsPool.dbSize(); chk = yacyCore.newsPool.dbCacheChunkSize(); slt = yacyCore.newsPool.dbCacheFillStatus(); calc(); putprop(prop, "News"); - prop.put("ramCacheNews", sb.getConfig("ramCacheNews", "0")); + prop.put("ramCacheNews", Long.parseLong(sb.getConfig("ramCacheNews", "0")) / KB); // return rewrite values for templates return prop; @@ -173,8 +179,8 @@ public class PerformanceMemory_p { prop.put("slhig" + db, slt[1]); prop.put("slmed" + db, slt[2]); prop.put("sllow" + db, slt[3]); - prop.put("used" + db, usd); - prop.put("good" + db, god); - prop.put("best" + db, bst); + prop.put("used" + db, usd / KB); + prop.put("good" + db, god / KB); + prop.put("best" + db, bst / KB); } } diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index 0275a92fe..1dc288ca4 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -88,6 +88,7 @@ public interface serverSwitch { public Iterator /*of serverThread-Names (String)*/ threadNames(); // the switchboard can be used to set and read properties + public void setConfig(String key, long value); public void setConfig(String key, String value); public String getConfig(String key, String dflt); public Iterator configKeys(); diff --git a/source/yacy.java b/source/yacy.java index afe3656f1..486fea7de 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -165,7 +165,7 @@ public final class yacy { * @param homePath Root-path where all information is to be found. * @param startupFree free memory at startup time, to be used later for statistics */ - private static void startup(String homePath, long startupMemFree, long startupMemTotal, long startupMemMax) { + private static void startup(String homePath, long startupMemFree, long startupMemTotal) { long startup = yacyCore.universalTime(); try { @@ -203,7 +203,6 @@ public final class yacy { // save information about available memory at startup time sb.setConfig("memoryFreeAfterStartup", startupMemFree); sb.setConfig("memoryTotalAfterStartup", startupMemTotal); - sb.setConfig("memoryMaxAfterStartup", startupMemMax); // hardcoded, forced, temporary value-migration sb.setConfig("htTemplatePath", "htroot/env/templates"); @@ -426,11 +425,12 @@ public final class yacy { run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb)); // save information about available memory after all initializations + sb.setConfig("memoryFreeAfterInitBGC", Runtime.getRuntime().freeMemory()); + sb.setConfig("memoryTotalAfterInitBGC", Runtime.getRuntime().totalMemory()); System.gc(); - sb.setConfig("memoryFreeAfterInit", Runtime.getRuntime().freeMemory()); - sb.setConfig("memoryTotalAfterInit", Runtime.getRuntime().totalMemory()); - sb.setConfig("memoryMaxAfterInit", Runtime.getRuntime().maxMemory()); - + sb.setConfig("memoryFreeAfterInitAGC", Runtime.getRuntime().freeMemory()); + sb.setConfig("memoryTotalAfterInitAGC", Runtime.getRuntime().totalMemory()); + // wait for server shutdown try { sb.waitForShutdown(); @@ -788,15 +788,14 @@ public final class yacy { System.gc(); long startupMemFree = Runtime.getRuntime().freeMemory(); // the amount of free memory in the Java Virtual Machine long startupMemTotal = Runtime.getRuntime().totalMemory(); // the total amount of memory in the Java virtual machine; may vary over time - long startupMemMax = Runtime.getRuntime().maxMemory(); // the maximum amount of memory that the Java virtual machine will attempt to use - + String applicationRoot = System.getProperty("user.dir").replace('\\', '/'); //System.out.println("args.length=" + args.length); //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]"); if ((args.length >= 1) && ((args[0].equals("-startup")) || (args[0].equals("-start")))) { // normal start-up of yacy if (args.length == 2) applicationRoot= args[1]; - startup(applicationRoot, startupMemFree, startupMemTotal, startupMemMax); + startup(applicationRoot, startupMemFree, startupMemTotal); } else if ((args.length >= 1) && ((args[0].equals("-shutdown")) || (args[0].equals("-stop")))) { // normal shutdown of yacy if (args.length == 2) applicationRoot= args[1]; @@ -824,7 +823,7 @@ public final class yacy { cleanwordlist(args[1], minlength, maxlength); } else { if (args.length == 1) applicationRoot= args[0]; - startup(applicationRoot, startupMemFree, startupMemTotal, startupMemMax); + startup(applicationRoot, startupMemFree, startupMemTotal); } } }