From fb52a82008da0b15d675c3e25a0cb664fe918203 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 20 Sep 2005 10:10:34 +0000 Subject: [PATCH] added new performance page for memory settings git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@751 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/PerformanceMemory_p.html | 189 ++++++++++++++++++ htroot/PerformanceMemory_p.java | 159 +++++++++++++++ ...rmance_p.html => PerformanceQueues_p.html} | 12 +- ...rmance_p.java => PerformanceQueues_p.java} | 4 +- htroot/env/templates/header.template | 2 +- source/de/anomic/data/messageBoard.java | 12 ++ source/de/anomic/data/wikiBoard.java | 22 +- source/de/anomic/kelondro/kelondroMap.java | 8 + .../de/anomic/kelondro/kelondroRecords.java | 78 ++++---- source/de/anomic/plasma/plasmaHTCache.java | 8 + .../de/anomic/plasma/plasmaSwitchboard.java | 2 +- source/de/anomic/plasma/plasmaURL.java | 2 +- source/de/anomic/plasma/plasmaWordIndex.java | 10 +- .../plasma/plasmaWordIndexAssortment.java | 8 + .../plasmaWordIndexAssortmentCluster.java | 17 +- .../anomic/plasma/plasmaWordIndexCache.java | 8 + source/de/anomic/yacy/yacyCore.java | 12 +- source/de/anomic/yacy/yacyNewsDB.java | 8 + source/de/anomic/yacy/yacyNewsPool.java | 12 ++ source/de/anomic/yacy/yacySeedDB.java | 15 +- source/yacy.java | 14 +- yacy.init | 3 + 22 files changed, 534 insertions(+), 71 deletions(-) create mode 100644 htroot/PerformanceMemory_p.html create mode 100644 htroot/PerformanceMemory_p.java rename htroot/{Performance_p.html => PerformanceQueues_p.html} (91%) rename htroot/{Performance_p.java => PerformanceQueues_p.java} (97%) diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html new file mode 100644 index 000000000..78fd2aac6 --- /dev/null +++ b/htroot/PerformanceMemory_p.html @@ -0,0 +1,189 @@ + + + +YaCy '#[clientname]#': Performance Settings for Memory +#[metas]# + + +#[header]# +

+

Performance Settings for Memory

+ +

+

RAM Cache for Database Files:
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Database
Chunk Size
(bytes)		#Needed Slots
(= DB Size)		#Empty Slots
 		#Used Slots
High Priority		#Used Slots
Medium Priority		#Used Slots
Low Priority		Used
Size		Assigned
Max-Size		Recommended
Max-Size		Optimum
Max-Size		Description
 
RWI Assortment Cluster#[chunkRWI]##[slreqRWI]##[slempRWI]##[slhigRWI]##[slmedRWI]##[sllowRWI]##[usedRWI]# bytes#[goodRWI]##[bestRWI]#The Assortment Cluster stores most of the page indexes. +Flushing speed of the temporary RWI cache depends on the size of this file cache. Increasing the space of this +cache will speed up crawls with a depth > 3.
HTTP Response Header#[chunkHTTP]##[slreqHTTP]##[slempHTTP]##[slhigHTTP]##[slmedHTTP]##[sllowHTTP]##[usedHTTP]# bytes#[goodHTTP]##[bestHTTP]#The Response Header database stores the HTTP heades that other servers send when YaCy retrieves web pages +during proxy mode, when performing crawls or if it fetches pages for snippet generation. +Increasing this cache will be most important for a fast proxy mode.
'loaded' URLs#[chunkLURL]##[slreqLURL]##[slempLURL]##[slhigLURL]##[slmedLURL]##[sllowLURL]##[usedLURL]# bytes#[goodLURL]##[bestLURL]#This is the database that holds the hash/url - relation and properties regarding the url like load date and server date. +This cache is very important for a fast search process. Increasing the cache size will result in more search results and less IO during DHT transfer.
'noticed' URLs#[chunkNURL]##[slreqNURL]##[slempNURL]##[slhigNURL]##[slmedNURL]##[sllowNURL]##[usedNURL]# bytes#[goodNURL]##[bestNURL]#A noticed URL is one that was discovered during crawling but was not loaded yet. +Increasing the cache size will result in faster double-check during URL recognition when doing crawls.
'error' URLs#[chunkEURL]##[slreqEURL]##[slempEURL]##[slhigEURL]##[slmedEURL]##[sllowEURL]##[usedEURL]# bytes#[goodEURL]##[bestEURL]#URLs that cannot be loaded are stored in this database. It is also used for double-checked during crawling. +Increasing the cache size will most probably speed up crawling slightly, but not significantly.
DHT Control#[chunkDHT]##[slreqDHT]##[slempDHT]##[slhigDHT]##[slmedDHT]##[sllowDHT]##[usedDHT]# bytes#[goodDHT]##[bestDHT]#This is simply the cache for the seed-dbs (active, passive, potential). +This cache is divided into three equal parts. +Increasing this cache may speed up many functions, but we need to test this to see the effects.
Messages#[chunkMessage]##[slreqMessage]##[slempMessage]##[slhigMessage]##[slmedMessage]##[sllowMessage]##[usedMessage]# bytes#[goodMessage]##[bestMessage]#The Message cache for peer-to-peer messages. Less important.
Wiki#[chunkWiki]##[slreqWiki]##[slempWiki]##[slhigWiki]##[slmedWiki]##[sllowWiki]##[usedWiki]# bytes#[goodWiki]##[bestWiki]#The YaCy-Wiki uses a database to store its pages. +This cache is divided in two parts, one for the wiki database and one for its backup. +Increasing this cache may speed up access to the wiki pages.
News#[chunkNews]##[slreqNews]##[slempNews]##[slhigNews]##[slmedNews]##[sllowNews]##[usedNews]# bytes#[goodNews]##[bestNews]#The News-DB stores property-lists for news that are included in seeds. +Increasing this cache may speed up the peer-ping.
+    +Changes take effect after re-start of YaCy
+

+ +#[footer]# + + diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java new file mode 100644 index 000000000..a20770e70 --- /dev/null +++ b/htroot/PerformanceMemory_p.java @@ -0,0 +1,159 @@ +//PerformaceMemory_p.java +//----------------------- +//part of YaCy +//(C) by Michael Peter Christen; mc@anomic.de +//first published on http://www.anomic.de +//Frankfurt, Germany, 2005 +//last major change: 19.09.2005 +// +//This program is free software; you can redistribute it and/or modify +//it under the terms of the GNU General Public License as published by +//the Free Software Foundation; either version 2 of the License, or +//(at your option) any later version. +// +//This program is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. +// +//You should have received a copy of the GNU General Public License +//along with this program; if not, write to the Free Software +//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +//Using this software in any meaning (reading, learning, copying, compiling, +//running) means that you agree that the Author(s) is (are) not responsible +//for cost, loss of data or any harm that may be caused directly or indirectly +//by usage of this softare or this documentation. The usage of this software +//is on your own risk. The installation and usage (starting/running) of this +//software may allow other people or application to access your computer and +//any attached devices and is highly dependent on the configuration of the +//software which must be done by the user of the software; the author(s) is +//(are) also not responsible for proper configuration and usage of the +//software, even if provoked by documentation provided together with +//the software. +// +//Any changes to this file according to the GPL as documented in the file +//gpl.txt aside this file in the shipment you received can be done to the +//lines that follows this copyright notice here, but changes must not be +//done inside the copyright notive above. A re-distribution must contain +//the intact and unchanged copyright notice. +//Contributions and changes to the program code must be marked as such. + +//You must compile this file with +//javac -classpath .:../classes PerformanceMemory_p.java +//if the shell's current path is HTROOT + +import java.util.Iterator; +import java.util.Map; +import java.io.File; + +import de.anomic.http.httpHeader; +import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.server.serverCore; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; +import de.anomic.server.serverThread; +import de.anomic.server.serverFileUtils; +import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacySeedDB; + +public class PerformanceMemory_p { + + private static int[] slt; + private static int req,chk,usd,bst,god; + + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) { + // return variable that accumulates replacements + plasmaSwitchboard switchboard = (plasmaSwitchboard) sb; + serverObjects prop = new serverObjects(); + File defaultSettingsFile = new File(switchboard.getRootPath(), "yacy.init"); + Map defaultSettings = ((post == null) || (!(post.containsKey("submitdefault")))) ? null : serverFileUtils.loadHashMap(defaultSettingsFile); + + if ((post != null) && (post.containsKey("submitcache"))) { + sb.setConfig("ramCacheRWI", post.get("ramCacheRWI", "0")); + sb.setConfig("ramCacheHTTP", post.get("ramCacheHTTP", "0")); + sb.setConfig("ramCacheLURL", post.get("ramCacheLURL", "0")); + sb.setConfig("ramCacheNURL", post.get("ramCacheNURL", "0")); + sb.setConfig("ramCacheEURL", post.get("ramCacheEURL", "0")); + sb.setConfig("ramCacheDHT", post.get("ramCacheDHT", "0")); + sb.setConfig("ramCacheMessage", post.get("ramCacheMessage", "0")); + sb.setConfig("ramCacheWiki", post.get("ramCacheWiki", "0")); + sb.setConfig("ramCacheNews", post.get("ramCacheNews", "0")); + } + + req = switchboard.wordIndex.size(); + chk = switchboard.wordIndex.assortmentsCacheChunkSizeAvg(); + slt = switchboard.wordIndex.assortmentsCacheFillStatusCml(); + calc(); putprop(prop, "RWI"); + prop.put("ramCacheRWI", sb.getConfig("ramCacheRWI", "0")); + + req = switchboard.cacheManager.size(); + chk = switchboard.cacheManager.dbCacheChunkSize(); + slt = switchboard.cacheManager.dbCacheFillStatus(); + calc(); putprop(prop, "HTTP"); + prop.put("ramCacheHTTP", sb.getConfig("ramCacheHTTP", "0")); + + req = switchboard.urlPool.loadedURL.urlHashCache.size(); + chk = switchboard.urlPool.loadedURL.urlHashCache.cacheChunkSize(); + slt = switchboard.urlPool.loadedURL.urlHashCache.cacheFillStatus(); + calc(); putprop(prop, "LURL"); + prop.put("ramCacheLURL", sb.getConfig("ramCacheLURL", "0")); + + req = switchboard.urlPool.noticeURL.urlHashCache.size(); + chk = switchboard.urlPool.noticeURL.urlHashCache.cacheChunkSize(); + slt = switchboard.urlPool.noticeURL.urlHashCache.cacheFillStatus(); + calc(); putprop(prop, "NURL"); + prop.put("ramCacheNURL", sb.getConfig("ramCacheNURL", "0")); + + req = switchboard.urlPool.errorURL.urlHashCache.size(); + chk = switchboard.urlPool.errorURL.urlHashCache.cacheChunkSize(); + slt = switchboard.urlPool.errorURL.urlHashCache.cacheFillStatus(); + calc(); putprop(prop, "EURL"); + prop.put("ramCacheEURL", sb.getConfig("ramCacheEURL", "0")); + + req = yacyCore.seedDB.sizeConnected() + yacyCore.seedDB.sizeDisconnected() + yacyCore.seedDB.sizePotential(); + chk = yacyCore.seedDB.dbCacheChunkSize(); + slt = yacyCore.seedDB.dbCacheFillStatus(); + calc(); putprop(prop, "DHT"); + prop.put("ramCacheDHT", sb.getConfig("ramCacheDHT", "0")); + + req = switchboard.messageDB.size(); + chk = switchboard.messageDB.dbCacheChunkSize(); + slt = switchboard.messageDB.dbCacheFillStatus(); + calc(); putprop(prop, "Message"); + prop.put("ramCacheMessage", sb.getConfig("ramCacheMessage", "0")); + + req = switchboard.wikiDB.sizeOfTwo(); + chk = switchboard.wikiDB.dbCacheChunkSize(); + slt = switchboard.wikiDB.dbCacheFillStatus(); + calc(); putprop(prop, "Wiki"); + prop.put("ramCacheWiki", sb.getConfig("ramCacheWiki", "0")); + + req = yacyCore.newsPool.dbSize(); + chk = yacyCore.newsPool.dbCacheChunkSize(); + slt = yacyCore.newsPool.dbCacheFillStatus(); + calc(); putprop(prop, "News"); + prop.put("ramCacheNews", sb.getConfig("ramCacheNews", "0")); + + // return rewrite values for templates + return prop; + } + + private static void calc() { + usd = chk * (slt[1]+slt[2]+slt[3]); + bst = (((chk * req) >> 10) + 1) << 10; + god = (((bst / (1+slt[1]+slt[2]+slt[3]) * slt[1]) >> 10) + 1) << 10; + } + + private static void putprop(serverObjects prop, String db) { + prop.put("chunk" + db, chk); + prop.put("slreq" + db, req); + prop.put("slemp" + db, slt[0]); + prop.put("slhig" + db, slt[1]); + prop.put("slmed" + db, slt[2]); + prop.put("sllow" + db, slt[3]); + prop.put("used" + db, usd); + prop.put("good" + db, god); + prop.put("best" + db, bst); + } +} diff --git a/htroot/Performance_p.html b/htroot/PerformanceQueues_p.html similarity index 91% rename from htroot/Performance_p.html rename to htroot/PerformanceQueues_p.html index 72fb46cd5..a28e2a689 100644 --- a/htroot/Performance_p.html +++ b/htroot/PerformanceQueues_p.html @@ -1,17 +1,17 @@ -YaCy '#[clientname]#': Performance +YaCy '#[clientname]#': Performance Settings of Queues and Processes #[metas]# #[header]#

-

Performance

+

Performance Settings of Queues and Processes

Scheduled tasks overview and waiting time settings:
- + @@ -63,7 +63,7 @@ Changes take effect immediately

Indexing Cache Settings:
- +
Thread Queue Size
@@ -102,7 +102,7 @@ Changes take effect immediately

Thread pool settings:
- +
Words in RAM Cache:
@@ -149,7 +149,7 @@ Changes take effect immediately

Proxy Performance Settings:
- +
Thread Pool
diff --git a/htroot/Performance_p.java b/htroot/PerformanceQueues_p.java similarity index 97% rename from htroot/Performance_p.java rename to htroot/PerformanceQueues_p.java index 3b821311a..695f26cb4 100644 --- a/htroot/Performance_p.java +++ b/htroot/PerformanceQueues_p.java @@ -1,4 +1,4 @@ -//Performace_p.java +//PerformaceQueues_p.java //----------------------- //part of YaCy //(C) by Michael Peter Christen; mc@anomic.de @@ -57,7 +57,7 @@ import de.anomic.server.serverSwitch; import de.anomic.server.serverThread; import de.anomic.server.serverFileUtils; -public class Performance_p { +public class PerformanceQueues_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) { // return variable that accumulates replacements diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template index 4eb8f1022..d6a53515c 100644 --- a/htroot/env/templates/header.template +++ b/htroot/env/templates/header.template @@ -64,7 +64,7 @@ - + diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 9751f0523..e0d2bb45d 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -78,6 +78,18 @@ public class messageBoard { sn = 0; } + public int size() { + return database.size(); + } + + public int dbCacheChunkSize() { + return database.cacheChunkSize(); + } + + public int[] dbCacheFillStatus() { + return database.cacheFillStatus(); + } + public void close() throws IOException { database.close(); } diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index edb5fd8fe..01fa1193a 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -74,20 +74,34 @@ public class wikiBoard { new File(actpath.getParent()).mkdir(); if (datbase == null) { if (actpath.exists()) - datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb * 0x40)); + datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x40)); else - datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb * 0x400, keyLength, recordSize)); + datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x400, keyLength, recordSize)); } new File(bkppath.getParent()).mkdir(); if (bkpbase == null) { if (bkppath.exists()) - bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb * 0x400)); + bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400)); else - bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb * 0x400, keyLength + dateFormat.length(), recordSize)); + bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400, keyLength + dateFormat.length(), recordSize)); } sn = 0; } + public int sizeOfTwo() { + return datbase.size() + bkpbase.size(); + } + + public int dbCacheChunkSize() { + return (datbase.cacheChunkSize() + bkpbase.cacheChunkSize()) / 2; + } + + public int[] dbCacheFillStatus() { + int[] a = datbase.cacheFillStatus(); + int[] b = bkpbase.cacheFillStatus(); + return new int[]{a[0] + b[0], a[1] + b[1], a[2] + b[2], a[3] + b[3]}; + } + public void close() throws IOException { datbase.close(); bkpbase.close(); diff --git a/source/de/anomic/kelondro/kelondroMap.java b/source/de/anomic/kelondro/kelondroMap.java index 9308d0590..38e333f32 100644 --- a/source/de/anomic/kelondro/kelondroMap.java +++ b/source/de/anomic/kelondro/kelondroMap.java @@ -127,6 +127,14 @@ public class kelondroMap { return dyn.columnSize(0); } + public int cacheChunkSize() { + return dyn.cacheChunkSize(); + } + + public int[] cacheFillStatus() { + return dyn.cacheFillStatus(); + } + public synchronized void set(String key, Map newMap) throws IOException { // update elementCount if ((sortfields != null) || (accfields != null)) { diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index ea9d42466..f08333ece 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -159,19 +159,7 @@ public class kelondroRecords { //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename)); //kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth); - this.XcacheSize = (int) (buffersize / ((long) (overhead + columns[0]))); - if (XcacheSize <= 0) { - XcacheSize = 0; - this.XcacheHeaders = null; - this.XcacheScore = null; - } else { - // initialize cache - // in case that we have more cache space than elements - // we dont need cache control by the XcacheScore - this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()}; - this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster(); - } - this.XcacheStartup = System.currentTimeMillis(); + initCache(buffersize); } public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, @@ -179,16 +167,7 @@ public class kelondroRecords { int[] columns, int FHandles, int txtProps, int txtPropWidth) throws IOException { this.filename = null; init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth); - this.XcacheSize = (int) (buffersize / ((long) (overhead + columns[0]))); - if (XcacheSize <= 0) { - XcacheSize = 0; - this.XcacheHeaders = null; - this.XcacheScore = null; - } else { - this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()}; - this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster(); - } - this.XcacheStartup = System.currentTimeMillis(); + initCache(buffersize); } private void init(kelondroRA ra, short ohbytec, short ohhandlec, @@ -275,31 +254,13 @@ public class kelondroRecords { //kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000); //kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000); init(raf); - this.XcacheSize = (int) (buffersize / ((long) (overhead + COLWIDTHS[0]))); - if (XcacheSize <= 0) { - XcacheSize = 0; - this.XcacheHeaders = null; - this.XcacheScore = null; - } else { - this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()}; - this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster(); - } - this.XcacheStartup = System.currentTimeMillis(); + initCache(buffersize); } public kelondroRecords(kelondroRA ra, long buffersize) throws IOException{ this.filename = null; init(ra); - this.XcacheSize = (int) (buffersize / ((long) (overhead + COLWIDTHS[0]))); - if (XcacheSize <= 0) { - XcacheSize = 0; - this.XcacheHeaders = null; - this.XcacheScore = null; - } else { - this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()}; - this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster(); - } - this.XcacheStartup = System.currentTimeMillis(); + initCache(buffersize); } private void init(kelondroRA ra) throws IOException { @@ -350,7 +311,38 @@ public class kelondroRecords { this.tailchunksize = this.recordsize - this.headchunksize; } + private void initCache(long buffersize) { + if (buffersize <= 0) { + this.XcacheSize = 0; + this.XcacheHeaders = null; + this.XcacheScore = null; + } else { + if ((buffersize / cacheChunkSize(false)) > size()) { + this.XcacheSize = (int) (buffersize / cacheChunkSize(false)); + this.XcacheScore = null; + } else { + this.XcacheSize = (int) (buffersize / cacheChunkSize(true)); + this.XcacheScore = new kelondroMScoreCluster(); + } + this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()}; + } + this.XcacheStartup = System.currentTimeMillis(); + } + + private int cacheChunkSize(boolean cacheControl) { + return this.headchunksize + 14 + ((cacheControl) ? 16 : 0); + } + + public int cacheChunkSize() { + return cacheChunkSize(this.XcacheScore != null); + } + public int[] cacheFillStatus() { + if (XcacheHeaders == null) return new int[]{0,0,0,0}; + return new int[]{XcacheSize - (XcacheHeaders[CP_HIGH].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_LOW].size()), XcacheHeaders[CP_HIGH].size(), XcacheHeaders[CP_MEDIUM].size(), XcacheHeaders[CP_LOW].size()}; + } + + protected Node newNode() throws IOException { return new Node(); } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 637535906..62bb75bbd 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -135,6 +135,14 @@ public final class plasmaHTCache { } } + public int dbCacheChunkSize() { + return responseHeaderDB.cacheChunkSize(); + } + + public int[] dbCacheFillStatus() { + return responseHeaderDB.cacheFillStatus(); + } + public void push(Entry entry) { synchronized (this.cacheStack) { this.cacheStack.add(entry); diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 6f2bdda7f..18c96eb81 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -154,7 +154,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public static plasmaURLPattern urlBlacklist; // storage management - private File cachePath; // do we need that ? + private File cachePath; private File plasmaPath; public File listsPath; public plasmaURLPool urlPool; diff --git a/source/de/anomic/plasma/plasmaURL.java b/source/de/anomic/plasma/plasmaURL.java index 7c905c36d..39cb4c1d6 100644 --- a/source/de/anomic/plasma/plasmaURL.java +++ b/source/de/anomic/plasma/plasmaURL.java @@ -93,7 +93,7 @@ public class plasmaURL { } // the class object - protected kelondroTree urlHashCache; + public kelondroTree urlHashCache; private HashSet existsIndex; public plasmaURL() throws IOException { diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 1c433355b..8799b3a2f 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -82,7 +82,15 @@ public final class plasmaWordIndex { public int[] assortmentSizes() { return ramCache.assortmentsSizes(); } - + + public int assortmentsCacheChunkSizeAvg() { + return ramCache.assortmentsCacheChunkSizeAvg(); + } + + public int[] assortmentsCacheFillStatusCml() { + return ramCache.assortmentsCacheFillStatusCml(); + } + public void setMaxWords(int maxWords) { ramCache.setMaxWords(maxWords); } diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index bbff454c4..3380530fe 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -213,6 +213,14 @@ public final class plasmaWordIndexAssortment { return assortments.size(); } + public int cacheChunkSize() { + return assortments.cacheChunkSize(); + } + + public int[] cacheFillStatus() { + return assortments.cacheFillStatus(); + } + public void close() { try { assortments.close(); diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java b/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java index 35fed67a5..ec969f4a8 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java @@ -190,7 +190,22 @@ public final class plasmaWordIndexAssortmentCluster { for (int i = 0; i < clusterCount; i++) sizes[i] = assortments[i].size(); return sizes; } - + + public int cacheChunkSizeAvg() { + int total = 0; + for (int i = 0; i < clusterCount; i++) total += assortments[i].cacheChunkSize(); + return total / clusterCount; + } + + public int[] cacheFillStatusCml() { + int[] a, cml = new int[]{0, 0, 0, 0}; + for (int i = 0; i < clusterCount; i++) { + a = assortments[i].cacheFillStatus(); + for (int j = 0; j < 4; j++) cml[j] += a[j]; + } + return cml; + } + public void close() { for (int i = 0; i < clusterCount; i++) assortments[i].close(); } diff --git a/source/de/anomic/plasma/plasmaWordIndexCache.java b/source/de/anomic/plasma/plasmaWordIndexCache.java index fb0aea3fe..6bf2de41c 100644 --- a/source/de/anomic/plasma/plasmaWordIndexCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexCache.java @@ -265,6 +265,14 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { return assortmentCluster.sizes(); } + public int assortmentsCacheChunkSizeAvg() { + return assortmentCluster.cacheChunkSizeAvg(); + } + + public int[] assortmentsCacheFillStatusCml() { + return assortmentCluster.cacheFillStatusCml(); + } + public int size() { return java.lang.Math.max(assortmentCluster.sizeTotal(), java.lang.Math.max(backend.size(), cache.size())); } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index acbedc2d5..cd4c910d8 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -153,20 +153,20 @@ public class yacyCore { yacyDBPath = new File(sb.getRootPath(), sb.getConfig("yacyDB", "DATA/YACYDB")); if (!(yacyDBPath.exists())) yacyDBPath.mkdir(); - // read memory amount - int mem = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024; - log.logConfig("DHT Cache memory = " + mem + " KB"); - // create or init seed cache + int memDHT = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024; + log.logConfig("DHT Cache memory = " + memDHT + " KB"); seedDB = new yacySeedDB( sb, new File(yacyDBPath, "seed.new.db"), new File(yacyDBPath, "seed.old.db"), new File(yacyDBPath, "seed.pot.db"), - mem); + memDHT); // create or init news database - newsPool = new yacyNewsPool(yacyDBPath, 1024); + int memNews = Integer.parseInt(switchboard.getConfig("ramCacheNews", "1024")) / 1024; + log.logConfig("News Cache memory = " + memNews + " KB"); + newsPool = new yacyNewsPool(yacyDBPath, memNews); loadSeedUploadMethods(); diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index d8cb17d85..82ac59961 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -89,6 +89,14 @@ public class yacyNewsDB { news = createDB(path, bufferkb); } + public int dbCacheChunkSize() { + return news.cacheChunkSize(); + } + + public int[] dbCacheFillStatus() { + return news.cacheFillStatus(); + } + public void close() { if (news != null) try {news.close();} catch (IOException e) {} news = null; diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index 5f299761f..1e2c060d0 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -99,6 +99,18 @@ public class yacyNewsPool { maxDistribution = 30; } + public int dbSize() { + return newsDB.size(); + } + + public int dbCacheChunkSize() { + return newsDB.dbCacheChunkSize(); + } + + public int[] dbCacheFillStatus() { + return newsDB.dbCacheFillStatus(); + } + public void publishMyNews(yacyNewsRecord record) throws IOException { // this shall be called if our peer generated a new news record and wants to publish it if (newsDB.get(record.id()) == null) { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index e2142e53c..86806d566 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -147,10 +147,21 @@ public final class yacySeedDB { } catch (IOException e) {} } + public int dbCacheChunkSize() { + return (seedActiveDB.cacheChunkSize() + seedPassiveDB.cacheChunkSize() + seedPotentialDB.cacheChunkSize()) / 3; + } + + public int[] dbCacheFillStatus() { + int[] ac = seedActiveDB.cacheFillStatus(); + int[] pa = seedPassiveDB.cacheFillStatus(); + int[] po = seedPotentialDB.cacheFillStatus(); + return new int[]{ac[0] + pa[0] + po[0], ac[1] + pa[1] + po[1], ac[2] + pa[2] + po[2], ac[3] + pa[3] + po[3]}; + } + private synchronized kelondroMap openSeedTable(File seedDBFile) throws IOException { if (seedDBFile.exists()) try { // open existing seed database - return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB * 0x400), sortFields, accFields); + return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB/3 * 0x400), sortFields, accFields); } catch (kelondroException e) { // if we have an error, we start with a fresh database if (seedDBFile.exists()) seedDBFile.delete(); @@ -160,7 +171,7 @@ public final class yacySeedDB { } // create new seed database new File(seedDBFile.getParent()).mkdir(); - return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB * 0x400, commonHashLength, 480), sortFields, accFields); + return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB/3 * 0x400, commonHashLength, 480), sortFields, accFields); } private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) { diff --git a/source/yacy.java b/source/yacy.java index 7496f0d81..b64ab53c0 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -166,8 +166,9 @@ public final class yacy { * the main threads. * * @param homePath Root-path where all information is to be found. + * @param startupFree free memory at startup time, to be used later for statistics */ - private static void startup(String homePath) { + private static void startup(String homePath, long startupFree) { long startup = yacyCore.universalTime(); try { // start up @@ -201,6 +202,9 @@ public final class yacy { plasmaSwitchboard sb = new plasmaSwitchboard(homePath, "yacy.init", "DATA/SETTINGS/httpProxy.conf"); + // save information about available memory at startup time + sb.setConfig("memoryFreeAfterStartup", startupFree); // for statistics and memory configuration + // hardcoded, forced, temporary value-migration sb.setConfig("htTemplatePath", "htroot/env/templates"); sb.setConfig("parseableExt", "html,htm,txt,php,shtml,asp"); @@ -413,6 +417,9 @@ public final class yacy { Runtime run = Runtime.getRuntime(); run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb)); + // save information about available memory after all initializations + System.gc(); sb.setConfig("memoryFreeAfterInit", Runtime.getRuntime().freeMemory()); + // wait for server shutdown try { sb.waitForShutdown(); @@ -766,13 +773,14 @@ public final class yacy { * @param args Given arguments from the command line. */ public static void main(String args[]) { + System.gc(); long startupFree = Runtime.getRuntime().freeMemory(); String applicationRoot = System.getProperty("user.dir").replace('\\', '/'); //System.out.println("args.length=" + args.length); //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]"); if ((args.length >= 1) && ((args[0].equals("-startup")) || (args[0].equals("-start")))) { // normal start-up of yacy if (args.length == 2) applicationRoot= args[1]; - startup(applicationRoot); + startup(applicationRoot, startupFree); } else if ((args.length >= 1) && ((args[0].equals("-shutdown")) || (args[0].equals("-stop")))) { // normal shutdown of yacy if (args.length == 2) applicationRoot= args[1]; @@ -800,7 +808,7 @@ public final class yacy { cleanwordlist(args[1], minlength, maxlength); } else { if (args.length == 1) applicationRoot= args[0]; - startup(applicationRoot); + startup(applicationRoot, startupFree); } } } diff --git a/yacy.init b/yacy.init index a608251eb..1c9832a38 100644 --- a/yacy.init +++ b/yacy.init @@ -461,6 +461,9 @@ ramCacheMessage = 8192 # ram cache for wiki.db ramCacheWiki = 8192 +# ram cache for news1.db +ramCacheNews = 8192 + # performance properties for the word index cache # wordCacheMax is the number of word indexes that shall be held in the # ram cache during indexing. When YaCy is shut down, this cache must be
Online Caution Delay (milliseconds):
 News
  Log
  Settings
  Performance
  Performance
  Language
  Skins