From 7fb645b0ab782d496418a9b6cab6455d214a2187 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 24 Apr 2005 23:15:40 +0000 Subject: [PATCH] enhanced crawling performance, changed memory settings, new performace options git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@51 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Performance_p.html | 62 ++++++++++++++++++- htroot/Performance_p.java | 23 +++++-- source/de/anomic/http/httpdSwitchboard.java | 10 ++- .../anomic/kelondro/kelondroBufferedRA.java | 31 ++++------ .../de/anomic/kelondro/kelondroRecords.java | 28 +-------- source/de/anomic/plasma/plasmaHTCache.java | 2 +- .../de/anomic/plasma/plasmaSwitchboard.java | 50 +++++++++------ source/de/anomic/plasma/plasmaWordIndex.java | 14 ++++- .../plasma/plasmaWordIndexRAMCache.java | 24 +++++-- .../anomic/server/serverAbstractSwitch.java | 2 +- source/de/anomic/server/serverSwitch.java | 2 +- yacy.init | 52 ++++++++++------ 12 files changed, 196 insertions(+), 104 deletions(-) diff --git a/htroot/Performance_p.html b/htroot/Performance_p.html index b63860fe8..0b565dad5 100644 --- a/htroot/Performance_p.html +++ b/htroot/Performance_p.html @@ -9,7 +9,7 @@

Performance

-You can manipulate some performance attributes here.
+

Scheduled tasks overview and waiting time settings:
@@ -47,10 +47,66 @@ You can manipulate some performance attributes here.
#{/table}# -
#[longdescr]#

+ + -Changes take effect immidiately +Changes take effect immidiately + +

+ +

+

Indexing Cache Settings:
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Words in RAM Cache:#[wordCacheRAMSize]# + This is the current size of the word cache. + The smaller this number, the faster will be the shut-down procedure. + The maximum of this cache can be set below. +
Maximum URLs currently assigned
to one cached word:		#[maxURLinWordCache]# + This is the maximum size of URLs assigned to a single word cache entry. + If this is a big number, it shows that the caching works efficiently. +
Maximum number of Word Caches: + This is is the number of word indexes that shall be held in the + ram cache during indexing. When YaCy is shut down, this cache must be + flushed to disc; this may last some minutes. +
Maximum waitingtime to flush word cache
during shut-down (seconds):		 + This gives the number of seconds that the shutdown + may last for the final word flush. Words that cannot be flushed within this + time are lost. +
+ Changes take effect immidiately
+

#[footer]# diff --git a/htroot/Performance_p.java b/htroot/Performance_p.java index 9ca8d1552..f79777696 100644 --- a/htroot/Performance_p.java +++ b/htroot/Performance_p.java @@ -109,11 +109,7 @@ public class Performance_p { prop.put("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? "-" : ("" + (sleeptime / (idleCycles + busyCycles)))); prop.put("table_" + c + "_execpercycle", (busyCycles == 0) ? "-" : ("" + (exectime / busyCycles))); - if (post == null) { - // load with old values - idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000")); - busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000")); - } else { + if ((post != null) && (post.containsKey("delaysubmit"))) { // load with new values idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep", "1")) * 1000; busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1")) * 1000; @@ -122,6 +118,10 @@ public class Performance_p { switchboard.setThreadSleep(threadName, idlesleep, busysleep); switchboard.setConfig(threadName + "_idlesleep", idlesleep); switchboard.setConfig(threadName + "_busysleep", busysleep); + } else { + // load with old values + idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000")); + busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000")); } prop.put("table_" + c + "_idlesleep", idlesleep / 1000); prop.put("table_" + c + "_busysleep", busysleep / 1000); @@ -130,6 +130,19 @@ public class Performance_p { } prop.put("table", c); + if ((post != null) && (post.containsKey("cacheSizeSubmit"))) { + int wordCacheMax = Integer.parseInt((String) post.get("wordCacheMax", "10000")); + switchboard.setConfig("wordCacheMax", "" + wordCacheMax); + switchboard.wordIndex.setMaxWords(wordCacheMax); + int maxWaitingWordFlush = Integer.parseInt((String) post.get("maxWaitingWordFlush", "180")); + switchboard.setConfig("maxWaitingWordFlush", "" + maxWaitingWordFlush); + } + // table cache settings + prop.put("wordCacheRAMSize", switchboard.wordIndex.wordCacheRAMSize()); + prop.put("maxURLinWordCache", "" + switchboard.wordIndex.maxURLinWordCache()); + prop.put("maxWaitingWordFlush", switchboard.getConfig("maxWaitingWordFlush", "180")); + prop.put("wordCacheMax", switchboard.getConfig("wordCacheMax", "10000")); + // return rewrite values for templates return prop; } diff --git a/source/de/anomic/http/httpdSwitchboard.java b/source/de/anomic/http/httpdSwitchboard.java index 55f20cae9..1e9396431 100644 --- a/source/de/anomic/http/httpdSwitchboard.java +++ b/source/de/anomic/http/httpdSwitchboard.java @@ -61,8 +61,14 @@ public final class httpdSwitchboard extends serverAbstractSwitch implements serv cacheStack.addLast(job); } - public void deQueue() { - System.out.println("Process: " + cacheStack.removeFirst().toString()); + public boolean deQueue() { + if (cacheStack.size() > 0) { + System.out.println("Process: " + cacheStack.removeFirst().toString()); + return true; + } else { + System.out.println("Process: queue is empty"); + return false; + } } public serverObjects action(String actionName, serverObjects actionInput) { diff --git a/source/de/anomic/kelondro/kelondroBufferedRA.java b/source/de/anomic/kelondro/kelondroBufferedRA.java index 83439e0c7..42f19b093 100644 --- a/source/de/anomic/kelondro/kelondroBufferedRA.java +++ b/source/de/anomic/kelondro/kelondroBufferedRA.java @@ -79,16 +79,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA // delete elements in buffer if buffer too big Iterator it = bufferScore.scores(true); Integer element = (Integer) it.next(); + writeBuffer((byte[]) bufferMemory.get(element), element.intValue()); bufferMemory.remove(element); int age = bufferScore.deleteScore(element); - int minclean = bufferMaxElements / 8; - int cleaned = 1; - while ((cleaned++ < minclean) && (it.hasNext())) { - element = (Integer) it.next(); - bufferMemory.remove(element); - age = bufferScore.deleteScore(element); - } - de.anomic.server.serverLog.logDebug("CACHE: " + name, "GC; cleaned=" + (cleaned - 1) + ", age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000)); + de.anomic.server.serverLog.logDebug("CACHE: " + name, "GC; age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000)); } // add new element buffer = new byte[bufferElementSize]; @@ -101,14 +95,6 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA return buffer; } - /* - private static int log2i(int x) { - int log = 0; - while (x != 0) {x >>= 1; log++;} - return log; - } - */ - private void writeBuffer(byte[] buffer, int bufferNr) throws IOException { if (buffer == null) return; Integer bufferNrI = new Integer(bufferNr); @@ -132,7 +118,7 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA byte[] buffer = readBuffer(bn); seekpos++; buffer[offset] = (byte) b; - writeBuffer(buffer, bn); + //writeBuffer(buffer, bn); } public int read(byte[] b, int off, int len) throws IOException { @@ -165,13 +151,13 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA // simple case System.arraycopy(b, off, buffer, offset, len); seekpos += len; - writeBuffer(buffer, bn1); + //writeBuffer(buffer, bn1); } else { // do recursively int thislen = bufferElementSize - offset; System.arraycopy(b, off, buffer, offset, thislen); seekpos += thislen; - writeBuffer(buffer, bn1); + //writeBuffer(buffer, bn1); write(b, thislen, len - thislen); } } @@ -181,6 +167,13 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA } public void close() throws IOException { + // write all unwritten buffers + Iterator it = bufferScore.scores(true); + while (it.hasNext()) { + Integer element = (Integer) it.next(); + writeBuffer((byte[]) bufferMemory.get(element), element.intValue()); + bufferMemory.remove(element); + } ra.close(); bufferScore = null; bufferMemory = null; diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index aff2261a4..2070b16c3 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -142,6 +142,7 @@ public class kelondroRecords { throw new IOException("kelondroRecords: tree file " + file + " already exist"); this.filename = file.getCanonicalPath(); kelondroRA raf = new kelondroFileRA(this.filename); + //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000); //kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth); this.cachesize = (int) (buffersize / ((long) (overhead + recordsize))); @@ -240,6 +241,7 @@ public class kelondroRecords { this.filename = file.getCanonicalPath(); kelondroRA raf = new kelondroFileRA(this.filename); + //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000); //kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000); init(raf); this.cachesize = (int) (buffersize / ((long) (overhead + recordsize))); @@ -353,32 +355,6 @@ public class kelondroRecords { // we simply clear the cache this.cacheScore = new kelondroMScoreCluster(); this.cache = new HashMap(); - - /* - java.util.NoSuchElementException - at java.util.TreeMap.key(TreeMap.java:431) - at java.util.TreeMap.firstKey(TreeMap.java:286) - at de.anomic.kelondro.kelondroMScoreCluster$scoreIterator.internalNext(kelondroMScoreCluster.java:235) - at de.anomic.kelondro.kelondroMScoreCluster$scoreIterator.(kelondroMScoreCluster.java:224) - at de.anomic.kelondro.kelondroMScoreCluster.scores(kelondroMScoreCluster.java:209) - at de.anomic.kelondro.kelondroMScoreCluster.getScores(kelondroMScoreCluster.java:191) - at de.anomic.kelondro.kelondroMScoreCluster.getScores(kelondroMScoreCluster.java:185) - at de.anomic.kelondro.kelondroMScoreCluster.getMinObject(kelondroMScoreCluster.java:181) - at de.anomic.kelondro.kelondroRecords.checkCacheSpace(kelondroRecords.java:344) - at de.anomic.kelondro.kelondroRecords.access$1500(kelondroRecords.java:74) - at de.anomic.kelondro.kelondroRecords$Node.updateNode(kelondroRecords.java:624) - at de.anomic.kelondro.kelondroRecords$Node.(kelondroRecords.java:399) - at de.anomic.kelondro.kelondroRecords$Node.(kelondroRecords.java:350) - at de.anomic.kelondro.kelondroRecords.getNode(kelondroRecords.java:321) - at de.anomic.kelondro.kelondroTree$Search.searchproc(kelondroTree.java:265) - at de.anomic.kelondro.kelondroTree$Search.(kelondroTree.java:246) - at de.anomic.kelondro.kelondroTree.get(kelondroTree.java:317) - at de.anomic.plasma.plasmaURL.exists(plasmaURL.java:106) - at de.anomic.plasma.plasmaSwitchboard.stackCrawl(plasmaSwitchboard.java:389) - at de.anomic.plasma.plasmaSwitchboard.processResourceStack(plasmaSwitchboard.java:304) - at de.anomic.plasma.plasmaSwitchboard.deQueue(plasmaSwitchboard.java:262) - at yacyProxy.main(yacyProxy.java:228) - */ } } } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 57c5ec71b..65dba7449 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -257,7 +257,7 @@ public final class plasmaHTCache { entry.cacheFile.getParentFile().mkdirs(); log.logInfo("WRITE FILE (" + entry.cacheArray.length + " bytes) " + entry.cacheFile); serverFileUtils.write(entry.cacheArray, entry.cacheFile); - log.logInfo("AFTER WRITE cacheArray = " + entry.cacheFile + ": " + ((entry.cacheArray == null) ? "empty" : "full")); + log.logDebug("AFTER WRITE cacheArray = " + entry.cacheFile + ": " + ((entry.cacheArray == null) ? "empty" : "full")); //entry.cacheArray = null; } catch (FileNotFoundException e) { // this is the case of a "(Not a directory)" error, which should be prohibited diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index d91b983e3..d4ffd40b9 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -190,19 +190,20 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi } // read memory amount - int ramCacheKB = Integer.parseInt(getConfig("ramCacheSize", "1")) * 0x400; - int ramLURL = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentLURL", "1")) / 100; - int ramPURL = ramLURL / 2; - int ramEURL = ramLURL / 2; - int ramRWI = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentRWI", "1")) / 100; - int ramHTTP = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentHTTP", "1")) / 100; - int ramMessage = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentMessage", "1")) / 100; - int ramWiki = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentWiki", "1")) / 100; - log.logSystem("LURL Cache memory = " + ramLURL + " KB"); - log.logSystem("RWI Cache memory = " + ramRWI + " KB"); - log.logSystem("HTTP Cache memory = " + ramHTTP + " KB"); - log.logSystem("Message Cache memory = " + ramMessage + " KB"); - log.logSystem("Wiki Cache memory = " + ramWiki + " KB"); + int ramLURL = Integer.parseInt(getConfig("ramCacheLURL", "1024")) / 1024; + int ramNURL = Integer.parseInt(getConfig("ramCacheNURL", "1024")) / 1024; + int ramEURL = Integer.parseInt(getConfig("ramCacheEURL", "1024")) / 1024; + int ramRWI = Integer.parseInt(getConfig("ramCacheRWI", "1024")) / 1024; + int ramHTTP = Integer.parseInt(getConfig("ramCacheHTTP", "1024")) / 1024; + int ramMessage = Integer.parseInt(getConfig("ramCacheMessage", "1024")) / 1024; + int ramWiki = Integer.parseInt(getConfig("ramCacheWiki", "1024")) / 1024; + log.logSystem("LURL Cache memory = " + ppRamString(ramLURL)); + log.logSystem("NURL Cache memory = " + ppRamString(ramNURL)); + log.logSystem("EURL Cache memory = " + ppRamString(ramEURL)); + log.logSystem("RWI Cache memory = " + ppRamString(ramRWI)); + log.logSystem("HTTP Cache memory = " + ppRamString(ramHTTP)); + log.logSystem("Message Cache memory = " + ppRamString(ramMessage)); + log.logSystem("Wiki Cache memory = " + ppRamString(ramWiki)); // make crawl profiles database and default profiles profiles = new plasmaCrawlProfile(new File(plasmaPath, "crawlProfiles0.db")); @@ -213,10 +214,10 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi // start indexing management loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL); - noticeURL = new plasmaCrawlNURL(plasmaPath, ramPURL); + noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL); errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL); - //indexCache = new plasmaWordIndexRAMCache(plasmaPath, 2000, ramRWI); - wordIndex = new plasmaWordIndex(plasmaPath, ramRWI); + wordIndex = new plasmaWordIndex(plasmaPath, ramRWI); + wordIndex.setMaxWords(10000); searchManager = new plasmaSearch(loadedURL, wordIndex); // start a cache manager @@ -290,6 +291,15 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi new serverInstantThread(indexDistribution, "job", null), log, 120000); } + private static String ppRamString(int bytes) { + if (bytes < 1024) return bytes + " KByte"; + bytes = bytes / 1024; + if (bytes < 1024) return bytes + " MByte"; + bytes = bytes / 1024; + if (bytes < 1024) return bytes + " GByte"; + return (bytes / 1024) + "TByte"; + } + public void handleBusyState(int jobs) { this.serverJobs = jobs; } @@ -359,7 +369,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi log.logSystem("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:"); terminateAllThreads(true); log.logSystem("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing (stand by..)"); - int waitingBoundSeconds = Integer.parseInt(getConfig("shutdownWaiting", "120")); + int waitingBoundSeconds = Integer.parseInt(getConfig("maxWaitingWordFlush", "120")); wordIndex.close(waitingBoundSeconds); log.logSystem("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager"); try { @@ -397,16 +407,18 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi processStack.addLast(entry); } - public synchronized void deQueue() { + public synchronized boolean deQueue() { if (serverJobs < 6) { if (processStack.size() > 0) { processResourceStack((plasmaHTCache.Entry) processStack.removeFirst()); + return true; } } else { //if (processStack.size() > 0) { log.logDebug("DEQUEUE: serverJobs=" + serverJobs + " 'busy' - no dequeueing (processStack=" + processStack.size() + ", localStackSize=" + noticeURL.localStackSize() + ", remoteStackSize=" + noticeURL.remoteStackSize() + ")"); //} } + return false; } public int cleanupJobSize() { @@ -444,7 +456,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi } public boolean localCrawlJob() { - if ((serverJobs < 2) && + if ((serverJobs < 6) && (processStack.size() < crawlSlots) && (noticeURL.localStackSize() > 0) && (cacheLoader.size() < crawlSlots)) { diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 2cb9c431f..a7256ee8f 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -57,10 +57,22 @@ public class plasmaWordIndex { public plasmaWordIndex(File databaseRoot, int bufferkb) throws IOException { this.databaseRoot = databaseRoot; - this.ramCache = new plasmaWordIndexRAMCache(databaseRoot, 2000, bufferkb); + this.ramCache = new plasmaWordIndexRAMCache(databaseRoot, bufferkb); ramCache.start(); } + public int maxURLinWordCache() { + return ramCache.maxURLinWordCache(); + } + + public int wordCacheRAMSize() { + return ramCache.wordCacheRAMSize(); + } + + public void setMaxWords(int maxWords) { + ramCache.setMaxWords(maxWords); + } + public synchronized int addEntry(String wordHash, plasmaWordIndexEntry entry) throws IOException { return ramCache.addEntryToIndexMem(wordHash, entry); } diff --git a/source/de/anomic/plasma/plasmaWordIndexRAMCache.java b/source/de/anomic/plasma/plasmaWordIndexRAMCache.java index 26890a893..fc3d595c8 100644 --- a/source/de/anomic/plasma/plasmaWordIndexRAMCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexRAMCache.java @@ -70,14 +70,26 @@ public class plasmaWordIndexRAMCache extends Thread { for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += '-'; } - public plasmaWordIndexRAMCache(File databaseRoot, int maxWords, int bufferkb) throws IOException { + public plasmaWordIndexRAMCache(File databaseRoot, int bufferkb) throws IOException { this.pic = new plasmaWordIndexFileCache(databaseRoot, bufferkb); this.cache = new TreeMap(); this.hashScore = new kelondroMScoreCluster(); - this.maxWords = maxWords; + this.maxWords = 1000; this.terminate = false; } + public int maxURLinWordCache() { + return hashScore.getScore(hashScore.getMaxObject()); + } + + public int wordCacheRAMSize() { + return cache.size(); + } + + public void setMaxWords(int maxWords) { + this.maxWords = maxWords; + } + public void run() { serverLog.logSystem("PLASMA INDEXING", "started word cache management"); int check; @@ -86,9 +98,9 @@ public class plasmaWordIndexRAMCache extends Thread { if (hashScore.size() < 100) try {Thread.currentThread().sleep(10000);} catch (InterruptedException e) {} while ((!(terminate)) && (cache != null) && (hashScore.size() > 0)) try { //check = hashScore.size(); - flushSpecific(true); + flushSpecific(false); //serverLog.logDebug("PLASMA INDEXING", "single flush. bevore=" + check + "; after=" + hashScore.size()); - try {Thread.currentThread().sleep(200 + (maxWords / (1 + hashScore.size())));} catch (InterruptedException e) {} + try {Thread.currentThread().sleep(10 + ((maxWords / 10) / (1 + hashScore.size())));} catch (InterruptedException e) {} } catch (IOException e) { serverLog.logError("PLASMA INDEXING", "PANIK! exception in main cache loop: " + e.getMessage()); e.printStackTrace(); @@ -158,7 +170,7 @@ public class plasmaWordIndexRAMCache extends Thread { String key = (String) ((greatest) ? hashScore.getMaxObject() : hashScore.getMinObject()); return flushKey(key, "flushSpecific"); } - + private synchronized int flushKey(String key, String caller) throws IOException { Vector v = (Vector) cache.get(key); if (v == null) { @@ -197,7 +209,7 @@ public class plasmaWordIndexRAMCache extends Thread { // make space for new words int flushc = 0; //serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem: cache.size=" + cache.size() + "; hashScore.size=" + hashScore.size()); - while (hashScore.size() > maxWords) flushc += flushSpecific(false); + while (hashScore.size() > maxWords) flushc += flushSpecific(true); //if (flushc > 0) serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem - flushed " + flushc + " entries"); // put new words into cache diff --git a/source/de/anomic/server/serverAbstractSwitch.java b/source/de/anomic/server/serverAbstractSwitch.java index 25aeb2756..7530dae24 100644 --- a/source/de/anomic/server/serverAbstractSwitch.java +++ b/source/de/anomic/server/serverAbstractSwitch.java @@ -225,7 +225,7 @@ public abstract class serverAbstractSwitch implements serverSwitch { abstract public int queueSize(); abstract public void enQueue(Object job); - abstract public void deQueue(); + abstract public boolean deQueue(); // authentification routines: diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index e926371fb..a2ce82963 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -74,7 +74,7 @@ public interface serverSwitch { // work off exactly only one job, not all public int queueSize(); public void enQueue(Object job); - public void deQueue(); + public boolean deQueue(); // returns true if there had been dequeued anything // authentification routines: sets and reads access attributes according to host addresses public void setAuthentify(InetAddress host, String user, String rigth); diff --git a/yacy.init b/yacy.init index f666272d4..8a2aa99ef 100644 --- a/yacy.init +++ b/yacy.init @@ -12,13 +12,6 @@ # port number of server port = 8080 -# shutdown time limit -# this is the time that a peer takes at most for shutdown -# the shutdown-procedure is difficult since there are many -# caches that must be flushed first -# measured in seconds -shutdownWaiting = 120 - # time-out of client control socket in milliseconds # since this applies only to the client-proxy connection, # it can be rather short @@ -94,19 +87,6 @@ parseableMime=application/xhtml+xml,text/html,text/plain # These files will be excluded from indexing mediaExt=swf,wmv,jpg,jpeg,jpe,rm,mov,mpg,mpeg,mp3,asf,gif,png,avi,zip,rar,sit,hqx,img,dmg,tar,gz,ps,pdf,doc,xls,ppt,ram,bz2,arj,jar,deb,torrent,ogg,iso,bin,ace,tgz,rpm,css -# the proxy's and indexing maximum ram cache size in megabytes -ramCacheSize = 12 -# ram cache is partitioned into different separate caches -# we define percentual shares of the complete ram cache -# values here are computed from usage profiles and should not be changed -# it is not computed if the complete sum of percentages computes to 100 -ramCachePercentRWI = 85 -ramCachePercentHTTP = 6 -ramCachePercentLURL = 4 -ramCachePercentDHT = 3 -ramCachePercentMessage = 1 -ramCachePercentWiki = 1 - # Promotion Strings # These strings appear in the Web Mask of the YACY search client # Set these Strings to cusomize your peer and give any message to @@ -401,3 +381,35 @@ xpstopw=true 80_dequeue_busysleep=1000 90_cleanup_idlesleep=300000 90_cleanup_busysleep=300000 + +# ram cache for database files + +# ram cache for indexCache.db +ramCacheRWI = 16777216 + +# ram cache for responseHeader.db +ramCacheHTTP = 1048576 + +# ram cache for urlHash.db +ramCacheLURL = 8388608 + +# ram cache for urlNotice.db +ramCacheNURL = 1048576 + +# ram cache for urlErr.db +ramCacheEURL = 131072 + +# ram cache for message.db +ramCacheMessage = 8192 + +# ram cache for wiki.db +ramCacheWiki = 8192 + +# performance properties for the word index cache +# wordCacheMax is the number of word indexes that shall be held in the +# ram cache during indexing. When YaCy is shut down, this cache must be +# flushed to disc; this may last some minutes. +# maxWaitingWordFlush gives the number of seconds that the shutdown +# may last for the word flush +wordCacheMax = 10000 +maxWaitingWordFlush = 180