boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced crawling performance, changed memory settings, new performace options

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@51 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 10078bb354
commit 7fb645b0ab
12 changed files with 196 additions and 104 deletions
Show Stats Download Patch File Download Diff File

62
htroot/Performance_p.html
Unescape View File

@ -9,7 +9,7 @@
<br><br>
<h2>Performance</h2>
<p>
You can manipulate some performance attributes here.<br>
<div class=small><b>Scheduled tasks overview and waiting time settings:</b></div>
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
@ -47,10 +47,66 @@ You can manipulate some performance attributes here.<br>
<td class="small" align="left">#[longdescr]#</td>
</tr>
#{/table}#
</table><br>
<tr class="TableCellLight">
<td class="small" align="left" colspan="16">
<input type="submit" name="delaysubmit" value="Submit New Delay Values">
Changes take effect immidiately
Changes take effect immidiately</td>
</tr>
</table>
</p></form>
<p>
<div class=small><b>Indexing Cache Settings:</b></div>
<table border="0" cellpadding="5" cellspacing="0" width="100%">
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<tr valign="top" class="TableCellDark">
<td width="120"></td>
<td></td>
<td></td>
</tr>
<tr valign="top" class="TableCellDark">
<td class=small>Words in RAM Cache:</td>
<td class=small>#[wordCacheRAMSize]#</td>
<td class=small>
This is the current size of the word cache.
The smaller this number, the faster will be the shut-down procedure.
The maximum of this cache can be set below.
</td>
</tr>
<tr valign="top" class="TableCellDark">
<td class=small>Maximum URLs currently assigned<br>to one cached word:</td>
<td class=small>#[maxURLinWordCache]#</td>
<td class=small>
This is the maximum size of URLs assigned to a single word cache entry.
If this is a big number, it shows that the caching works efficiently.
</td>
</tr>
<tr valign="top" class="TableCellDark">
<td class=small>Maximum number of Word Caches:</td>
<td class=small><input name="wordCacheMax" type="text" size="20" maxlength="100" value="#[wordCacheMax]#"></td>
<td class=small>
This is is the number of word indexes that shall be held in the
ram cache during indexing. When YaCy is shut down, this cache must be
flushed to disc; this may last some minutes.
</td>
</tr>
<tr valign="top" class="TableCellDark">
<td class=small>Maximum waitingtime to flush word cache<br>during shut-down (seconds):</td>
<td class=small><input name="maxWaitingWordFlush" type="text" size="6" maxlength="6" value="#[maxWaitingWordFlush]#"></td>
<td class=small>
This gives the number of seconds that the shutdown
may last for the final word flush. Words that cannot be flushed within this
time are lost.
</td>
</tr>
<tr valign="top" class="TableCellLight">
<td class=small colspan="3"><input type="submit" name="cacheSizeSubmit" value="Enter New Cache Size">
Changes take effect immidiately</td>
</td>
</tr>
</form>
</table>
</p>
#[footer]#
</body>
</html>

23
htroot/Performance_p.java
Unescape View File

@ -109,11 +109,7 @@ public class Performance_p {
prop.put("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? "-" : ("" + (sleeptime / (idleCycles + busyCycles))));
prop.put("table_" + c + "_execpercycle", (busyCycles == 0) ? "-" : ("" + (exectime / busyCycles)));
if (post == null) {
// load with old values
idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000"));
} else {
if ((post != null) && (post.containsKey("delaysubmit"))) {
// load with new values
idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep", "1")) * 1000;
busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1")) * 1000;
@ -122,6 +118,10 @@ public class Performance_p {
switchboard.setThreadSleep(threadName, idlesleep, busysleep);
switchboard.setConfig(threadName + "_idlesleep", idlesleep);
switchboard.setConfig(threadName + "_busysleep", busysleep);
} else {
// load with old values
idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000"));
}
prop.put("table_" + c + "_idlesleep", idlesleep / 1000);
prop.put("table_" + c + "_busysleep", busysleep / 1000);
@ -130,6 +130,19 @@ public class Performance_p {
}
prop.put("table", c);
if ((post != null) && (post.containsKey("cacheSizeSubmit"))) {
int wordCacheMax = Integer.parseInt((String) post.get("wordCacheMax", "10000"));
switchboard.setConfig("wordCacheMax", "" + wordCacheMax);
switchboard.wordIndex.setMaxWords(wordCacheMax);
int maxWaitingWordFlush = Integer.parseInt((String) post.get("maxWaitingWordFlush", "180"));
switchboard.setConfig("maxWaitingWordFlush", "" + maxWaitingWordFlush);
}
// table cache settings
prop.put("wordCacheRAMSize", switchboard.wordIndex.wordCacheRAMSize());
prop.put("maxURLinWordCache", "" + switchboard.wordIndex.maxURLinWordCache());
prop.put("maxWaitingWordFlush", switchboard.getConfig("maxWaitingWordFlush", "180"));
prop.put("wordCacheMax", switchboard.getConfig("wordCacheMax", "10000"));
// return rewrite values for templates
return prop;
}

10
source/de/anomic/http/httpdSwitchboard.java
Unescape View File

@ -61,8 +61,14 @@ public final class httpdSwitchboard extends serverAbstractSwitch implements serv
cacheStack.addLast(job);
}
public void deQueue() {
System.out.println("Process: " + cacheStack.removeFirst().toString());
public boolean deQueue() {
if (cacheStack.size() > 0) {
System.out.println("Process: " + cacheStack.removeFirst().toString());
return true;
} else {
System.out.println("Process: queue is empty");
return false;
}
}
public serverObjects action(String actionName, serverObjects actionInput) {

31
source/de/anomic/kelondro/kelondroBufferedRA.java
Unescape View File

@ -79,16 +79,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
// delete elements in buffer if buffer too big
Iterator it = bufferScore.scores(true);
Integer element = (Integer) it.next();
writeBuffer((byte[]) bufferMemory.get(element), element.intValue());
bufferMemory.remove(element);
int age = bufferScore.deleteScore(element);
int minclean = bufferMaxElements / 8;
int cleaned = 1;
while ((cleaned++ < minclean) && (it.hasNext())) {
element = (Integer) it.next();
bufferMemory.remove(element);
age = bufferScore.deleteScore(element);
}
de.anomic.server.serverLog.logDebug("CACHE: " + name, "GC; cleaned=" + (cleaned - 1) + ", age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000));
de.anomic.server.serverLog.logDebug("CACHE: " + name, "GC; age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000));
}
// add new element
buffer = new byte[bufferElementSize];
@ -101,14 +95,6 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
return buffer;
}
/*
private static int log2i(int x) {
int log = 0;
while (x != 0) {x >>= 1; log++;}
return log;
}
*/
private void writeBuffer(byte[] buffer, int bufferNr) throws IOException {
if (buffer == null) return;
Integer bufferNrI = new Integer(bufferNr);
@ -132,7 +118,7 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
byte[] buffer = readBuffer(bn);
seekpos++;
buffer[offset] = (byte) b;
writeBuffer(buffer, bn);
//writeBuffer(buffer, bn);
}
public int read(byte[] b, int off, int len) throws IOException {
@ -165,13 +151,13 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
// simple case
System.arraycopy(b, off, buffer, offset, len);
seekpos += len;
writeBuffer(buffer, bn1);
//writeBuffer(buffer, bn1);
} else {
// do recursively
int thislen = bufferElementSize - offset;
System.arraycopy(b, off, buffer, offset, thislen);
seekpos += thislen;
writeBuffer(buffer, bn1);
//writeBuffer(buffer, bn1);
write(b, thislen, len - thislen);
}
}
@ -181,6 +167,13 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
}
public void close() throws IOException {
// write all unwritten buffers
Iterator it = bufferScore.scores(true);
while (it.hasNext()) {
Integer element = (Integer) it.next();
writeBuffer((byte[]) bufferMemory.get(element), element.intValue());
bufferMemory.remove(element);
}
ra.close();
bufferScore = null;
bufferMemory = null;

28
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -142,6 +142,7 @@ public class kelondroRecords {
throw new IOException("kelondroRecords: tree file " + file + " already exist");
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
this.cachesize = (int) (buffersize / ((long) (overhead + recordsize)));
@ -240,6 +241,7 @@ public class kelondroRecords {
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
init(raf);
this.cachesize = (int) (buffersize / ((long) (overhead + recordsize)));
@ -353,32 +355,6 @@ public class kelondroRecords {
// we simply clear the cache
this.cacheScore = new kelondroMScoreCluster();
this.cache = new HashMap();
/*
java.util.NoSuchElementException
at java.util.TreeMap.key(TreeMap.java:431)
at java.util.TreeMap.firstKey(TreeMap.java:286)
at de.anomic.kelondro.kelondroMScoreCluster$scoreIterator.internalNext(kelondroMScoreCluster.java:235)
at de.anomic.kelondro.kelondroMScoreCluster$scoreIterator.<init>(kelondroMScoreCluster.java:224)
at de.anomic.kelondro.kelondroMScoreCluster.scores(kelondroMScoreCluster.java:209)
at de.anomic.kelondro.kelondroMScoreCluster.getScores(kelondroMScoreCluster.java:191)
at de.anomic.kelondro.kelondroMScoreCluster.getScores(kelondroMScoreCluster.java:185)
at de.anomic.kelondro.kelondroMScoreCluster.getMinObject(kelondroMScoreCluster.java:181)
at de.anomic.kelondro.kelondroRecords.checkCacheSpace(kelondroRecords.java:344)
at de.anomic.kelondro.kelondroRecords.access$1500(kelondroRecords.java:74)
at de.anomic.kelondro.kelondroRecords$Node.updateNode(kelondroRecords.java:624)
at de.anomic.kelondro.kelondroRecords$Node.<init>(kelondroRecords.java:399)
at de.anomic.kelondro.kelondroRecords$Node.<init>(kelondroRecords.java:350)
at de.anomic.kelondro.kelondroRecords.getNode(kelondroRecords.java:321)
at de.anomic.kelondro.kelondroTree$Search.searchproc(kelondroTree.java:265)
at de.anomic.kelondro.kelondroTree$Search.<init>(kelondroTree.java:246)
at de.anomic.kelondro.kelondroTree.get(kelondroTree.java:317)
at de.anomic.plasma.plasmaURL.exists(plasmaURL.java:106)
at de.anomic.plasma.plasmaSwitchboard.stackCrawl(plasmaSwitchboard.java:389)
at de.anomic.plasma.plasmaSwitchboard.processResourceStack(plasmaSwitchboard.java:304)
at de.anomic.plasma.plasmaSwitchboard.deQueue(plasmaSwitchboard.java:262)
at yacyProxy.main(yacyProxy.java:228)
*/
}
}
}

2
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -257,7 +257,7 @@ public final class plasmaHTCache {
entry.cacheFile.getParentFile().mkdirs();
log.logInfo("WRITE FILE (" + entry.cacheArray.length + " bytes) " + entry.cacheFile);
serverFileUtils.write(entry.cacheArray, entry.cacheFile);
log.logInfo("AFTER WRITE cacheArray = " + entry.cacheFile + ": " + ((entry.cacheArray == null) ? "empty" : "full"));
log.logDebug("AFTER WRITE cacheArray = " + entry.cacheFile + ": " + ((entry.cacheArray == null) ? "empty" : "full"));
//entry.cacheArray = null;
} catch (FileNotFoundException e) {
// this is the case of a "(Not a directory)" error, which should be prohibited

50
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -190,19 +190,20 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
}
// read memory amount
int ramCacheKB = Integer.parseInt(getConfig("ramCacheSize", "1")) * 0x400;
int ramLURL = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentLURL", "1")) / 100;
int ramPURL = ramLURL / 2;
int ramEURL = ramLURL / 2;
int ramRWI = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentRWI", "1")) / 100;
int ramHTTP = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentHTTP", "1")) / 100;
int ramMessage = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentMessage", "1")) / 100;
int ramWiki = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentWiki", "1")) / 100;
log.logSystem("LURL Cache memory = " + ramLURL + " KB");
log.logSystem("RWI Cache memory = " + ramRWI + " KB");
log.logSystem("HTTP Cache memory = " + ramHTTP + " KB");
log.logSystem("Message Cache memory = " + ramMessage + " KB");
log.logSystem("Wiki Cache memory = " + ramWiki + " KB");
int ramLURL = Integer.parseInt(getConfig("ramCacheLURL", "1024")) / 1024;
int ramNURL = Integer.parseInt(getConfig("ramCacheNURL", "1024")) / 1024;
int ramEURL = Integer.parseInt(getConfig("ramCacheEURL", "1024")) / 1024;
int ramRWI = Integer.parseInt(getConfig("ramCacheRWI", "1024")) / 1024;
int ramHTTP = Integer.parseInt(getConfig("ramCacheHTTP", "1024")) / 1024;
int ramMessage = Integer.parseInt(getConfig("ramCacheMessage", "1024")) / 1024;
int ramWiki = Integer.parseInt(getConfig("ramCacheWiki", "1024")) / 1024;
log.logSystem("LURL Cache memory = " + ppRamString(ramLURL));
log.logSystem("NURL Cache memory = " + ppRamString(ramNURL));
log.logSystem("EURL Cache memory = " + ppRamString(ramEURL));
log.logSystem("RWI Cache memory = " + ppRamString(ramRWI));
log.logSystem("HTTP Cache memory = " + ppRamString(ramHTTP));
log.logSystem("Message Cache memory = " + ppRamString(ramMessage));
log.logSystem("Wiki Cache memory = " + ppRamString(ramWiki));
// make crawl profiles database and default profiles
profiles = new plasmaCrawlProfile(new File(plasmaPath, "crawlProfiles0.db"));
@ -213,10 +214,10 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
// start indexing management
loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL);
noticeURL = new plasmaCrawlNURL(plasmaPath, ramPURL);
noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL);
errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL);
//indexCache = new plasmaWordIndexRAMCache(plasmaPath, 2000, ramRWI);
wordIndex = new plasmaWordIndex(plasmaPath, ramRWI);
wordIndex = new plasmaWordIndex(plasmaPath, ramRWI);
wordIndex.setMaxWords(10000);
searchManager = new plasmaSearch(loadedURL, wordIndex);
// start a cache manager
@ -290,6 +291,15 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
new serverInstantThread(indexDistribution, "job", null), log, 120000);
}
private static String ppRamString(int bytes) {
if (bytes < 1024) return bytes + " KByte";
bytes = bytes / 1024;
if (bytes < 1024) return bytes + " MByte";
bytes = bytes / 1024;
if (bytes < 1024) return bytes + " GByte";
return (bytes / 1024) + "TByte";
}
public void handleBusyState(int jobs) {
this.serverJobs = jobs;
}
@ -359,7 +369,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
log.logSystem("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:");
terminateAllThreads(true);
log.logSystem("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing (stand by..)");
int waitingBoundSeconds = Integer.parseInt(getConfig("shutdownWaiting", "120"));
int waitingBoundSeconds = Integer.parseInt(getConfig("maxWaitingWordFlush", "120"));
wordIndex.close(waitingBoundSeconds);
log.logSystem("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager");
try {
@ -397,16 +407,18 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
processStack.addLast(entry);
}
public synchronized void deQueue() {
public synchronized boolean deQueue() {
if (serverJobs < 6) {
if (processStack.size() > 0) {
processResourceStack((plasmaHTCache.Entry) processStack.removeFirst());
return true;
}
} else {
//if (processStack.size() > 0) {
log.logDebug("DEQUEUE: serverJobs=" + serverJobs + " 'busy' - no dequeueing (processStack=" + processStack.size() + ", localStackSize=" + noticeURL.localStackSize() + ", remoteStackSize=" + noticeURL.remoteStackSize() + ")");
//}
}
return false;
}
public int cleanupJobSize() {
@ -444,7 +456,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
}
public boolean localCrawlJob() {
if ((serverJobs < 2) &&
if ((serverJobs < 6) &&
(processStack.size() < crawlSlots) &&
(noticeURL.localStackSize() > 0) &&
(cacheLoader.size() < crawlSlots)) {

14
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -57,10 +57,22 @@ public class plasmaWordIndex {
public plasmaWordIndex(File databaseRoot, int bufferkb) throws IOException {
this.databaseRoot = databaseRoot;
this.ramCache = new plasmaWordIndexRAMCache(databaseRoot, 2000, bufferkb);
this.ramCache = new plasmaWordIndexRAMCache(databaseRoot, bufferkb);
ramCache.start();
}
public int maxURLinWordCache() {
return ramCache.maxURLinWordCache();
}
public int wordCacheRAMSize() {
return ramCache.wordCacheRAMSize();
}
public void setMaxWords(int maxWords) {
ramCache.setMaxWords(maxWords);
}
public synchronized int addEntry(String wordHash, plasmaWordIndexEntry entry) throws IOException {
return ramCache.addEntryToIndexMem(wordHash, entry);
}

24
source/de/anomic/plasma/plasmaWordIndexRAMCache.java
Unescape View File

@ -70,14 +70,26 @@ public class plasmaWordIndexRAMCache extends Thread {
for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += '-';
}
public plasmaWordIndexRAMCache(File databaseRoot, int maxWords, int bufferkb) throws IOException {
public plasmaWordIndexRAMCache(File databaseRoot, int bufferkb) throws IOException {
this.pic = new plasmaWordIndexFileCache(databaseRoot, bufferkb);
this.cache = new TreeMap();
this.hashScore = new kelondroMScoreCluster();
this.maxWords = maxWords;
this.maxWords = 1000;
this.terminate = false;
}
public int maxURLinWordCache() {
return hashScore.getScore(hashScore.getMaxObject());
}
public int wordCacheRAMSize() {
return cache.size();
}
public void setMaxWords(int maxWords) {
this.maxWords = maxWords;
}
public void run() {
serverLog.logSystem("PLASMA INDEXING", "started word cache management");
int check;
@ -86,9 +98,9 @@ public class plasmaWordIndexRAMCache extends Thread {
if (hashScore.size() < 100) try {Thread.currentThread().sleep(10000);} catch (InterruptedException e) {}
while ((!(terminate)) && (cache != null) && (hashScore.size() > 0)) try {
//check = hashScore.size();
flushSpecific(true);
flushSpecific(false);
//serverLog.logDebug("PLASMA INDEXING", "single flush. bevore=" + check + "; after=" + hashScore.size());
try {Thread.currentThread().sleep(200 + (maxWords / (1 + hashScore.size())));} catch (InterruptedException e) {}
try {Thread.currentThread().sleep(10 + ((maxWords / 10) / (1 + hashScore.size())));} catch (InterruptedException e) {}
} catch (IOException e) {
serverLog.logError("PLASMA INDEXING", "PANIK! exception in main cache loop: " + e.getMessage());
e.printStackTrace();
@ -158,7 +170,7 @@ public class plasmaWordIndexRAMCache extends Thread {
String key = (String) ((greatest) ? hashScore.getMaxObject() : hashScore.getMinObject());
return flushKey(key, "flushSpecific");
}
private synchronized int flushKey(String key, String caller) throws IOException {
Vector v = (Vector) cache.get(key);
if (v == null) {
@ -197,7 +209,7 @@ public class plasmaWordIndexRAMCache extends Thread {
// make space for new words
int flushc = 0;
//serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem: cache.size=" + cache.size() + "; hashScore.size=" + hashScore.size());
while (hashScore.size() > maxWords) flushc += flushSpecific(false);
while (hashScore.size() > maxWords) flushc += flushSpecific(true);
//if (flushc > 0) serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem - flushed " + flushc + " entries");
// put new words into cache

2
source/de/anomic/server/serverAbstractSwitch.java
Unescape View File

@ -225,7 +225,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
abstract public int queueSize();
abstract public void enQueue(Object job);
abstract public void deQueue();
abstract public boolean deQueue();
// authentification routines:

2
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -74,7 +74,7 @@ public interface serverSwitch {
// work off exactly only one job, not all
public int queueSize();
public void enQueue(Object job);
public void deQueue();
public boolean deQueue(); // returns true if there had been dequeued anything
// authentification routines: sets and reads access attributes according to host addresses
public void setAuthentify(InetAddress host, String user, String rigth);

52
yacy.init
Unescape View File

@ -12,13 +12,6 @@
# port number of server
port = 8080
# shutdown time limit
# this is the time that a peer takes at most for shutdown
# the shutdown-procedure is difficult since there are many
# caches that must be flushed first
# measured in seconds
shutdownWaiting = 120
# time-out of client control socket in milliseconds
# since this applies only to the client-proxy connection,
# it can be rather short
@ -94,19 +87,6 @@ parseableMime=application/xhtml+xml,text/html,text/plain
# These files will be excluded from indexing
mediaExt=swf,wmv,jpg,jpeg,jpe,rm,mov,mpg,mpeg,mp3,asf,gif,png,avi,zip,rar,sit,hqx,img,dmg,tar,gz,ps,pdf,doc,xls,ppt,ram,bz2,arj,jar,deb,torrent,ogg,iso,bin,ace,tgz,rpm,css
# the proxy's and indexing maximum ram cache size in megabytes
ramCacheSize = 12
# ram cache is partitioned into different separate caches
# we define percentual shares of the complete ram cache
# values here are computed from usage profiles and should not be changed
# it is not computed if the complete sum of percentages computes to 100
ramCachePercentRWI = 85
ramCachePercentHTTP = 6
ramCachePercentLURL = 4
ramCachePercentDHT = 3
ramCachePercentMessage = 1
ramCachePercentWiki = 1
# Promotion Strings
# These strings appear in the Web Mask of the YACY search client
# Set these Strings to cusomize your peer and give any message to
@ -401,3 +381,35 @@ xpstopw=true
80_dequeue_busysleep=1000
90_cleanup_idlesleep=300000
90_cleanup_busysleep=300000
# ram cache for database files
# ram cache for indexCache.db
ramCacheRWI = 16777216
# ram cache for responseHeader.db
ramCacheHTTP = 1048576
# ram cache for urlHash.db
ramCacheLURL = 8388608
# ram cache for urlNotice.db
ramCacheNURL = 1048576
# ram cache for urlErr.db
ramCacheEURL = 131072
# ram cache for message.db
ramCacheMessage = 8192
# ram cache for wiki.db
ramCacheWiki = 8192
# performance properties for the word index cache
# wordCacheMax is the number of word indexes that shall be held in the
# ram cache during indexing. When YaCy is shut down, this cache must be
# flushed to disc; this may last some minutes.
# maxWaitingWordFlush gives the number of seconds that the shutdown
# may last for the word flush
wordCacheMax = 10000
maxWaitingWordFlush = 180

Write Preview
Loading…
Cancel
Save