word cache behaviour fix and other fixes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@519 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/httpdProxyHandler.java

@@ -369,7 +369,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
                     switchboard.defaultProxyProfile  // profile
             );
             
-            if (cacheExists && cacheEntry.shallUseCache()) {
+            if (cacheExists && cacheEntry.shallUseCacheForProxy()) {
                 fulfillRequestFromCache(conProp,url,ext,requestHeader,cachedResponseHeader,cacheFile,respond);
             } else {            
                 fulfillRequestFromWeb(conProp,url,ext,requestHeader,cachedResponseHeader,cacheFile,respond);
@@ -527,7 +527,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
                             res.responseHeader);
             
             String storeError;
-            if ((storeError = cacheEntry.shallStoreCache()) == null) {
+            if ((storeError = cacheEntry.shallStoreCacheForProxy()) == null) {
                 // we write a new cache entry
                 if ((contentLength > 0) && (contentLength < 1048576)) // if the length is known and < 1 MB
                 {

source/de/anomic/plasma/plasmaHTCache.java

@@ -529,7 +529,7 @@ public final class plasmaHTCache {
 	// the following three methods for cache read/write granting shall be as loose as possible
 	// but also as strict as necessary to enable caching of most items
 	
-	public String shallStoreCache() {
+	public String shallStoreCacheForProxy() {
             // returns NULL if the answer is TRUE
             // in case of FALSE, the reason as String is returned
 	    
@@ -612,7 +612,7 @@ public final class plasmaHTCache {
 	    return null;
 	}
 
-        public boolean shallUseCache() {
+        public boolean shallUseCacheForProxy() {
 	    // decide upon header information if a specific file should be taken from the cache or not
 	    
 	    //System.out.println("SHALL READ CACHE: requestHeader = " + requestHeader.toString() + ", responseHeader = " + responseHeader.toString());

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -495,7 +495,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         
         // work off unwritten files
         if (entry.cacheArray != null)  {
-            String error = entry.shallStoreCache();
+            String error = entry.shallStoreCacheForProxy();
             if (error == null) {
             cacheManager.writeFile(entry.url, entry.cacheArray);
                 log.logInfo("WROTE FILE (" + entry.cacheArray.length + " bytes) for " + entry.cacheFile);

source/de/anomic/plasma/plasmaWordIndexCache.java

@@ -352,6 +352,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
         public void run() {
             String nextHash;
             Runtime rt = Runtime.getRuntime();
+            long pausetime;
             while (!terminate) {
 		if (intermission > 0) {
 		    if (this.intermission > System.currentTimeMillis()) {
@@ -363,8 +364,10 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
                     try {this.sleep(300);} catch (InterruptedException e) {}
                 } else {
                     flushFromMem();
-                    if ((rt.freeMemory() > 1000000) || (cache.size() == 0)) try {
-                        this.sleep(10 + java.lang.Math.min(1000, 10 * maxWords/(cache.size() + 1)));
+                    try {
+                        pausetime = 1 + java.lang.Math.min(1000, 5 * maxWords/(cache.size() + 1));
+                        if (cache.size() == 0) pausetime = 2000;
+                        this.sleep(pausetime);
                     } catch (InterruptedException e) {}
                 }              
             }
@@ -510,13 +513,21 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
     
     public synchronized int addEntries(plasmaWordIndexEntryContainer container, long updateTime) {
         // this puts the entries into the cache, not into the assortment directly
+
+        // check cache space
+        if (cache.size() > 0) try {
+            // pause until space is in the cache
+            while (cache.size() >= this.maxWords) Thread.sleep(1000);
+            
+            // slow down if we reach cache limit
+            long pausetime = java.lang.Math.min(10, 3 * cache.size() / (maxWords + 1));
+            //System.out.println("Pausetime=" + pausetime);
+            Thread.sleep(pausetime);
+        } catch (InterruptedException e) {}
+        
+        // stop flushing now for one moment
         flushThread.pause();
 	//serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem: cache.size=" + cache.size() + "; hashScore.size=" + hashScore.size());
-        while (cache.size() >= this.maxWords) flushFromMem();
-        if ((cache.size() > 10000) && (Runtime.getRuntime().freeMemory() < 5000000)) flushFromMem();
-        if ((cache.size() > 0) && (Runtime.getRuntime().freeMemory() < 1000000)) flushFromMem();
-        
-	//if (flushc > 0) serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem - flushed " + flushc + " entries");
         
 	// put new words into cache
         int added = 0;

yacy.init

@@ -404,7 +404,7 @@ xpstopw=true
 # be available/free in the heap; othervise the loop is not executed
 # and another idlesleep is performed
 20_dhtdistribution_idlesleep=20000
-20_dhtdistribution_busysleep=1000
+20_dhtdistribution_busysleep=10000
 20_dhtdistribution_memprereq=1000000
 30_peerping_idlesleep=120000
 30_peerping_busysleep=120000