- fixed a memory leak in the httpc.post method (no finish)

- patched some more memory-saving relevant code
- some more minor bug fixes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7541 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/PerformanceMemory_p.java

@@ -26,6 +26,7 @@
 
 //import java.util.Iterator;
 import java.io.File;
+import java.util.ConcurrentModificationException;
 import java.util.Iterator;
 import java.util.Map;
 
@@ -127,7 +128,11 @@ public class PerformanceMemory_p {
         RAMIndex cache;
         long hitmem, totalhitmem = 0;
         while (oi.hasNext()) {
-            oie = oi.next();
+            try {
+                oie = oi.next();
+            } catch (ConcurrentModificationException e) {
+                break;
+            }
             filename = oie.getKey();
             cache = oie.getValue();
             prop.put("indexcache_" + c + "_Name", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p));

source/de/anomic/http/client/Cache.java

@@ -237,6 +237,9 @@ public final class Cache {
         } catch (RowSpaceExceededException e) {
             Log.logException(e);
             return null;
+        } catch (OutOfMemoryError e) {
+            Log.logException(e);
+            return null;
         }
     }
     

source/de/anomic/search/SearchEvent.java

@@ -81,7 +81,7 @@ public final class SearchEvent {
     private byte[] IAmaxcounthash, IAneardhthash;
     private final ReferenceOrder order;
     
-    public SearchEvent(final QueryParams query,
+    protected SearchEvent(final QueryParams query,
                              final yacySeedDB peers,
                              final WorkTables workTables,
                              final SortedMap<byte[], String> preselectedPeerHashes,
@@ -89,6 +89,7 @@ public final class SearchEvent {
                              final LoaderDispatcher loader,
                              final int burstRobinsonPercent,
                              final int burstMultiwordPercent) {
+        if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true);
         this.eventTime = System.currentTimeMillis(); // for lifetime check
         this.peers = peers;
         this.workTables = workTables;
@@ -202,7 +203,7 @@ public final class SearchEvent {
         EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), Type.CLEANUP, "", 0, 0), false);
         
         // store this search to a cache so it can be re-used
-        if (MemoryControl.available() < 1024 * 1024 * 10) SearchEventCache.cleanupEvents(true);
+        if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true);
         SearchEventCache.put(query.id(false), this);
    }
    

source/de/anomic/search/SearchEventCache.java

@@ -46,8 +46,8 @@ public class SearchEventCache {
     public static final long eventLifetimeBigMem = 600000; // the time an event will stay in the cache when available memory is high, 10 Minutes
     public static final long eventLifetimeMediumMem = 60000; // the time an event will stay in the cache when available memory is medium, 1 Minute
     public static final long eventLifetimeShortMem = 10000; // the time an event will stay in the cache when memory is low, 10 seconds
-    public static final long memlimitHigh = 400 * 1024 * 1024; // 400 MB
-    public static final long memlimitMedium = 100 * 1024 * 1024; // 100 MB
+    public static final long memlimitHigh = 600 * 1024 * 1024; // 400 MB
+    public static final long memlimitMedium = 200 * 1024 * 1024; // 100 MB
     public static String lastEventID = "";
     public static long cacheInsert = 0, cacheHit = 0, cacheMiss = 0, cacheDelete = 0;
     

source/de/anomic/search/Switchboard.java

@@ -1588,7 +1588,7 @@ public final class Switchboard extends serverSwitch {
                     seed = e.next();
                     if (seed != null) {
                         //list is sorted -> break when peers are too young to delete
-                        if (seed.getLastSeenUTC() > (System.currentTimeMillis()-deleteOldSeedsTime)) break;
+                        if (seed.isLastSeenTimeout(deleteOldSeedsTime)) break;
                         deleteQueue.add(seed.hash);
                     }
                 }
@@ -1601,7 +1601,7 @@ public final class Switchboard extends serverSwitch {
                     seed = e.next();
                     if (seed != null) {
                         //list is sorted -> break when peers are too young to delete
-                        if (seed.getLastSeenUTC() > (System.currentTimeMillis() - deleteOldSeedsTime)) break;
+                        if (seed.isLastSeenTimeout(deleteOldSeedsTime)) break;
                         deleteQueue.add(seed.hash);
                     }
                 }

source/de/anomic/yacy/dht/PeerSelection.java

@@ -102,6 +102,7 @@ public class PeerSelection {
         while (dhtEnum.hasNext() && c-- > 0) {
             seed = dhtEnum.next();
             if (seed == null) continue;
+            if (seed.isLastSeenTimeout(3600000)) continue;
             if (seed.getAge() < 1) { // the 'workshop feature'
                 Log.logInfo("DHT", "selectPeers/Age: " + seed.hash + ":" + seed.getName() + ", is newbie, age = " + seed.getAge());
                 regularSeeds.put(seed.hash, seed);
@@ -120,6 +121,7 @@ public class PeerSelection {
         while (dhtEnum.hasNext()) {
             seed = dhtEnum.next();
             if (seed == null) continue;
+            if (seed.isLastSeenTimeout(3600000)) continue;
             if (!seed.getFlagAcceptRemoteIndex()) robinson.add(seed);
         }
 
@@ -128,6 +130,8 @@ public class PeerSelection {
         c = robinson.size() * burstRobinsonPercent / 100;
         while (dhtEnum.hasNext() && c-- > 0) {
             seed = dhtEnum.next();
+            if (seed == null) continue;
+            if (seed.isLastSeenTimeout(3600000)) continue;
             if (Math.random() * 100 + burstRobinsonPercent >= 100) {
                 if (Log.isFine("DHT")) Log.logFine("DHT", "selectPeers/RobinsonBurst: " + seed.hash + ":" + seed.getName());
                 regularSeeds.put(seed.hash, seed);
@@ -140,6 +144,8 @@ public class PeerSelection {
         dhtEnum = robinson.iterator();
         while (dhtEnum.hasNext()) {
             seed = dhtEnum.next();
+            if (seed == null) continue;
+            if (seed.isLastSeenTimeout(3600000)) continue;
             if (seed.matchPeerTags(wordhashes)) {
                 // peer tags match
                 String specialized = seed.getPeerTags().toString();

source/de/anomic/yacy/yacySeed.java

@@ -518,6 +518,15 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
             return System.currentTimeMillis() - AbstractFormatter.dayMillis;
         }
     }
+    
+    /**
+     * test if the lastSeen time of the seed has a time-out
+     * @param milliseconds the maximum age of the last-seen value
+     * @return true, if the time between the last-seen time and now is greater then the given time-out
+     */
+    public final boolean isLastSeenTimeout(long milliseconds) {
+        return Math.abs(System.currentTimeMillis() - this.getLastSeenUTC()) > milliseconds;
+    }
 
     /** @return the age of the seed in number of days */
     public final int getAge() {

source/net/yacy/cora/protocol/http/HTTPConnector.java

@@ -79,8 +79,13 @@ public class HTTPConnector {
 		client.setTimout(timeout);
 		client.setUserAgent(this.userAgent);
 		client.setHost(vhost);
-		
-		return client.POSTbytes(url.toNormalform(true, false, true, false), post, usegzip);
+		byte[] b;
+		try {
+		    b = client.POSTbytes(url.toNormalform(true, false, true, false), post, usegzip);
+		} finally {
+		    client.finish();
+		}
+		return b;
 	}
 
 }

source/net/yacy/kelondro/blob/HeapReader.java

@@ -672,7 +672,11 @@ public class HeapReader {
         
         public entries(final File blobFile, final int keylen) throws IOException {
             if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist");
-            this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 8*1024*1024));
+            try {
+                this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 8*1024*1024));
+            } catch (OutOfMemoryError e) {
+                this.is = new DataInputStream(new FileInputStream(blobFile));
+            }
             this.keylen = keylen;
             this.blobFile = blobFile;
         }