From 42d90664f3f00b1455d65225aef4731978c52d58 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 1 Mar 2011 09:03:33 +0000 Subject: [PATCH] - fixed a memory leak in the httpc.post method (no finish) - patched some more memory-saving relevant code - some more minor bug fixes git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7541 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/PerformanceMemory_p.java | 7 ++++++- source/de/anomic/http/client/Cache.java | 3 +++ source/de/anomic/search/SearchEvent.java | 5 +++-- source/de/anomic/search/SearchEventCache.java | 4 ++-- source/de/anomic/search/Switchboard.java | 4 ++-- source/de/anomic/yacy/dht/PeerSelection.java | 6 ++++++ source/de/anomic/yacy/yacySeed.java | 9 +++++++++ source/net/yacy/cora/protocol/http/HTTPConnector.java | 9 +++++++-- source/net/yacy/kelondro/blob/HeapReader.java | 6 +++++- 9 files changed, 43 insertions(+), 10 deletions(-) diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index bb3c87ac7..b9b5e05f9 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -26,6 +26,7 @@ //import java.util.Iterator; import java.io.File; +import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.Map; @@ -127,7 +128,11 @@ public class PerformanceMemory_p { RAMIndex cache; long hitmem, totalhitmem = 0; while (oi.hasNext()) { - oie = oi.next(); + try { + oie = oi.next(); + } catch (ConcurrentModificationException e) { + break; + } filename = oie.getKey(); cache = oie.getValue(); prop.put("indexcache_" + c + "_Name", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p)); diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index fbe554b18..4f1f887e4 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -237,6 +237,9 @@ public final class Cache { } catch (RowSpaceExceededException e) { Log.logException(e); return null; + } catch (OutOfMemoryError e) { + Log.logException(e); + return null; } } diff --git a/source/de/anomic/search/SearchEvent.java b/source/de/anomic/search/SearchEvent.java index 33f7c92bc..5639369f9 100644 --- a/source/de/anomic/search/SearchEvent.java +++ b/source/de/anomic/search/SearchEvent.java @@ -81,7 +81,7 @@ public final class SearchEvent { private byte[] IAmaxcounthash, IAneardhthash; private final ReferenceOrder order; - public SearchEvent(final QueryParams query, + protected SearchEvent(final QueryParams query, final yacySeedDB peers, final WorkTables workTables, final SortedMap preselectedPeerHashes, @@ -89,6 +89,7 @@ public final class SearchEvent { final LoaderDispatcher loader, final int burstRobinsonPercent, final int burstMultiwordPercent) { + if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true); this.eventTime = System.currentTimeMillis(); // for lifetime check this.peers = peers; this.workTables = workTables; @@ -202,7 +203,7 @@ public final class SearchEvent { EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), Type.CLEANUP, "", 0, 0), false); // store this search to a cache so it can be re-used - if (MemoryControl.available() < 1024 * 1024 * 10) SearchEventCache.cleanupEvents(true); + if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true); SearchEventCache.put(query.id(false), this); } diff --git a/source/de/anomic/search/SearchEventCache.java b/source/de/anomic/search/SearchEventCache.java index 194cb89dc..2abf1f5c6 100644 --- a/source/de/anomic/search/SearchEventCache.java +++ b/source/de/anomic/search/SearchEventCache.java @@ -46,8 +46,8 @@ public class SearchEventCache { public static final long eventLifetimeBigMem = 600000; // the time an event will stay in the cache when available memory is high, 10 Minutes public static final long eventLifetimeMediumMem = 60000; // the time an event will stay in the cache when available memory is medium, 1 Minute public static final long eventLifetimeShortMem = 10000; // the time an event will stay in the cache when memory is low, 10 seconds - public static final long memlimitHigh = 400 * 1024 * 1024; // 400 MB - public static final long memlimitMedium = 100 * 1024 * 1024; // 100 MB + public static final long memlimitHigh = 600 * 1024 * 1024; // 400 MB + public static final long memlimitMedium = 200 * 1024 * 1024; // 100 MB public static String lastEventID = ""; public static long cacheInsert = 0, cacheHit = 0, cacheMiss = 0, cacheDelete = 0; diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index d17661e51..ca1075d1b 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -1588,7 +1588,7 @@ public final class Switchboard extends serverSwitch { seed = e.next(); if (seed != null) { //list is sorted -> break when peers are too young to delete - if (seed.getLastSeenUTC() > (System.currentTimeMillis()-deleteOldSeedsTime)) break; + if (seed.isLastSeenTimeout(deleteOldSeedsTime)) break; deleteQueue.add(seed.hash); } } @@ -1601,7 +1601,7 @@ public final class Switchboard extends serverSwitch { seed = e.next(); if (seed != null) { //list is sorted -> break when peers are too young to delete - if (seed.getLastSeenUTC() > (System.currentTimeMillis() - deleteOldSeedsTime)) break; + if (seed.isLastSeenTimeout(deleteOldSeedsTime)) break; deleteQueue.add(seed.hash); } } diff --git a/source/de/anomic/yacy/dht/PeerSelection.java b/source/de/anomic/yacy/dht/PeerSelection.java index 4795049d1..d920767bf 100755 --- a/source/de/anomic/yacy/dht/PeerSelection.java +++ b/source/de/anomic/yacy/dht/PeerSelection.java @@ -102,6 +102,7 @@ public class PeerSelection { while (dhtEnum.hasNext() && c-- > 0) { seed = dhtEnum.next(); if (seed == null) continue; + if (seed.isLastSeenTimeout(3600000)) continue; if (seed.getAge() < 1) { // the 'workshop feature' Log.logInfo("DHT", "selectPeers/Age: " + seed.hash + ":" + seed.getName() + ", is newbie, age = " + seed.getAge()); regularSeeds.put(seed.hash, seed); @@ -120,6 +121,7 @@ public class PeerSelection { while (dhtEnum.hasNext()) { seed = dhtEnum.next(); if (seed == null) continue; + if (seed.isLastSeenTimeout(3600000)) continue; if (!seed.getFlagAcceptRemoteIndex()) robinson.add(seed); } @@ -128,6 +130,8 @@ public class PeerSelection { c = robinson.size() * burstRobinsonPercent / 100; while (dhtEnum.hasNext() && c-- > 0) { seed = dhtEnum.next(); + if (seed == null) continue; + if (seed.isLastSeenTimeout(3600000)) continue; if (Math.random() * 100 + burstRobinsonPercent >= 100) { if (Log.isFine("DHT")) Log.logFine("DHT", "selectPeers/RobinsonBurst: " + seed.hash + ":" + seed.getName()); regularSeeds.put(seed.hash, seed); @@ -140,6 +144,8 @@ public class PeerSelection { dhtEnum = robinson.iterator(); while (dhtEnum.hasNext()) { seed = dhtEnum.next(); + if (seed == null) continue; + if (seed.isLastSeenTimeout(3600000)) continue; if (seed.matchPeerTags(wordhashes)) { // peer tags match String specialized = seed.getPeerTags().toString(); diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 8deb3d32b..574db0f4f 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -518,6 +518,15 @@ public class yacySeed implements Cloneable, Comparable, Comparator milliseconds; + } /** @return the age of the seed in number of days */ public final int getAge() { diff --git a/source/net/yacy/cora/protocol/http/HTTPConnector.java b/source/net/yacy/cora/protocol/http/HTTPConnector.java index d8b32fa7f..fa9ae0108 100644 --- a/source/net/yacy/cora/protocol/http/HTTPConnector.java +++ b/source/net/yacy/cora/protocol/http/HTTPConnector.java @@ -79,8 +79,13 @@ public class HTTPConnector { client.setTimout(timeout); client.setUserAgent(this.userAgent); client.setHost(vhost); - - return client.POSTbytes(url.toNormalform(true, false, true, false), post, usegzip); + byte[] b; + try { + b = client.POSTbytes(url.toNormalform(true, false, true, false), post, usegzip); + } finally { + client.finish(); + } + return b; } } diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 3bd744a3c..7002edabd 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -672,7 +672,11 @@ public class HeapReader { public entries(final File blobFile, final int keylen) throws IOException { if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist"); - this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 8*1024*1024)); + try { + this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 8*1024*1024)); + } catch (OutOfMemoryError e) { + this.is = new DataInputStream(new FileInputStream(blobFile)); + } this.keylen = keylen; this.blobFile = blobFile; }