From c998dc6556c5d7b0089b4b2aa2898d8f90cbb580 Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 16 Jun 2008 21:39:58 +0000 Subject: [PATCH] - added security functions to flush url and search caches in case that memory is full git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4933 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacysearch.java | 6 +++ .../index/indexRepositoryReference.java | 6 ++- source/de/anomic/kelondro/kelondroCache.java | 38 +++++++++++++++---- .../de/anomic/plasma/plasmaSwitchboard.java | 9 +++++ source/de/anomic/plasma/plasmaWordIndex.java | 6 ++- 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 7063160a8..605df6212 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -63,6 +63,7 @@ import de.anomic.plasma.plasmaSearchRankingProfile; import de.anomic.plasma.plasmaSnippetCache; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; +import de.anomic.server.serverMemory; import de.anomic.server.serverObjects; import de.anomic.server.serverProfiling; import de.anomic.server.serverSwitch; @@ -197,6 +198,11 @@ public class yacysearch { } if ((!block) && (post == null || post.get("cat", "href").equals("href"))) { + // check available memory and clean up if necessary + if (!serverMemory.request(8000000L, false)) { + sb.webIndex.clearCache(); + plasmaSearchEvent.cleanupEvents(true); + } plasmaSearchRankingProfile ranking = sb.getRanking(); final TreeSet[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute diff --git a/source/de/anomic/index/indexRepositoryReference.java b/source/de/anomic/index/indexRepositoryReference.java index 8445f1063..3d74aa340 100644 --- a/source/de/anomic/index/indexRepositoryReference.java +++ b/source/de/anomic/index/indexRepositoryReference.java @@ -61,9 +61,13 @@ public final class indexRepositoryReference { public indexRepositoryReference(File indexSecondaryPath) { super(); this.location = new File(indexSecondaryPath, "TEXT"); - urlIndexFile = new kelondroSplitTable(this.location, "urls", indexURLReference.rowdef, false); + urlIndexFile = new kelondroCache(new kelondroSplitTable(this.location, "urls", indexURLReference.rowdef, false)); } + public void clearCache() { + if (urlIndexFile instanceof kelondroCache) ((kelondroCache) urlIndexFile).clearCache(); + } + public void clear() throws IOException { if (exportthread != null) exportthread.interrupt(); urlIndexFile.clear(); diff --git a/source/de/anomic/kelondro/kelondroCache.java b/source/de/anomic/kelondro/kelondroCache.java index 16fee3be0..ca8a51a20 100644 --- a/source/de/anomic/kelondro/kelondroCache.java +++ b/source/de/anomic/kelondro/kelondroCache.java @@ -51,8 +51,8 @@ public class kelondroCache implements kelondroIndex { // static object tracker; stores information about object cache usage private static final TreeMap objectTracker = new TreeMap(); - private static long memStopGrow = 10000000; // a limit for the node cache to stop growing if less than this memory amount is available - private static long memStartShrink = 6000000; // a limit for the node cache to start with shrinking if less than this memory amount is available + private static long memStopGrow = 12 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available + private static long memStartShrink = 8 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available // class objects private kelondroRowSet readHitCache; @@ -180,6 +180,11 @@ public class kelondroCache implements kelondroIndex { return true; } + public synchronized void clearCache() { + readMissCache.clear(); + readHitCache.clear(); + } + public synchronized void close() { index.close(); readHitCache = null; @@ -187,7 +192,27 @@ public class kelondroCache implements kelondroIndex { } public boolean has(byte[] key) throws IOException { - return (get(key) != null); + // first look into the miss cache + if (readMissCache != null) { + if (readMissCache.get(key) != null) { + this.hasnotHit++; + return false; + } else { + this.hasnotMiss++; + } + } + + // then try the hit cache and the buffers + if (readHitCache != null) { + if (readHitCache.get(key) != null) { + this.readHit++; + return true; + } + } + + // finally ask the back-end index + this.readMiss++; + return index.has(key); } public synchronized Entry get(byte[] key) throws IOException { @@ -212,7 +237,7 @@ public class kelondroCache implements kelondroIndex { } } - // finally ask the backend index + // finally ask the back-end index this.readMiss++; entry = index.get(key); // learn from result @@ -289,9 +314,8 @@ public class kelondroCache implements kelondroIndex { public synchronized Entry put(Entry row, Date entryDate) throws IOException { // a put with a date is bad for the cache: the date cannot be handled - // The write buffer does not work here, because it does not store dates. - - throw new UnsupportedOperationException("put with date is inefficient in kelondroCache"); + // we omit the date here and use the current Date everywhere + return this.put(row); } public synchronized boolean addUnique(Entry row) throws IOException { diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 9a95e39f2..d5d0b0303 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -1661,6 +1661,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch