diff --git a/source/de/anomic/index/indexWord.java b/source/de/anomic/index/indexWord.java index 4eb5d9940..1070e180a 100644 --- a/source/de/anomic/index/indexWord.java +++ b/source/de/anomic/index/indexWord.java @@ -26,8 +26,8 @@ package de.anomic.index; +import java.util.HashMap; import java.util.HashSet; -import java.util.Hashtable; import java.util.Iterator; import java.util.Locale; import java.util.Set; @@ -40,13 +40,14 @@ import de.anomic.yacy.yacySeedDB; public class indexWord { - // object carries statistics for words and sentences + private static final HashMap hashCache = new HashMap(1000); - public int count; // number of occurrences - public int posInText; // unique handle, is initialized with word position (excluding double occurring words) - public int posInPhrase; // position of word in phrase - public int numOfPhrase; // number of phrase. 'normal' phrases begin with number 100 - HashSet phrases; // a set of handles to all phrases where this word appears + // object carries statistics for words and sentences + public int count; // number of occurrences + public int posInText; // unique handle, is initialized with word position (excluding double occurring words) + public int posInPhrase; // position of word in phrase + public int numOfPhrase; // number of phrase. 'normal' phrases begin with number 100 + HashSet phrases; // a set of handles to all phrases where this word appears public Bitfield flags; // the flag bits for each word public indexWord(final int handle, final int pip, final int nop) { @@ -78,13 +79,15 @@ public class indexWord { // static methods // create a word hash - private static final Hashtable hashCache = new Hashtable(); public static final String word2hash(final String word) { String h = hashCache.get(word); if (h != null) return h; h = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(word.toLowerCase(Locale.ENGLISH))).substring(0, yacySeedDB.commonHashLength); hashCache.put(word, h); // prevent expensive MD5 computation and encoding - if (hashCache.size() > 100000) hashCache.clear(); // prevent memory laeak + if (hashCache.size() > 20000) { + // prevent memory leak + hashCache.clear(); + } return h; } diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index fc24fb50b..94c33baca 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -42,6 +42,7 @@ import de.anomic.index.indexRWIEntry; import de.anomic.index.indexRWIVarEntry; import de.anomic.index.indexURLReference; import de.anomic.kelondro.order.Bitfield; +import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.SetTools; import de.anomic.kelondro.util.SortStack; import de.anomic.kelondro.util.SortStore; @@ -66,7 +67,7 @@ public final class plasmaSearchEvent { public static int workerThreadCount = 10; public static String lastEventID = ""; private static ConcurrentHashMap lastEvents = new ConcurrentHashMap(); // a cache for objects from this class: re-use old search requests - public static final long eventLifetime = 600000; // the time an event will stay in the cache, 10 Minutes + public static final long eventLifetime = 60000; // the time an event will stay in the cache, 1 Minute private static final int max_results_preparation = 300; private long eventTime; @@ -215,6 +216,7 @@ public final class plasmaSearchEvent { serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "event-cleanup", 0, 0)); // store this search to a cache so it can be re-used + if (MemoryControl.available() < 1024 * 1024 * 10) cleanupEvents(true); lastEventID = query.id(false); lastEvents.put(lastEventID, this); } diff --git a/source/de/anomic/yacy/logging/ConsoleOutErrHandler.java b/source/de/anomic/yacy/logging/ConsoleOutErrHandler.java index fe7bbe75f..c0edfe160 100644 --- a/source/de/anomic/yacy/logging/ConsoleOutErrHandler.java +++ b/source/de/anomic/yacy/logging/ConsoleOutErrHandler.java @@ -135,6 +135,7 @@ public final class ConsoleOutErrHandler extends Handler { } else { this.stdOutHandler.publish(record); } + flush(); } public void flush() { diff --git a/source/de/anomic/yacy/logging/GuiHandler.java b/source/de/anomic/yacy/logging/GuiHandler.java index 76d2f655f..045d453a7 100644 --- a/source/de/anomic/yacy/logging/GuiHandler.java +++ b/source/de/anomic/yacy/logging/GuiHandler.java @@ -126,7 +126,8 @@ public class GuiHandler extends Handler { this.count++; } else { this.start++; - } + } + flush(); } public synchronized LogRecord[] getLogArray() { diff --git a/source/de/anomic/yacy/logging/LogalizerHandler.java b/source/de/anomic/yacy/logging/LogalizerHandler.java index a7a88577c..4c4b78eb1 100644 --- a/source/de/anomic/yacy/logging/LogalizerHandler.java +++ b/source/de/anomic/yacy/logging/LogalizerHandler.java @@ -131,6 +131,7 @@ public class LogalizerHandler extends Handler { if (debug) System.out.println("Logalizertest: " + returnV + " --- " + record.getLevel()); } } + flush(); } public Set getParserNames() { diff --git a/source/de/anomic/yacy/yacyURL.java b/source/de/anomic/yacy/yacyURL.java index 3ac2a72c1..5bfc8cb77 100644 --- a/source/de/anomic/yacy/yacyURL.java +++ b/source/de/anomic/yacy/yacyURL.java @@ -47,6 +47,7 @@ public class yacyURL implements Serializable { */ private static final long serialVersionUID = -1173233022912141884L; public static final int TLD_any_zone_filter = 255; // from TLD zones can be filtered during search; this is the catch-all filter + private final static Pattern backPathPattern = Pattern.compile("(/[^/]+(?