From 2c58af6874f474ff581efe0ec1f3e559a4f63c41 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 17 Aug 2011 22:24:17 +0000 Subject: [PATCH] - added a short memory status simulation mode - added a button in PerformanceMemory_p.html to set the simulated short memory status - bugfix: added a missing lowercase in KeyList - better concurrency in loader dispatcher git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7883 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/PerformanceMemory_p.html | 5 +- htroot/PerformanceMemory_p.java | 3 + source/net/yacy/cora/protocol/Domains.java | 16 ++++-- source/net/yacy/cora/storage/KeyList.java | 57 ++++++++++++------- .../net/yacy/kelondro/util/MemoryControl.java | 20 ++++++- .../net/yacy/repository/LoaderDispatcher.java | 5 +- 6 files changed, 74 insertions(+), 32 deletions(-) diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html index bc90a3412..f56c965a7 100644 --- a/htroot/PerformanceMemory_p.html +++ b/htroot/PerformanceMemory_p.html @@ -20,7 +20,10 @@

PerformanceGraph

- +
+

simulate short memory status

+
+

Memory Usage:

diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index 51dfbb81a..2375cdbf0 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -62,8 +62,11 @@ public class PerformanceMemory_p { System.gc(); prop.put("gc", "1"); } + MemoryControl.setSimulatedShortStatus(post.containsKey("simulatedshortmemory")); } + prop.put("simulatedshortmemory.checked", MemoryControl.getSimulatedShortStatus() ? 1 : 0); + final long memoryFreeNow = MemoryControl.free(); final long memoryFreeAfterInitBGC = env.getConfigLong("memoryFreeAfterInitBGC", 0L); final long memoryFreeAfterInitAGC = env.getConfigLong("memoryFreeAfterInitAGC", 0L); diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 24165ce31..e13e4840f 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -46,6 +46,7 @@ import java.util.regex.Pattern; import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.cora.storage.KeyList; +import net.yacy.kelondro.util.MemoryControl; public class Domains { @@ -522,18 +523,18 @@ public class Domains { } */ } - + /** * in case that the host name was resolved using a time-out request * it can be nice to push that information to the name cache * @param i the inet address * @param host the known host name */ - public static void setHostName(final InetAddress i, String host) { + public static void setHostName(final InetAddress i, final String host) { NAME_CACHE_HIT.insertIfAbsent(host, i); cacheHit_Insert++; } - + public static InetAddress dnsResolve(String host) { if ((host == null) || (host.length() == 0)) return null; host = host.toLowerCase().trim(); @@ -610,7 +611,14 @@ public class Domains { if (localp) { localHostNames.add(host); } else { - if (globalHosts != null) try {globalHosts.add(host);} catch (final IOException e) {} + if (globalHosts != null) try { + if (MemoryControl.shortStatus()) { + globalHosts.close(); + globalHosts = null; + } else { + globalHosts.add(host); + } + } catch (final IOException e) {} } } //LOOKUP_SYNC.remove(host); diff --git a/source/net/yacy/cora/storage/KeyList.java b/source/net/yacy/cora/storage/KeyList.java index ee88c01eb..7c8b40c7f 100644 --- a/source/net/yacy/cora/storage/KeyList.java +++ b/source/net/yacy/cora/storage/KeyList.java @@ -11,12 +11,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -46,13 +46,13 @@ import net.yacy.cora.document.UTF8; public class KeyList { private static final Object _obj = new Object(); - - private Map keys; - private RandomAccessFile raf; - - public KeyList(File file) throws IOException { + + private final Map keys; + private final RandomAccessFile raf; + + public KeyList(final File file) throws IOException { this.keys = new ConcurrentHashMap(); - + if (file.exists()) { InputStream is = new FileInputStream(file); if (file.getName().endsWith(".gz")) { @@ -66,34 +66,47 @@ public class KeyList { l = l.trim().toLowerCase(); this.keys.put(l, _obj); } - } catch (IOException e) { + } catch (final IOException e) { // finish } } - + this.raf = new RandomAccessFile(file, "rw"); - + } - - public boolean contains(String key) { - return this.keys.containsKey(key); + + public boolean contains(final String key) { + return this.keys.containsKey(key.trim().toLowerCase()); } - - public void add(String key) throws IOException { - if (keys.containsKey(key)) return; + + public void add(final String key) throws IOException { + if (this.keys.containsKey(key)) return; synchronized (this.raf) { - if (keys.containsKey(key)) return; // check again for those threads who come late (after another has written this) + if (this.keys.containsKey(key)) return; // check again for those threads who come late (after another has written this) this.keys.put(key, _obj); - this.raf.seek(raf.length()); + this.raf.seek(this.raf.length()); this.raf.write(UTF8.getBytes(key)); this.raf.writeByte('\n'); } } - + public void close() throws IOException { synchronized (this.raf) { - raf.close(); + this.raf.close(); } } - + + public static void main(final String[] args) { + try { + final KeyList kl = new KeyList(new File("/tmp/test")); + kl.add("eins"); + kl.add("zwei"); + kl.add("drei"); + System.out.println(kl.contains("eins") ? "drin" : "nicht"); + } catch (final IOException e) { + e.printStackTrace(); + } + + } + } diff --git a/source/net/yacy/kelondro/util/MemoryControl.java b/source/net/yacy/kelondro/util/MemoryControl.java index 4e9c4d1d3..2657bd375 100644 --- a/source/net/yacy/kelondro/util/MemoryControl.java +++ b/source/net/yacy/kelondro/util/MemoryControl.java @@ -43,7 +43,7 @@ public class MemoryControl { private static long prevDHTtreshold = 0L; private static int DHTtresholdCount = 0; private static boolean allowDHT = true; - private static boolean shortStatus = false; + private static boolean shortStatus = false, simulatedShortStatus = false; /** * Runs the garbage collector if last garbage collection is more than last millis ago @@ -178,9 +178,25 @@ public class MemoryControl { } } + /** + * the simulated short status can be set to find out if the short status has effects to the system + * @param status + */ + public static void setSimulatedShortStatus(final boolean status) { + simulatedShortStatus = status; + } + + /** + * the simulated short status can be retrieved to show that option in online interfaces + * @return + */ + public static boolean getSimulatedShortStatus() { + return simulatedShortStatus; + } + public static boolean shortStatus() { //if (shortStatus) System.out.println("**** SHORT MEMORY ****"); - return shortStatus; + return simulatedShortStatus || shortStatus; } /** diff --git a/source/net/yacy/repository/LoaderDispatcher.java b/source/net/yacy/repository/LoaderDispatcher.java index 2e0ed9808..478a25204 100644 --- a/source/net/yacy/repository/LoaderDispatcher.java +++ b/source/net/yacy/repository/LoaderDispatcher.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.util.Arrays; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Map; @@ -78,7 +77,7 @@ public final class LoaderDispatcher { private final FTPLoader ftpLoader; private final SMBLoader smbLoader; private final FileLoader fileLoader; - private final HashMap loaderSteering; // a map that delivers a 'finish' semaphore for urls + private final ConcurrentHashMap loaderSteering; // a map that delivers a 'finish' semaphore for urls private final Log log; public LoaderDispatcher(final Switchboard sb) { @@ -91,7 +90,7 @@ public final class LoaderDispatcher { this.ftpLoader = new FTPLoader(sb, this.log); this.smbLoader = new SMBLoader(sb, this.log); this.fileLoader = new FileLoader(sb, this.log); - this.loaderSteering = new HashMap(); + this.loaderSteering = new ConcurrentHashMap(); } public boolean isSupportedProtocol(final String protocol) {