- added a short memory status simulation mode

- added a button in PerformanceMemory_p.html to set the simulated short memory status
- bugfix: added a missing lowercase in KeyList
- better concurrency in loader dispatcher

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7883 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/PerformanceMemory_p.html

@@ -20,7 +20,10 @@
     
     <p><img src="PerformanceGraph.png" id="graph" alt="PerformanceGraph"/></p>
     <form id="optionreloadGraph" action="" method="get"><p><input type="checkbox" name="option" id="autoreload"/> <label for="autoreload">refresh graph</label></p></form>
-	    
+	<form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
+	<p><input type="checkbox" name="simulatedshortmemory" onclick = 'this.form.submit()' #(simulatedshortmemory.checked)#:: checked="checked"#(/simulatedshortmemory.checked)#/>simulate short memory status</label></p>
+	</form>
+    
     <p><strong>Memory Usage:</strong></p>
     <table border="0" cellpadding="2" cellspacing="1">
       <tr class="TableHeader" valign="bottom">

htroot/PerformanceMemory_p.java

@@ -62,8 +62,11 @@ public class PerformanceMemory_p {
                 System.gc();
                 prop.put("gc", "1");
             }
+            MemoryControl.setSimulatedShortStatus(post.containsKey("simulatedshortmemory"));
         }
 
+        prop.put("simulatedshortmemory.checked", MemoryControl.getSimulatedShortStatus() ? 1 : 0);
+
         final long memoryFreeNow = MemoryControl.free();
         final long memoryFreeAfterInitBGC = env.getConfigLong("memoryFreeAfterInitBGC", 0L);
         final long memoryFreeAfterInitAGC = env.getConfigLong("memoryFreeAfterInitAGC", 0L);

source/net/yacy/cora/protocol/Domains.java

@@ -46,6 +46,7 @@ import java.util.regex.Pattern;
 import net.yacy.cora.storage.ARC;
 import net.yacy.cora.storage.ConcurrentARC;
 import net.yacy.cora.storage.KeyList;
+import net.yacy.kelondro.util.MemoryControl;
 
 public class Domains {
 
@@ -522,18 +523,18 @@ public class Domains {
         }
         */
     }
-    
+
     /**
      * in case that the host name was resolved using a time-out request
      * it can be nice to push that information to the name cache
      * @param i the inet address
      * @param host the known host name
      */
-    public static void setHostName(final InetAddress i, String host) {
+    public static void setHostName(final InetAddress i, final String host) {
         NAME_CACHE_HIT.insertIfAbsent(host, i);
         cacheHit_Insert++;
     }
-    
+
     public static InetAddress dnsResolve(String host) {
         if ((host == null) || (host.length() == 0)) return null;
         host = host.toLowerCase().trim();
@@ -610,7 +611,14 @@ public class Domains {
                 if (localp) {
                     localHostNames.add(host);
                 } else {
-                    if (globalHosts != null) try {globalHosts.add(host);} catch (final IOException e) {}
+                    if (globalHosts != null) try {
+                        if (MemoryControl.shortStatus()) {
+                            globalHosts.close();
+                            globalHosts = null;
+                        } else {
+                            globalHosts.add(host);
+                        }
+                    } catch (final IOException e) {}
                 }
             }
             //LOOKUP_SYNC.remove(host);

source/net/yacy/cora/storage/KeyList.java

@@ -11,12 +11,12 @@
  *  modify it under the terms of the GNU Lesser General Public
  *  License as published by the Free Software Foundation; either
  *  version 2.1 of the License, or (at your option) any later version.
- *  
+ *
  *  This library is distributed in the hope that it will be useful,
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  *  Lesser General Public License for more details.
- *  
+ *
  *  You should have received a copy of the GNU Lesser General Public License
  *  along with this program in the file lgpl21.txt
  *  If not, see <http://www.gnu.org/licenses/>.
@@ -46,13 +46,13 @@ import net.yacy.cora.document.UTF8;
 public class KeyList {
 
     private static final Object _obj = new Object();
-    
-    private Map<String, Object> keys;
-    private RandomAccessFile raf;
-    
-    public KeyList(File file) throws IOException {
+
+    private final Map<String, Object> keys;
+    private final RandomAccessFile raf;
+
+    public KeyList(final File file) throws IOException {
         this.keys = new ConcurrentHashMap<String, Object>();
-        
+
         if (file.exists()) {
             InputStream is = new FileInputStream(file);
             if (file.getName().endsWith(".gz")) {
@@ -66,34 +66,47 @@ public class KeyList {
                     l = l.trim().toLowerCase();
                     this.keys.put(l, _obj);
                 }
-            } catch (IOException e) {
+            } catch (final IOException e) {
                 // finish
             }
         }
-        
+
         this.raf = new RandomAccessFile(file, "rw");
-        
+
     }
-    
-    public boolean contains(String key) {
-        return this.keys.containsKey(key);
+
+    public boolean contains(final String key) {
+        return this.keys.containsKey(key.trim().toLowerCase());
     }
-    
-    public void add(String key) throws IOException {
-        if (keys.containsKey(key)) return;
+
+    public void add(final String key) throws IOException {
+        if (this.keys.containsKey(key)) return;
         synchronized (this.raf) {
-            if (keys.containsKey(key)) return; // check again for those threads who come late (after another has written this)
+            if (this.keys.containsKey(key)) return; // check again for those threads who come late (after another has written this)
             this.keys.put(key, _obj);
-            this.raf.seek(raf.length());
+            this.raf.seek(this.raf.length());
             this.raf.write(UTF8.getBytes(key));
             this.raf.writeByte('\n');
         }
     }
-    
+
     public void close() throws IOException {
         synchronized (this.raf) {
-            raf.close();
+            this.raf.close();
         }
     }
-    
+
+    public static void main(final String[] args) {
+        try {
+            final KeyList kl = new KeyList(new File("/tmp/test"));
+            kl.add("eins");
+            kl.add("zwei");
+            kl.add("drei");
+            System.out.println(kl.contains("eins") ? "drin" : "nicht");
+        } catch (final IOException e) {
+            e.printStackTrace();
+        }
+
+    }
+
 }

source/net/yacy/kelondro/util/MemoryControl.java

@@ -43,7 +43,7 @@ public class MemoryControl {
     private static long prevDHTtreshold = 0L;
     private static int DHTtresholdCount = 0;
     private static boolean allowDHT = true;
-    private static boolean shortStatus = false;
+    private static boolean shortStatus = false, simulatedShortStatus = false;
 
     /**
      * Runs the garbage collector if last garbage collection is more than last millis ago
@@ -178,9 +178,25 @@ public class MemoryControl {
         }
     }
 
+    /**
+     * the simulated short status can be set to find out if the short status has effects to the system
+     * @param status
+     */
+    public static void setSimulatedShortStatus(final boolean status) {
+        simulatedShortStatus = status;
+    }
+
+    /**
+     * the simulated short status can be retrieved to show that option in online interfaces
+     * @return
+     */
+    public static boolean getSimulatedShortStatus() {
+        return simulatedShortStatus;
+    }
+
     public static boolean shortStatus() {
         //if (shortStatus) System.out.println("**** SHORT MEMORY ****");
-        return shortStatus;
+        return simulatedShortStatus || shortStatus;
     }
 
     /**

source/net/yacy/repository/LoaderDispatcher.java

@@ -32,7 +32,6 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.util.Arrays;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
@@ -78,7 +77,7 @@ public final class LoaderDispatcher {
     private final FTPLoader ftpLoader;
     private final SMBLoader smbLoader;
     private final FileLoader fileLoader;
-    private final HashMap<String, Semaphore> loaderSteering; // a map that delivers a 'finish' semaphore for urls
+    private final ConcurrentHashMap<String, Semaphore> loaderSteering; // a map that delivers a 'finish' semaphore for urls
     private final Log log;
 
     public LoaderDispatcher(final Switchboard sb) {
@@ -91,7 +90,7 @@ public final class LoaderDispatcher {
         this.ftpLoader = new FTPLoader(sb, this.log);
         this.smbLoader = new SMBLoader(sb, this.log);
         this.fileLoader = new FileLoader(sb, this.log);
-        this.loaderSteering = new HashMap<String, Semaphore>();
+        this.loaderSteering = new ConcurrentHashMap<String, Semaphore>();
     }
 
     public boolean isSupportedProtocol(final String protocol) {