more concurrency (enhancement) hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7759 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/search/TextSnippet.java

@@ -87,8 +87,8 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
         public void put(final String wordhashes, final String urlhash, final String snippet) {
             // generate key
             final String key = urlhash + wordhashes;
-
+
-            // do nothing if snippet is known or learn new snippet
+            // do nothing if snippet is known or otherwise learn new snippet
             cache.insertIfAbsent(key, snippet);
         }
         

source/net/yacy/cora/storage/ARC.java

@@ -63,6 +63,15 @@ public interface ARC<K, V> extends Iterable<Map.Entry<K, V>> {
      */
     public void insertIfAbsent(K s, V v);
 
+    /**
+     * put a value to the cache if there was not an entry before
+     * return a previous content value
+     * @param s
+     * @param v
+     * @return the value before inserting the new value
+     */
+    public V putIfAbsent(K s, V v);
+
     /**
      * put a value to the cache.
      * @param s

source/net/yacy/cora/storage/ConcurrentARC.java

@@ -101,6 +101,17 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
     public void insertIfAbsent(K s, V v) {
         this.arc[getPartition(s)].insertIfAbsent(s, v);
     }
+
+    /**
+     * put a value to the cache if there was not an entry before
+     * return a previous content value
+     * @param s
+     * @param v
+     * @return the value before inserting the new value
+     */
+    public V putIfAbsent(K s, V v) {
+        return this.arc[getPartition(s)].putIfAbsent(s, v);
+    }
     
     /**
      * put a value to the cache.
@@ -206,18 +217,24 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
     public int hashCode() {
         return this.arc.hashCode();
     }
+    
+    //private static String latestObject = "";
     /**
      * return in which partition the Object belongs
      * This function uses the objects hashCode() function
      * except for byte[] keys
-     * @return the partitrion number
+     * @return the partition number
      */
     private int getPartition(final Object x) {
         if (x instanceof byte[]) {
             int h = 0;
             for (byte c: (byte[])x) h = 31 * h + (c & 0xFF);
-            return h & mask;
+            int p = h & mask;
+            //String o = UTF8.String((byte[]) x); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objectb = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o;
+            return p;
         }
-        return x.hashCode() & mask;
+        int p = x.hashCode() & mask;
+        //String o = x.toString(); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objecto = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o;
+        return p;
     }
 }

source/net/yacy/cora/storage/SimpleARC.java

@@ -91,6 +91,25 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
         }
     }
     
+    /**
+     * put a value to the cache if there was not an entry before
+     * return a previous content value
+     * @param s
+     * @param v
+     * @return the value before inserting the new value
+     */
+    public V putIfAbsent(K s, V v) {
+        synchronized (this) {
+            V o = this.levelB.get(s);
+            if (o != null) return o;
+            o = this.levelA.get(s);
+            if (o != null) return o;
+            this.levelA.put(s, v);
+            assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically
+            return null;
+        }
+    }
+    
     /**
      * put a value to the cache.
      * @param s

source/net/yacy/kelondro/blob/MapDataMining.java

@@ -279,6 +279,23 @@ public class MapDataMining extends MapHeap {
         super.delete(key);
     }
     
+/* would be better but does not work (recursion)
+    @Override
+    public synchronized void delete(final byte[] key) throws IOException {
+        if (key == null) return;
+        
+        // update elementCount
+        Map<String, String> map = super.remove(key);
+        if (map != null && (sortfields != null || longaccfields != null || floataccfields != null)) {
+            // update accumulators (subtract)
+            if ((longaccfields != null) || (floataccfields != null)) updateAcc(map, false);
+
+            // remove from sortCluster
+            if (sortfields != null) deleteSortCluster(UTF8.String(key));
+        }
+    }
+*/
+
     private void deleteSortCluster(final String key) {
         if (key == null) return;
         ScoreMap<String> cluster;

source/net/yacy/kelondro/blob/MapHeap.java

@@ -69,7 +69,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
             final int cachesize,
             char fillchar) throws IOException {
         this.blob = new Heap(heapFile, keylength, ordering, buffermax);
-        this.cache = new ConcurrentARC<byte[], Map<String, String>>(cachesize, Runtime.getRuntime().availableProcessors(), ordering);
+        this.cache = new ConcurrentARC<byte[], Map<String, String>>(cachesize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()), ordering);
         this.fillchar = fillchar;
     }
    
@@ -285,6 +285,10 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
         if (cache == null) return null; // case may appear during shutdown
         key = normalizeKey(key);
         
+        if (MemoryControl.shortStatus()) {
+            cache.clear();
+        } 
+        
         Map<String, String> map;
         if (storeCache) {
             synchronized (this) {
@@ -300,16 +304,12 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
                     throw new IOException(e.getMessage());
                 }
         
-                if (MemoryControl.shortStatus()) {
+                // write map to cache
-                    cache.clear();
+                cache.insert(key, map);
-                } else {
+                
-                    // write map to cache
+                // return value
-                    cache.insert(key, map);
+                return map;
-                }
             }
-            
-            // return value
-            return map;
         } else {
             byte[] b;
             synchronized (this) {

source/net/yacy/kelondro/data/word/Word.java

@@ -30,7 +30,6 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Locale;
 import java.util.Set;
-
 import net.yacy.cora.storage.ARC;
 import net.yacy.cora.storage.ConcurrentARC;
 import net.yacy.document.LargeNumberCache;
@@ -54,16 +53,22 @@ public class Word {
      * these hashes all shall be generated by base64.enhancedCoder
      */
     public static final int commonHashLength = 12;
-    
+
     private static final int hashCacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
     private static ARC<String, byte[]> hashCache = null;
     static {
         try {
-            hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors());
+            hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()));
         } catch (OutOfMemoryError e) {
-            hashCache = new ConcurrentARC<String, byte[]>(1000, Runtime.getRuntime().availableProcessors());
+            hashCache = new ConcurrentARC<String, byte[]>(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
         }
     }
+    /*
+    private static ConcurrentHashMap<String, byte[]> hashCache = null;
+    static {
+        hashCache = new ConcurrentHashMap<String, byte[]>();
+    }
+    */
     
     // object carries statistics for words and sentences
     public  int      count;       // number of occurrences
@@ -122,6 +127,7 @@ public class Word {
         if (MemoryControl.shortStatus()) {
             hashCache.clear();
         } else {
+            //hashCache.putIfAbsent(wordlc, h); // prevent expensive MD5 computation and encoding
             hashCache.insertIfAbsent(wordlc, h); // prevent expensive MD5 computation and encoding
         }
         return h;

source/net/yacy/kelondro/util/FileUtils.java

@@ -142,7 +142,10 @@ public final class FileUtils {
     }
     
     public static int copy(final Reader source, final Writer dest) throws IOException {
+        assert source != null;
+        assert dest != null;
         if (source == null) throw new IOException("source is null");
+        if (dest == null) throw new IOException("dest is null");
         final char[] buffer = new char[DEFAULT_BUFFER_SIZE];
         int count = 0;
         int n = 0;
@@ -153,9 +156,10 @@ public final class FileUtils {
             }
             dest.flush();
         } catch (final Exception e) {
+            assert e != null;
             // an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here
             //Log.logException(e);
-            throw new IOException(e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), e);
+            //throw new IOException(e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), e);
         }
         return count;
     }