memory/performance hacks, less synchronization, better concurrency

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7544 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/search/SearchEvent.java

@@ -389,28 +389,29 @@ public final class SearchEvent {
          * @param singleAbstract // a mapping from url-hashes to a string of peer-hashes
          */
         public void addAbstract(String wordhash, TreeMap<String, String> singleAbstract) {
+            SortedMap<String, String> oldAbstract;
             synchronized (abstractsCache) {
-                SortedMap<String, String> oldAbstract = abstractsCache.get(wordhash);
+                oldAbstract = abstractsCache.get(wordhash);
                 if (oldAbstract == null) {
                     // new abstracts in the cache
                     abstractsCache.put(wordhash, singleAbstract);
-                } else {
-                    // extend the abstracts in the cache: join the single abstracts
-                    for (final Map.Entry<String, String> oneref: singleAbstract.entrySet()) {
-                        final String urlhash = oneref.getKey();
-                        final String peerlistNew = oneref.getValue();
-                        synchronized (oldAbstract) {
-                            final String peerlistOld = oldAbstract.get(urlhash);
-                            if (peerlistOld == null) {
-                                oldAbstract.put(urlhash, peerlistNew);
-                            } else {
-                                oldAbstract.put(urlhash, peerlistOld + peerlistNew);
-                            }
-                        }
+                    return;
+                }
+            }
+            // extend the abstracts in the cache: join the single abstracts
+            for (final Map.Entry<String, String> oneref: singleAbstract.entrySet()) {
+                final String urlhash = oneref.getKey();
+                final String peerlistNew = oneref.getValue();
+                synchronized (oldAbstract) {
+                    final String peerlistOld = oldAbstract.get(urlhash);
+                    if (peerlistOld == null) {
+                        oldAbstract.put(urlhash, peerlistNew);
+                    } else {
+                        oldAbstract.put(urlhash, peerlistOld + peerlistNew);
                     }
-                    // abstractsCache.put(wordhash, oldAbstract);
                 }
             }
+            // abstractsCache.put(wordhash, oldAbstract); // put not necessary since it is sufficient to just change the value content (it stays assigned)
         }
         
         public void commitAbstract() {

source/de/anomic/yacy/yacySeedDB.java

@@ -1020,8 +1020,12 @@ public final class yacySeedDB implements AlternativeDomainNames {
                     dna0 = it.next();
                     assert dna0 != null;
                     if (dna0 == null) continue;
-                    dna = new ConcurrentHashMap<String, String>();
-                    dna.putAll(dna0);
+                    if (dna0 instanceof ConcurrentHashMap) {
+                        dna = (ConcurrentHashMap<String, String>) dna0;
+                    } else {
+                        dna = new ConcurrentHashMap<String, String>();
+                        dna.putAll(dna0);
+                    }
                     final String hash = dna.remove("key");
                     assert hash != null;
                     if (hash == null) continue; // bad seed

source/net/yacy/kelondro/blob/ArrayStack.java

@@ -802,16 +802,22 @@ public class ArrayStack implements BLOB {
             blobItem bi = blobs.get(0);
             bi.blob.delete(key);
         } else {
-            Thread[] t = new Thread[blobs.size()];
+            Thread[] t = new Thread[blobs.size() - 1];
             int i = 0;
             for (blobItem bi: blobs) {
-                final blobItem bi0 = bi;
-                t[i] = new Thread() {
-                    public void run() {
-                        try { bi0.blob.delete(key); } catch (IOException e) {}
-                    }
-                };
-                t[i].start();
+                if (i < t.length) {
+                    // run this in a concurrent thread
+                    final blobItem bi0 = bi;
+                    t[i] = new Thread() {
+                        public void run() {
+                            try { bi0.blob.delete(key); } catch (IOException e) {}
+                        }
+                    };
+                    t[i].start();
+                } else {
+                    // no additional thread, run in this thread
+                    try { bi.blob.delete(key); } catch (IOException e) {}
+                }
                 i++;
             }
             for (Thread s: t) try {s.join();} catch (InterruptedException e) {}

source/net/yacy/kelondro/data/meta/DigestURI.java

@@ -225,7 +225,9 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
     }
     
     private static char subdomPortPath(final String subdom, final int port, final String rootpath) {
-        return Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(subdom + ":" + port + ":" + rootpath)).charAt(0);
+        StringBuilder sb = new StringBuilder(subdom.length() + rootpath.length() + 8);
+        sb.append(subdom).append(':').append(Integer.toString(port)).append(':').append(rootpath);
+        return Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(sb.toString())).charAt(0);
     }
 
     private static final char rootURLFlag0 = subdomPortPath("", 80, "");

source/net/yacy/kelondro/data/word/Word.java

@@ -56,7 +56,7 @@ public class Word {
     public static final int commonHashLength = 12;
     
     private static final int hashCacheSize = Math.max(100000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
-    private static final ARC<String, byte[]> hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, Runtime.getRuntime().availableProcessors() + 1);
+    private static final ARC<String, byte[]> hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors());
     
     // object carries statistics for words and sentences
     public  int      count;       // number of occurrences

source/net/yacy/kelondro/index/HandleSet.java

@@ -164,21 +164,19 @@ public final class HandleSet implements Iterable<byte[]>, Cloneable {
      * @throws IOException
      * @throws RowSpaceExceededException
      */
-    public final synchronized boolean put(final byte[] key) throws RowSpaceExceededException {
+    public final boolean put(final byte[] key) throws RowSpaceExceededException {
         assert (key != null);
-        final Row.Entry newentry = index.row().newEntry();
-        newentry.setCol(0, key);
+        final Row.Entry newentry = index.row().newEntry(key);
         return index.put(newentry);
     }
     
-    public final synchronized void putUnique(final byte[] key) throws RowSpaceExceededException {
+    public final void putUnique(final byte[] key) throws RowSpaceExceededException {
         assert (key != null);
-        final Row.Entry newentry = this.rowdef.newEntry();
-        newentry.setCol(0, key);
+        final Row.Entry newentry = index.row().newEntry(key);
         index.addUnique(newentry);
     }
     
-    public final synchronized boolean remove(final byte[] key) {
+    public final boolean remove(final byte[] key) {
         assert (key != null);
         Row.Entry indexentry;
         indexentry = index.remove(key);

source/net/yacy/kelondro/util/FileUtils.java

@@ -750,6 +750,9 @@ public final class FileUtils {
                 }
             } catch (IOException e) {
                 nextLine = null;
+            } catch (OutOfMemoryError e) {
+                Log.logException(e);
+                nextLine = null;
             }
             return line;
         }