one more performance hack to prevent costly md5 computation

source/net/yacy/kelondro/order/Digest.java

@@ -45,7 +45,10 @@ import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import net.yacy.cora.document.UTF8;
+import net.yacy.cora.storage.ARC;
+import net.yacy.cora.storage.ConcurrentARC;
 import net.yacy.kelondro.logging.Log;
+import net.yacy.kelondro.util.MemoryControl;
 
 
 
@@ -53,6 +56,16 @@ public class Digest {
 
 	public static BlockingQueue<MessageDigest> digestPool = new LinkedBlockingDeque<MessageDigest>();
 
+    private static final int md5CacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
+    private static ARC<String, byte[]> md5Cache = null;
+    static {
+        try {
+            md5Cache = new ConcurrentARC<String, byte[]>(md5CacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()));
+        } catch (final OutOfMemoryError e) {
+            md5Cache = new ConcurrentARC<String, byte[]>(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
+        }
+    }
+    
     public static String encodeHex(final long in, final int length) {
         String s = Long.toHexString(in);
         while (s.length() < length) s = "0" + s;
@@ -103,8 +116,11 @@ public class Digest {
     }
 
     public static byte[] encodeMD5Raw(final String key) {
-    	MessageDigest digest = null;
-    	digest = digestPool.poll();
+
+        byte[] h = md5Cache.get(key);
+        if (h != null) return h;
+        
+    	MessageDigest digest = digestPool.poll();
     	if (digest == null) {
     	    // if there are no digest objects left, create some on the fly
     	    // this is not the most effective way but if we wouldn't do that the encoder would block
@@ -124,6 +140,13 @@ public class Digest {
             //System.out.println("Digest Pool size = " + digestPool.size());
         } catch ( InterruptedException e ) {
         }
+
+        // update the cache
+        if (MemoryControl.shortStatus()) {
+            md5Cache.clear();
+        } else {
+            md5Cache.insertIfAbsent(key, result); // prevent expensive MD5 computation and encoding
+        }
         return result;
     }