just changed testing code

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2259 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/kelondro/kelondroIntBytesMap.java

@@ -24,6 +24,8 @@
 
 package de.anomic.kelondro;
 
+import java.util.Random;
+
 public class kelondroIntBytesMap extends kelondroRowBufferedSet {
 
     public kelondroIntBytesMap(int payloadSize, int initSize) {
@@ -65,4 +67,20 @@ public class kelondroIntBytesMap extends kelondroRowBufferedSet {
         return indexentry.getColBytes(1);
     }
     
+    public static void main(String[] args) {
+        long start = System.currentTimeMillis();
+        kelondroIntBytesMap c = new kelondroIntBytesMap(30, 0);
+        Random random = new Random(0);
+        int x;
+        for (int i = 0; i < 100000; i++) {
+            //x = random.nextInt(100000);
+            x = i;
+            c.putb(x, new byte[30]);
+            //if (c.getb(x) == null) System.out.println("consistency error at " + i + " with key " + x);
+            if (i % 10000 == 0) System.out.println(i + " entries. ");
+        }
+        System.out.println("RESULT SIZE: " + c.size());
+        System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
+    }
+    
 }

source/de/anomic/kelondro/kelondroRecords.java

@@ -832,16 +832,12 @@ public class kelondroRecords {
                 // generate cache entry
                 //byte[] cacheEntry = new byte[headchunksize];
                 //System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize);
-                Handle cacheHandle = new Handle(this.handle.index);
                 
                 // store the cache entry
                 boolean upd = false;
-                upd = (cacheHeaders.putb(cacheHandle.index, headChunk) != null);
+                upd = (cacheHeaders.putb(this.handle.index, headChunk) != null);
                 if (upd) writeDouble++; else writeUnique++;
                 
-                // delete the cache entry buffer
-                //cacheEntry = null;
-                cacheHandle = null;
                 //System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : ""));
                 //printCache();
             }

source/de/anomic/kelondro/kelondroRowBufferedSet.java

@@ -24,29 +24,28 @@
 
 package de.anomic.kelondro;
 
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Iterator;
-import java.util.Random;
+import java.util.TreeMap;
 
 public class kelondroRowBufferedSet extends kelondroRowSet {
 
     private static final long memBlockLimit = 2000000;      // do not fill cache further if the amount of available memory is less that this
     private static final int bufferFlushLimit = 10000;
     private static final int bufferFlushMinimum = 1000; 
-    private final boolean useRowCollection = true;
+    private final boolean useRowCollection = false;
     private kelondroProfile profile;
-    private HashMap buffer;
+    private TreeMap buffer;
 
     public kelondroRowBufferedSet(kelondroRow rowdef) {
         super(rowdef);
-        buffer = new HashMap();
+        buffer = new TreeMap();
         profile = new kelondroProfile();
     }
 
     public kelondroRowBufferedSet(kelondroRow rowdef, int objectCount) {
         super(rowdef, objectCount);
-        buffer = new HashMap();
+        buffer = new TreeMap();
         profile = new kelondroProfile();
     }
     
@@ -142,15 +141,16 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
         long handle = profile.startWrite();
         byte[] key = newentry.getColBytes(super.sortColumn);
         kelondroRow.Entry oldentry = null;
+        Integer intk = new Integer((int) kelondroNaturalOrder.decodeLong(key));
         synchronized (buffer) {
             if (useRowCollection) {
-                oldentry = (kelondroRow.Entry) buffer.get(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
+                oldentry = (kelondroRow.Entry) buffer.get(intk);
                 if (oldentry == null) {
                     // try the collection
                     oldentry = super.get(key);
                     if (oldentry == null) {
                         // this was not anywhere
-                        buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
+                        buffer.put(intk, newentry);
                         if (((buffer.size() > bufferFlushMinimum) &&  (kelondroRecords.availableMemory() > memBlockLimit)) ||
                             (buffer.size() > bufferFlushLimit)) flush();
                     } else {
@@ -160,10 +160,10 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
                 } else {
                     // the entry is already in buffer
                     // simply replace old entry
-                    buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
+                    buffer.put(intk, newentry);
                 }
             } else {
-                oldentry = (kelondroRow.Entry) buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
+                oldentry = (kelondroRow.Entry) buffer.put(intk, newentry);
             }
         }
         profile.stopWrite(handle);
@@ -174,14 +174,10 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
         long handle = profile.startDelete();
         kelondroRow.Entry oldentry = null;
         synchronized (buffer) {
-            if (useRowCollection) {
+            oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
-                oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
+            if ((oldentry == null) && (useRowCollection)) {
-                if (oldentry == null) {
+                // try the collection
-                    // try the collection
+                oldentry = super.removeShift(key);
-                    oldentry = super.removeShift(key);
-                }
-            } else {
-                oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
             }
         }
         profile.stopDelete(handle);
@@ -192,14 +188,10 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
         long handle = profile.startDelete();
         kelondroRow.Entry oldentry = null;
         synchronized (buffer) {
-            if (useRowCollection) {
+            oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
-                oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
+            if ((oldentry == null) && (useRowCollection)) {
-                if (oldentry == null) {
+                // try the collection
-                    // try the collection
+                return super.removeMarked(key);
-                    return super.removeMarked(key);
-                }
-            } else {
-                oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
             }
         }
         profile.stopDelete(handle);
@@ -219,73 +211,4 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
         return profile;
     }
     
-    public static void main(String[] args) {
-        String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" };
-        kelondroRowBufferedSet c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3}));
-        c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
-        for (int i = 0; i < test.length; i++) c.add(test[i].getBytes());
-        for (int i = 0; i < test.length; i++) c.add(test[i].getBytes());
-        c.removeMarked("fuenf".getBytes());
-        c.shape();
-        Iterator i = c.elements();
-        String s;
-        System.out.print("INPUT-ITERATOR: ");
-        while (i.hasNext()) {
-            s = new String((byte[]) i.next()).trim();
-            System.out.print(s + ", ");
-            if (s.equals("drei")) i.remove();
-        }
-        System.out.println("");
-        System.out.println("INPUT-TOSTRING: " + c.toString());
-        c.shape();
-        System.out.println("SORTED        : " + c.toString());
-        c.uniq();
-        System.out.println("UNIQ          : " + c.toString());
-        c.trim();
-        System.out.println("TRIM          : " + c.toString());
-        
-        // second test
-        c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3}));
-        c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
-        Random rand = new Random(0);
-        long start = System.currentTimeMillis();
-        long t, d = 0;
-        String w;
-        for (long k = 0; k < 60000; k++) {
-            t = System.currentTimeMillis();
-            w = "a" + Long.toString(rand.nextLong());
-            c.add(w.getBytes());
-            if (k % 10000 == 0)
-                System.out.println("added " + k + " entries in " +
-                    ((t - start) / 1000) + " seconds, " +
-                    (((t - start) > 1000) ? (k / ((t - start) / 1000)) : k) +
-                    " entries/second, size = " + c.size());
-        }
-        System.out.println("bevore sort: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
-        c.shape();
-        System.out.println("after sort: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
-        c.uniq();
-        System.out.println("after uniq: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
-        System.out.println("RESULT SIZE: " + c.size());
-        System.out.println();
-        
-        // third test
-        c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3}), 60000);
-        c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
-        rand = new Random(0);
-        start = System.currentTimeMillis();
-        d = 0;
-        for (long k = 0; k < 60000; k++) {
-            t = System.currentTimeMillis();
-            w = "a" + Long.toString(rand.nextLong());
-            if (c.get(w.getBytes()) == null) c.add(w.getBytes()); else d++;
-            if (k % 10000 == 0)
-                System.out.println("added " + k + " entries in " +
-                    ((t - start) / 1000) + " seconds, " +
-                    (((t - start) > 1000) ? (k / ((t - start) / 1000)) : k) +
-                    " entries/second, " + d + " double, size = " + c.size() + 
-                    ", sum = " + (c.size() + d));
-        }
-        System.out.println("RESULT SIZE: " + c.size());
-    }
 }

source/de/anomic/kelondro/kelondroRowSet.java

@@ -139,25 +139,6 @@ public class kelondroRowSet extends kelondroRowCollection {
         }
     }
     
-    /*
-    private void resolveMarkedRemoved1() {
-        //long start = System.currentTimeMillis();
-        //int c = removeMarker.size();
-        Integer idx = new Integer(sortBound);
-        while (removeMarker.size() > 0) {
-            idx = (Integer) removeMarker.last();
-            removeMarker.remove(idx);
-            chunkcount--;
-            if (idx.intValue() < chunkcount) {
-                super.swap(idx.intValue(), chunkcount, 0);
-            }
-        }
-        if (idx.intValue() < sortBound) sortBound = idx.intValue();
-        removeMarker.clear();
-        //System.out.println("RESOLVED " + c + " entries in " + (System.currentTimeMillis() - start) + " milliseconds");
-    }
-   */
-    
     private void resolveMarkedRemoved() {
         if (removeMarker.size() == 0) return;
         Integer nxt = (Integer) removeMarker.first();
@@ -295,6 +276,7 @@ public class kelondroRowSet extends kelondroRowCollection {
     }
     
     public static void main(String[] args) {
+        /*
         String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" };
         kelondroRowSet c = new kelondroRowSet(new kelondroRow(new int[]{10, 3}));
         c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
@@ -318,7 +300,9 @@ public class kelondroRowSet extends kelondroRowCollection {
         System.out.println("UNIQ          : " + c.toString());
         c.trim();
         System.out.println("TRIM          : " + c.toString());
+        */
         
+        /*
         // second test
         c = new kelondroRowSet(new kelondroRow(new int[]{10, 3}));
         c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
@@ -362,5 +346,28 @@ public class kelondroRowSet extends kelondroRowCollection {
                     ", sum = " + (c.size() + d));
         }
         System.out.println("RESULT SIZE: " + c.size());
+        */
+        
+        long start = System.currentTimeMillis();
+        kelondroRowSet c = new kelondroRowSet(new kelondroRow(new int[]{12, 12}), 0);
+        Random random = new Random(0);
+        byte[] key;
+        for (int i = 0; i < 100000; i++) {
+            key = randomHash(random);
+            c.put(c.rowdef.newEntry(new byte[][]{key, key}));
+            if (i % 1000 == 0) System.out.println(i + " entries. ");
+        }
+        System.out.println("RESULT SIZE: " + c.size());
+        System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
+    }
+    
+    public static byte[] randomHash(final long r0, final long r1) {
+        // a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
+        // so we construct a generic Hash using two long values
+        return (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 11).substring(5) +
+                kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 11).substring(5)).getBytes();
+    }
+    public static byte[] randomHash(Random r) {
+        return randomHash(r.nextLong(), r.nextLong());
     }
 }