From f9b9d085c4e2d1d9bac2d419dfc90e0a27284d16 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 29 Jun 2006 23:01:42 +0000 Subject: [PATCH] just changed testing code git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2259 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../anomic/kelondro/kelondroIntBytesMap.java | 18 +++ .../de/anomic/kelondro/kelondroRecords.java | 6 +- .../kelondro/kelondroRowBufferedSet.java | 113 +++--------------- source/de/anomic/kelondro/kelondroRowSet.java | 45 ++++--- 4 files changed, 63 insertions(+), 119 deletions(-) diff --git a/source/de/anomic/kelondro/kelondroIntBytesMap.java b/source/de/anomic/kelondro/kelondroIntBytesMap.java index 1bb582e01..99a1ebaf1 100644 --- a/source/de/anomic/kelondro/kelondroIntBytesMap.java +++ b/source/de/anomic/kelondro/kelondroIntBytesMap.java @@ -24,6 +24,8 @@ package de.anomic.kelondro; +import java.util.Random; + public class kelondroIntBytesMap extends kelondroRowBufferedSet { public kelondroIntBytesMap(int payloadSize, int initSize) { @@ -65,4 +67,20 @@ public class kelondroIntBytesMap extends kelondroRowBufferedSet { return indexentry.getColBytes(1); } + public static void main(String[] args) { + long start = System.currentTimeMillis(); + kelondroIntBytesMap c = new kelondroIntBytesMap(30, 0); + Random random = new Random(0); + int x; + for (int i = 0; i < 100000; i++) { + //x = random.nextInt(100000); + x = i; + c.putb(x, new byte[30]); + //if (c.getb(x) == null) System.out.println("consistency error at " + i + " with key " + x); + if (i % 10000 == 0) System.out.println(i + " entries. "); + } + System.out.println("RESULT SIZE: " + c.size()); + System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); + } + } diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index f7b983f67..d4bb4e47b 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -832,16 +832,12 @@ public class kelondroRecords { // generate cache entry //byte[] cacheEntry = new byte[headchunksize]; //System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize); - Handle cacheHandle = new Handle(this.handle.index); // store the cache entry boolean upd = false; - upd = (cacheHeaders.putb(cacheHandle.index, headChunk) != null); + upd = (cacheHeaders.putb(this.handle.index, headChunk) != null); if (upd) writeDouble++; else writeUnique++; - // delete the cache entry buffer - //cacheEntry = null; - cacheHandle = null; //System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : "")); //printCache(); } diff --git a/source/de/anomic/kelondro/kelondroRowBufferedSet.java b/source/de/anomic/kelondro/kelondroRowBufferedSet.java index 883fd48be..46a475e48 100644 --- a/source/de/anomic/kelondro/kelondroRowBufferedSet.java +++ b/source/de/anomic/kelondro/kelondroRowBufferedSet.java @@ -24,29 +24,28 @@ package de.anomic.kelondro; -import java.util.HashMap; import java.util.Map; import java.util.Iterator; -import java.util.Random; +import java.util.TreeMap; public class kelondroRowBufferedSet extends kelondroRowSet { private static final long memBlockLimit = 2000000; // do not fill cache further if the amount of available memory is less that this private static final int bufferFlushLimit = 10000; private static final int bufferFlushMinimum = 1000; - private final boolean useRowCollection = true; + private final boolean useRowCollection = false; private kelondroProfile profile; - private HashMap buffer; + private TreeMap buffer; public kelondroRowBufferedSet(kelondroRow rowdef) { super(rowdef); - buffer = new HashMap(); + buffer = new TreeMap(); profile = new kelondroProfile(); } public kelondroRowBufferedSet(kelondroRow rowdef, int objectCount) { super(rowdef, objectCount); - buffer = new HashMap(); + buffer = new TreeMap(); profile = new kelondroProfile(); } @@ -142,15 +141,16 @@ public class kelondroRowBufferedSet extends kelondroRowSet { long handle = profile.startWrite(); byte[] key = newentry.getColBytes(super.sortColumn); kelondroRow.Entry oldentry = null; + Integer intk = new Integer((int) kelondroNaturalOrder.decodeLong(key)); synchronized (buffer) { if (useRowCollection) { - oldentry = (kelondroRow.Entry) buffer.get(new Integer((int) kelondroNaturalOrder.decodeLong(key))); + oldentry = (kelondroRow.Entry) buffer.get(intk); if (oldentry == null) { // try the collection oldentry = super.get(key); if (oldentry == null) { // this was not anywhere - buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry); + buffer.put(intk, newentry); if (((buffer.size() > bufferFlushMinimum) && (kelondroRecords.availableMemory() > memBlockLimit)) || (buffer.size() > bufferFlushLimit)) flush(); } else { @@ -160,10 +160,10 @@ public class kelondroRowBufferedSet extends kelondroRowSet { } else { // the entry is already in buffer // simply replace old entry - buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry); + buffer.put(intk, newentry); } } else { - oldentry = (kelondroRow.Entry) buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry); + oldentry = (kelondroRow.Entry) buffer.put(intk, newentry); } } profile.stopWrite(handle); @@ -174,14 +174,10 @@ public class kelondroRowBufferedSet extends kelondroRowSet { long handle = profile.startDelete(); kelondroRow.Entry oldentry = null; synchronized (buffer) { - if (useRowCollection) { - oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); - if (oldentry == null) { - // try the collection - oldentry = super.removeShift(key); - } - } else { - oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); + oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); + if ((oldentry == null) && (useRowCollection)) { + // try the collection + oldentry = super.removeShift(key); } } profile.stopDelete(handle); @@ -192,14 +188,10 @@ public class kelondroRowBufferedSet extends kelondroRowSet { long handle = profile.startDelete(); kelondroRow.Entry oldentry = null; synchronized (buffer) { - if (useRowCollection) { - oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); - if (oldentry == null) { - // try the collection - return super.removeMarked(key); - } - } else { - oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); + oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); + if ((oldentry == null) && (useRowCollection)) { + // try the collection + return super.removeMarked(key); } } profile.stopDelete(handle); @@ -219,73 +211,4 @@ public class kelondroRowBufferedSet extends kelondroRowSet { return profile; } - public static void main(String[] args) { - String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" }; - kelondroRowBufferedSet c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3})); - c.setOrdering(kelondroNaturalOrder.naturalOrder, 0); - for (int i = 0; i < test.length; i++) c.add(test[i].getBytes()); - for (int i = 0; i < test.length; i++) c.add(test[i].getBytes()); - c.removeMarked("fuenf".getBytes()); - c.shape(); - Iterator i = c.elements(); - String s; - System.out.print("INPUT-ITERATOR: "); - while (i.hasNext()) { - s = new String((byte[]) i.next()).trim(); - System.out.print(s + ", "); - if (s.equals("drei")) i.remove(); - } - System.out.println(""); - System.out.println("INPUT-TOSTRING: " + c.toString()); - c.shape(); - System.out.println("SORTED : " + c.toString()); - c.uniq(); - System.out.println("UNIQ : " + c.toString()); - c.trim(); - System.out.println("TRIM : " + c.toString()); - - // second test - c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3})); - c.setOrdering(kelondroNaturalOrder.naturalOrder, 0); - Random rand = new Random(0); - long start = System.currentTimeMillis(); - long t, d = 0; - String w; - for (long k = 0; k < 60000; k++) { - t = System.currentTimeMillis(); - w = "a" + Long.toString(rand.nextLong()); - c.add(w.getBytes()); - if (k % 10000 == 0) - System.out.println("added " + k + " entries in " + - ((t - start) / 1000) + " seconds, " + - (((t - start) > 1000) ? (k / ((t - start) / 1000)) : k) + - " entries/second, size = " + c.size()); - } - System.out.println("bevore sort: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); - c.shape(); - System.out.println("after sort: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); - c.uniq(); - System.out.println("after uniq: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); - System.out.println("RESULT SIZE: " + c.size()); - System.out.println(); - - // third test - c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3}), 60000); - c.setOrdering(kelondroNaturalOrder.naturalOrder, 0); - rand = new Random(0); - start = System.currentTimeMillis(); - d = 0; - for (long k = 0; k < 60000; k++) { - t = System.currentTimeMillis(); - w = "a" + Long.toString(rand.nextLong()); - if (c.get(w.getBytes()) == null) c.add(w.getBytes()); else d++; - if (k % 10000 == 0) - System.out.println("added " + k + " entries in " + - ((t - start) / 1000) + " seconds, " + - (((t - start) > 1000) ? (k / ((t - start) / 1000)) : k) + - " entries/second, " + d + " double, size = " + c.size() + - ", sum = " + (c.size() + d)); - } - System.out.println("RESULT SIZE: " + c.size()); - } } diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index f97b63878..d188ea95d 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -139,25 +139,6 @@ public class kelondroRowSet extends kelondroRowCollection { } } - /* - private void resolveMarkedRemoved1() { - //long start = System.currentTimeMillis(); - //int c = removeMarker.size(); - Integer idx = new Integer(sortBound); - while (removeMarker.size() > 0) { - idx = (Integer) removeMarker.last(); - removeMarker.remove(idx); - chunkcount--; - if (idx.intValue() < chunkcount) { - super.swap(idx.intValue(), chunkcount, 0); - } - } - if (idx.intValue() < sortBound) sortBound = idx.intValue(); - removeMarker.clear(); - //System.out.println("RESOLVED " + c + " entries in " + (System.currentTimeMillis() - start) + " milliseconds"); - } - */ - private void resolveMarkedRemoved() { if (removeMarker.size() == 0) return; Integer nxt = (Integer) removeMarker.first(); @@ -295,6 +276,7 @@ public class kelondroRowSet extends kelondroRowCollection { } public static void main(String[] args) { + /* String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" }; kelondroRowSet c = new kelondroRowSet(new kelondroRow(new int[]{10, 3})); c.setOrdering(kelondroNaturalOrder.naturalOrder, 0); @@ -318,7 +300,9 @@ public class kelondroRowSet extends kelondroRowCollection { System.out.println("UNIQ : " + c.toString()); c.trim(); System.out.println("TRIM : " + c.toString()); + */ + /* // second test c = new kelondroRowSet(new kelondroRow(new int[]{10, 3})); c.setOrdering(kelondroNaturalOrder.naturalOrder, 0); @@ -362,5 +346,28 @@ public class kelondroRowSet extends kelondroRowCollection { ", sum = " + (c.size() + d)); } System.out.println("RESULT SIZE: " + c.size()); + */ + + long start = System.currentTimeMillis(); + kelondroRowSet c = new kelondroRowSet(new kelondroRow(new int[]{12, 12}), 0); + Random random = new Random(0); + byte[] key; + for (int i = 0; i < 100000; i++) { + key = randomHash(random); + c.put(c.rowdef.newEntry(new byte[][]{key, key})); + if (i % 1000 == 0) System.out.println(i + " entries. "); + } + System.out.println("RESULT SIZE: " + c.size()); + System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); + } + + public static byte[] randomHash(final long r0, final long r1) { + // a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits + // so we construct a generic Hash using two long values + return (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 11).substring(5) + + kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 11).substring(5)).getBytes(); + } + public static byte[] randomHash(Random r) { + return randomHash(r.nextLong(), r.nextLong()); } }