From 9416f5c26f1d784b06556396d8f6fb489ee11a64 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 21 Apr 2009 09:29:08 +0000 Subject: [PATCH] more speed test cases: kelondro provides map functions that are more than 20% faster than standard java classes and use less than halve of the memory of java classes: just start IndexTest (here with 1000000 test objects) Performance test: comparing HashMap, TreeMap and kelondroRow generated 1000000 test data entries STANDARD JAVA CLASS MAPS sorted map time for TreeMap generation: 2110 time for TreeMap test: 2516, 0 bugs memory for TreeMap: 29 MB unsorted map time for HashMap generation: 1157 time for HashMap test: 1516, 0 bugs memory for HashMap: 61 MB KELONDRO-ENHANCED MAPS sorted map time for kelondroMap generation: 1781 time for kelondroMap test: 2452, 0 bugs memory for kelondroMap: 15 MB unsorted map time for HashMap generation: 828 time for HashMap test: 953, 0 bugs memory for HashMap: 9 MB git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5847 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../de/anomic/kelondro/index/IndexTest.java | 30 ++- source/de/anomic/kelondro/util/ByteArray.java | 196 +++++------------- 2 files changed, 84 insertions(+), 142 deletions(-) diff --git a/source/de/anomic/kelondro/index/IndexTest.java b/source/de/anomic/kelondro/index/IndexTest.java index d6f8b4e5f..f664bfa9e 100644 --- a/source/de/anomic/kelondro/index/IndexTest.java +++ b/source/de/anomic/kelondro/index/IndexTest.java @@ -33,6 +33,7 @@ import java.util.Random; import java.util.TreeMap; import de.anomic.kelondro.order.Base64Order; +import de.anomic.kelondro.util.ByteArray; import de.anomic.kelondro.util.MemoryControl; /** @@ -56,7 +57,7 @@ public class IndexTest { public static final long mb = 1024 * 1024; public static void main(String[] args) { - System.out.println("Performance test: comparing HashMap, TreeMap and kelondroRow\n"); + System.out.println("Performance test: comparing HashMap, TreeMap and kelondroRow"); if (args.length == 0) { System.out.println("use one parameter: number of test entries"); System.exit(0); @@ -67,11 +68,14 @@ public class IndexTest { byte[][] tests = new byte[count][]; Random r = new Random(0); for (int i = 0; i < count; i++) tests[i] = randomHash(r); - + System.out.println("generated " + count + " test data entries \n"); + // start + System.out.println("\nSTANDARD JAVA CLASS MAPS \n"); long t1 = System.currentTimeMillis(); // test tree map + System.out.println("sorted map"); Runtime.getRuntime().gc(); long freeStartTree = MemoryControl.free(); TreeMap tm = new TreeMap(Base64Order.enhancedCoder); @@ -89,6 +93,7 @@ public class IndexTest { System.out.println("memory for TreeMap: " + (freeStartTree - freeEndTree) / mb + " MB\n"); // test hash map + System.out.println("unsorted map"); Runtime.getRuntime().gc(); long freeStartHash = MemoryControl.available(); HashMap hm = new HashMap(); @@ -105,7 +110,10 @@ public class IndexTest { System.out.println("time for HashMap test: " + (t5 - t4) + ", " + bugs + " bugs"); System.out.println("memory for HashMap: " + (freeStartHash - freeEndHash) / mb + " MB\n"); + System.out.println("\nKELONDRO-ENHANCED MAPS \n"); + // test kelondro index + System.out.println("sorted map"); Runtime.getRuntime().gc(); long freeStartKelondro = MemoryControl.available(); IntegerHandleIndex ii = new IntegerHandleIndex(12, Base64Order.enhancedCoder, count, count); @@ -123,6 +131,24 @@ public class IndexTest { System.out.println("time for kelondroMap test: " + (t7 - t6) + ", " + bugs + " bugs"); System.out.println("memory for kelondroMap: " + (freeStartKelondro - freeEndKelondro) / mb + " MB\n"); + // test ByteArray + System.out.println("unsorted map"); + Runtime.getRuntime().gc(); + long freeStartBA = MemoryControl.available(); + HashMap bm = new HashMap(); + for (int i = 0; i < count; i++) bm.put(new ByteArray(tests[i]), 1); + long t8 = System.currentTimeMillis(); + System.out.println("time for HashMap generation: " + (t8 - t7)); + + bugs = 0; + for (int i = 0; i < count; i++) if (bm.get(new ByteArray(tests[i])) == null) bugs++; + Runtime.getRuntime().gc(); + long freeEndBA = MemoryControl.available(); + bm.clear(); bm = null; + long t9 = System.currentTimeMillis(); + System.out.println("time for HashMap test: " + (t9 - t8) + ", " + bugs + " bugs"); + System.out.println("memory for HashMap: " + (freeStartBA - freeEndBA) / mb + " MB\n"); + System.exit(0); } } diff --git a/source/de/anomic/kelondro/util/ByteArray.java b/source/de/anomic/kelondro/util/ByteArray.java index 136700146..104802e0a 100644 --- a/source/de/anomic/kelondro/util/ByteArray.java +++ b/source/de/anomic/kelondro/util/ByteArray.java @@ -1,9 +1,7 @@ -//kelondrobyteArray.java +// ByteArray.java // (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // first published 30.03.2007 on http://yacy.net // -// This is a part of YaCy, a peer-to-peer based web search engine -// // $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ // $LastChangedRevision: 1986 $ // $LastChangedBy: orbiter $ @@ -26,156 +24,40 @@ package de.anomic.kelondro.util; -import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.util.HashMap; -import de.anomic.kelondro.io.RandomAccessInterface; -import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.ByteOrder; -import de.anomic.kelondro.order.NaturalOrder; -// this class is a experimental replacement of byte[]. It should be used -// if frequent System.arraycopy usage is common for byte[] data types -// when replaced by this class, all copies share the same byte[] but can -// access a different part of the byte array +/** + * this class is a experimental replacement of byte[]. + * It can be used if a byte[] shall be stored within a HashMap or HashSet + * which is faster than TreeMap but is generally not possible because storing + * a byte[] in a Hashtable does not work because the hash computation does not + * work for byte[]. This class extends byte[] with a cached hashing function, + * so it can be used in hashtables. + */ public class ByteArray { private byte[] buffer; - private int offset; - private int length; + private int hash; - public ByteArray(final int initLength) { - this.buffer = new byte[initLength]; - this.length = 0; - this.offset = 0; - } public ByteArray(final byte[] bb) { this.buffer = bb; - this.length = bb.length; - this.offset = 0; - } - - public ByteArray(final byte[] bb, final int offset, final int length) { - this.buffer = bb; - this.length = length; - this.offset = offset; - } - - public ByteArray(final ByteArray ba, final int offset, final int length) { - this.buffer = ba.buffer; - this.length = length; - this.offset = ba.offset + offset; + this.hash = 0; } - public void ensureSize(final int needed) { - if (buffer.length - offset >= needed) return; - byte[] newbuffer = new byte[needed]; - System.arraycopy(buffer, offset, newbuffer, 0, length); - buffer = newbuffer; - offset = 0; - } - - public void trim(final int needed) { - if (buffer.length - offset < needed) return; - if (buffer.length - offset == length) return; - byte[] newbuffer = new byte[needed]; - System.arraycopy(buffer, offset, newbuffer, 0, length); - buffer = newbuffer; - offset = 0; - } - - public final void removeShift(final int pos, final int dist, final int upBound) { - assert (pos + dist >= 0) : "pos = " + pos + ", dist = " + dist; - assert (pos >= 0) : "pos = " + pos; - assert (this.offset + upBound <= buffer.length) : "upBound = " + upBound + ", buffer.length = " + buffer.length; - assert (this.offset + upBound - dist <= buffer.length) : "dist = " + dist + ", upBound = " + upBound + ", buffer.length = " + buffer.length; - System.arraycopy(buffer, this.offset + pos + dist, buffer, this.offset + pos, upBound - pos - dist); - } - - public final void swap(final int i, final int j, final int size) { - if (this.offset + this.length + size <= buffer.length) { - // there is space in the chunkcache that we can use as buffer - System.arraycopy(buffer, this.offset + size * i, buffer, buffer.length - size, size); - System.arraycopy(buffer, this.offset + size * j, buffer, this.offset + size * i, size); - System.arraycopy(buffer, buffer.length - size, buffer, this.offset + size * j, size); - } else { - // allocate a chunk to use as buffer - final byte[] a = new byte[size]; - System.arraycopy(buffer, this.offset + size * i, a, 0, size); - System.arraycopy(buffer, this.offset + size * j, buffer, this.offset + size * i, size); - System.arraycopy(a, 0, buffer, this.offset + size * j, size); - } - } - - public void clear() { - length = 0; - offset = 0; - } - public int length() { - return length; + return buffer.length; } public byte[] asBytes() { - final byte[] tmp = new byte[length]; - System.arraycopy(buffer, offset, tmp, 0, length); - return tmp; + return this.buffer; } public byte readByte(final int pos) { - return buffer[this.offset + pos]; - } - - public long readLongB64e(final int pos, final int length) { - return Base64Order.enhancedCoder.decodeLong(buffer, this.offset + pos, length); - } - - public long readLongB256(final int pos, final int length) { - return NaturalOrder.decodeLong(buffer, this.offset + pos, length); - } - - public byte[] readBytes(final int from_pos, final int length) { - final byte[] buf = new byte[length]; - System.arraycopy(buffer, this.offset + from_pos, buf, 0, length); - return buf; - } - - public String readString(final int from_pos, final int length) { - try { - return new String(buffer, this.offset + from_pos, length, "UTF-8"); - } catch (final UnsupportedEncodingException e) { - return ""; - } - } - - public String readString(final int from_pos, final int length, final String encoding) { - try { - return new String(buffer, this.offset + from_pos, length, encoding); - } catch (final UnsupportedEncodingException e) { - return ""; - } - } - - public void readToRA(final int from_pos, final RandomAccessInterface to_file, final int len) throws IOException { - to_file.write(this.buffer, from_pos, len); - } - - public void write(final int to_position, final byte b) { - buffer[this.offset + to_position] = b; - } - - public void write(final int to_position, final byte[] from_array, final int from_offset, final int from_length) { - System.arraycopy(from_array, from_offset, this.buffer, this.offset + to_position, from_length); - } - - public void write(final int to_position, final ByteArray from_array) { - System.arraycopy(from_array.buffer, from_array.offset, this.buffer, this.offset + to_position, from_array.length); - } - - public void write(final int to_position, final ByteArray from_array, final int from_offset, final int from_length) { - System.arraycopy(from_array.buffer, from_array.offset + from_offset, this.buffer, this.offset + to_position, from_length); + return buffer[pos]; } public static boolean equals(final byte[] buffer, final byte[] pattern) { @@ -186,17 +68,51 @@ public class ByteArray { for (int i = 0; i < pattern.length; i++) if (buffer[i] != pattern[i]) return false; return true; } - - public void reset() { - this.length = 0; - this.offset = 0; - } public int compareTo(final ByteArray b, final ByteOrder order) { - return order.compare(this.buffer, this.offset, this.length, b.buffer, b.offset, b.length); + return order.compare(this.buffer, 0, this.buffer.length, b.buffer, 0, b.buffer.length); } public int compareTo(final int aoffset, final int alength, final ByteArray b, final int boffset, final int blength, final ByteOrder order) { - return order.compare(this.buffer, this.offset + aoffset, alength, b.buffer, b.offset + boffset, blength); + return order.compare(this.buffer, aoffset, alength, b.buffer, boffset, blength); + } + + public int hashCode() { + if (this.hash != 0) return this.hash; + int l = this.buffer.length; + int h = 0; + while (--l >= 0) h = 31*h + buffer[l]; + + this.hash = h; + return h; + } + + public boolean equals(Object other) { + ByteArray b = (ByteArray) other; + if (buffer == null && b == null) return true; + if (buffer == null || b == null) return false; + if (this.buffer.length != b.buffer.length) return false; + int l = this.buffer.length; + while (--l >= 0) if (this.buffer[l] != b.buffer[l]) return false; + return true; + } + + public static void main(String[] args) { + ByteArray a0 = new ByteArray("abc".getBytes()); + ByteArray a1 = new ByteArray("abc".getBytes()); + ByteArray b = new ByteArray("bbb".getBytes()); + System.out.println("a0 " + ((a0.equals(a1)) ? "=" : "!=") + " a1"); + System.out.println("a0 " + ((a0.equals(b)) ? "=" : "!=") + " b"); + HashMap map = new HashMap(); + map.put(a0, 1); + //map.put(a1, 1); + map.put(b, 2); + System.out.println("map.size() = " + map.size()); + System.out.println("hashCode(a0) = " + a0.hashCode()); + System.out.println("hashCode(a1) = " + a1.hashCode()); + System.out.println("hashCode(b) = " + b.hashCode()); + System.out.println("a0 " + ((map.containsKey(a0)) ? "in" : "not in") + " map"); + System.out.println("a1 " + ((map.containsKey(a1)) ? "in" : "not in") + " map"); + System.out.println("b " + ((map.containsKey(b)) ? "in" : "not in") + " map"); } }