boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more speed test cases: kelondro provides map functions that are more than 20% faster than standard java classes and use less than halve of the memory of java classes:

Browse Source 
just start IndexTest (here with 1000000 test objects)

Performance test: comparing HashMap, TreeMap and kelondroRow
generated 1000000 test data entries 


STANDARD JAVA CLASS MAPS 

sorted map
time   for TreeMap<byte[]> generation: 2110
time   for TreeMap<byte[]> test: 2516, 0 bugs
memory for TreeMap<byte[]>: 29 MB

unsorted map
time   for HashMap<String> generation: 1157
time   for HashMap<String> test: 1516, 0 bugs
memory for HashMap<String>: 61 MB


KELONDRO-ENHANCED MAPS 

sorted map
time   for kelondroMap<byte[]> generation: 1781
time   for kelondroMap<byte[]> test: 2452, 0 bugs
memory for kelondroMap<byte[]>: 15 MB

unsorted map
time   for HashMap<ByteArray> generation: 828
time   for HashMap<ByteArray> test: 953, 0 bugs
memory for HashMap<ByteArray>: 9 MB

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5847 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent b53790abb1
commit 9416f5c26f
2 changed files with 84 additions and 142 deletions
Show Stats Download Patch File Download Diff File

30
source/de/anomic/kelondro/index/IndexTest.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.Random;
import java.util.TreeMap;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.util.ByteArray;
import de.anomic.kelondro.util.MemoryControl;
/**
@ -56,7 +57,7 @@ public class IndexTest {
public static final long mb = 1024 * 1024;
public static void main(String[] args) {
System.out.println("Performance test: comparing HashMap, TreeMap and kelondroRow\n");
System.out.println("Performance test: comparing HashMap, TreeMap and kelondroRow");
if (args.length == 0) {
System.out.println("use one parameter: number of test entries");
System.exit(0);
@ -67,11 +68,14 @@ public class IndexTest {
byte[][] tests = new byte[count][];
Random r = new Random(0);
for (int i = 0; i < count; i++) tests[i] = randomHash(r);
System.out.println("generated " + count + " test data entries \n");
// start
System.out.println("\nSTANDARD JAVA CLASS MAPS \n");
long t1 = System.currentTimeMillis();
// test tree map
System.out.println("sorted map");
Runtime.getRuntime().gc();
long freeStartTree = MemoryControl.free();
TreeMap<byte[], Integer> tm = new TreeMap<byte[], Integer>(Base64Order.enhancedCoder);
@ -89,6 +93,7 @@ public class IndexTest {
System.out.println("memory for TreeMap<byte[]>: " + (freeStartTree - freeEndTree) / mb + " MB\n");
// test hash map
System.out.println("unsorted map");
Runtime.getRuntime().gc();
long freeStartHash = MemoryControl.available();
HashMap<String, Integer> hm = new HashMap<String, Integer>();
@ -105,7 +110,10 @@ public class IndexTest {
System.out.println("time for HashMap<String> test: " + (t5 - t4) + ", " + bugs + " bugs");
System.out.println("memory for HashMap<String>: " + (freeStartHash - freeEndHash) / mb + " MB\n");
System.out.println("\nKELONDRO-ENHANCED MAPS \n");
// test kelondro index
System.out.println("sorted map");
Runtime.getRuntime().gc();
long freeStartKelondro = MemoryControl.available();
IntegerHandleIndex ii = new IntegerHandleIndex(12, Base64Order.enhancedCoder, count, count);
@ -123,6 +131,24 @@ public class IndexTest {
System.out.println("time for kelondroMap<byte[]> test: " + (t7 - t6) + ", " + bugs + " bugs");
System.out.println("memory for kelondroMap<byte[]>: " + (freeStartKelondro - freeEndKelondro) / mb + " MB\n");
// test ByteArray
System.out.println("unsorted map");
Runtime.getRuntime().gc();
long freeStartBA = MemoryControl.available();
HashMap<ByteArray, Integer> bm = new HashMap<ByteArray, Integer>();
for (int i = 0; i < count; i++) bm.put(new ByteArray(tests[i]), 1);
long t8 = System.currentTimeMillis();
System.out.println("time for HashMap<ByteArray> generation: " + (t8 - t7));
bugs = 0;
for (int i = 0; i < count; i++) if (bm.get(new ByteArray(tests[i])) == null) bugs++;
Runtime.getRuntime().gc();
long freeEndBA = MemoryControl.available();
bm.clear(); bm = null;
long t9 = System.currentTimeMillis();
System.out.println("time for HashMap<ByteArray> test: " + (t9 - t8) + ", " + bugs + " bugs");
System.out.println("memory for HashMap<ByteArray>: " + (freeStartBA - freeEndBA) / mb + " MB\n");
System.exit(0);
}
}

196
source/de/anomic/kelondro/util/ByteArray.java
Unescape View File

@ -1,9 +1,7 @@
//kelondrobyteArray.java
// ByteArray.java
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 30.03.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
@ -26,156 +24,40 @@
package de.anomic.kelondro.util;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import de.anomic.kelondro.io.RandomAccessInterface;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.NaturalOrder;
// this class is a experimental replacement of byte[]. It should be used
// if frequent System.arraycopy usage is common for byte[] data types
// when replaced by this class, all copies share the same byte[] but can
// access a different part of the byte array
/**
* this class is a experimental replacement of byte[].
* It can be used if a byte[] shall be stored within a HashMap or HashSet
* which is faster than TreeMap but is generally not possible because storing
* a byte[] in a Hashtable does not work because the hash computation does not
* work for byte[]. This class extends byte[] with a cached hashing function,
* so it can be used in hashtables.
*/
public class ByteArray {
private byte[] buffer;
private int offset;
private int length;
private int hash;
public ByteArray(final int initLength) {
this.buffer = new byte[initLength];
this.length = 0;
this.offset = 0;
}
public ByteArray(final byte[] bb) {
this.buffer = bb;
this.length = bb.length;
this.offset = 0;
}
public ByteArray(final byte[] bb, final int offset, final int length) {
this.buffer = bb;
this.length = length;
this.offset = offset;
}
public ByteArray(final ByteArray ba, final int offset, final int length) {
this.buffer = ba.buffer;
this.length = length;
this.offset = ba.offset + offset;
this.hash = 0;
}
public void ensureSize(final int needed) {
if (buffer.length - offset >= needed) return;
byte[] newbuffer = new byte[needed];
System.arraycopy(buffer, offset, newbuffer, 0, length);
buffer = newbuffer;
offset = 0;
}
public void trim(final int needed) {
if (buffer.length - offset < needed) return;
if (buffer.length - offset == length) return;
byte[] newbuffer = new byte[needed];
System.arraycopy(buffer, offset, newbuffer, 0, length);
buffer = newbuffer;
offset = 0;
}
public final void removeShift(final int pos, final int dist, final int upBound) {
assert (pos + dist >= 0) : "pos = " + pos + ", dist = " + dist;
assert (pos >= 0) : "pos = " + pos;
assert (this.offset + upBound <= buffer.length) : "upBound = " + upBound + ", buffer.length = " + buffer.length;
assert (this.offset + upBound - dist <= buffer.length) : "dist = " + dist + ", upBound = " + upBound + ", buffer.length = " + buffer.length;
System.arraycopy(buffer, this.offset + pos + dist, buffer, this.offset + pos, upBound - pos - dist);
}
public final void swap(final int i, final int j, final int size) {
if (this.offset + this.length + size <= buffer.length) {
// there is space in the chunkcache that we can use as buffer
System.arraycopy(buffer, this.offset + size * i, buffer, buffer.length - size, size);
System.arraycopy(buffer, this.offset + size * j, buffer, this.offset + size * i, size);
System.arraycopy(buffer, buffer.length - size, buffer, this.offset + size * j, size);
} else {
// allocate a chunk to use as buffer
final byte[] a = new byte[size];
System.arraycopy(buffer, this.offset + size * i, a, 0, size);
System.arraycopy(buffer, this.offset + size * j, buffer, this.offset + size * i, size);
System.arraycopy(a, 0, buffer, this.offset + size * j, size);
}
}
public void clear() {
length = 0;
offset = 0;
}
public int length() {
return length;
return buffer.length;
}
public byte[] asBytes() {
final byte[] tmp = new byte[length];
System.arraycopy(buffer, offset, tmp, 0, length);
return tmp;
return this.buffer;
}
public byte readByte(final int pos) {
return buffer[this.offset + pos];
}
public long readLongB64e(final int pos, final int length) {
return Base64Order.enhancedCoder.decodeLong(buffer, this.offset + pos, length);
}
public long readLongB256(final int pos, final int length) {
return NaturalOrder.decodeLong(buffer, this.offset + pos, length);
}
public byte[] readBytes(final int from_pos, final int length) {
final byte[] buf = new byte[length];
System.arraycopy(buffer, this.offset + from_pos, buf, 0, length);
return buf;
}
public String readString(final int from_pos, final int length) {
try {
return new String(buffer, this.offset + from_pos, length, "UTF-8");
} catch (final UnsupportedEncodingException e) {
return "";
}
}
public String readString(final int from_pos, final int length, final String encoding) {
try {
return new String(buffer, this.offset + from_pos, length, encoding);
} catch (final UnsupportedEncodingException e) {
return "";
}
}
public void readToRA(final int from_pos, final RandomAccessInterface to_file, final int len) throws IOException {
to_file.write(this.buffer, from_pos, len);
}
public void write(final int to_position, final byte b) {
buffer[this.offset + to_position] = b;
}
public void write(final int to_position, final byte[] from_array, final int from_offset, final int from_length) {
System.arraycopy(from_array, from_offset, this.buffer, this.offset + to_position, from_length);
}
public void write(final int to_position, final ByteArray from_array) {
System.arraycopy(from_array.buffer, from_array.offset, this.buffer, this.offset + to_position, from_array.length);
}
public void write(final int to_position, final ByteArray from_array, final int from_offset, final int from_length) {
System.arraycopy(from_array.buffer, from_array.offset + from_offset, this.buffer, this.offset + to_position, from_length);
return buffer[pos];
}
public static boolean equals(final byte[] buffer, final byte[] pattern) {
@ -186,17 +68,51 @@ public class ByteArray {
for (int i = 0; i < pattern.length; i++) if (buffer[i] != pattern[i]) return false;
return true;
}
public void reset() {
this.length = 0;
this.offset = 0;
}
public int compareTo(final ByteArray b, final ByteOrder order) {
return order.compare(this.buffer, this.offset, this.length, b.buffer, b.offset, b.length);
return order.compare(this.buffer, 0, this.buffer.length, b.buffer, 0, b.buffer.length);
}
public int compareTo(final int aoffset, final int alength, final ByteArray b, final int boffset, final int blength, final ByteOrder order) {
return order.compare(this.buffer, this.offset + aoffset, alength, b.buffer, b.offset + boffset, blength);
return order.compare(this.buffer, aoffset, alength, b.buffer, boffset, blength);
}
public int hashCode() {
if (this.hash != 0) return this.hash;
int l = this.buffer.length;
int h = 0;
while (--l >= 0) h = 31*h + buffer[l];
this.hash = h;
return h;
}
public boolean equals(Object other) {
ByteArray b = (ByteArray) other;
if (buffer == null && b == null) return true;
if (buffer == null || b == null) return false;
if (this.buffer.length != b.buffer.length) return false;
int l = this.buffer.length;
while (--l >= 0) if (this.buffer[l] != b.buffer[l]) return false;
return true;
}
public static void main(String[] args) {
ByteArray a0 = new ByteArray("abc".getBytes());
ByteArray a1 = new ByteArray("abc".getBytes());
ByteArray b = new ByteArray("bbb".getBytes());
System.out.println("a0 " + ((a0.equals(a1)) ? "=" : "!=") + " a1");
System.out.println("a0 " + ((a0.equals(b)) ? "=" : "!=") + " b");
HashMap<ByteArray, Integer> map = new HashMap<ByteArray, Integer>();
map.put(a0, 1);
//map.put(a1, 1);
map.put(b, 2);
System.out.println("map.size() = " + map.size());
System.out.println("hashCode(a0) = " + a0.hashCode());
System.out.println("hashCode(a1) = " + a1.hashCode());
System.out.println("hashCode(b) = " + b.hashCode());
System.out.println("a0 " + ((map.containsKey(a0)) ? "in" : "not in") + " map");
System.out.println("a1 " + ((map.containsKey(a1)) ? "in" : "not in") + " map");
System.out.println("b " + ((map.containsKey(b)) ? "in" : "not in") + " map");
}
}

Write Preview
Loading…
Cancel
Save