boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

just changed testing code

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2259 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 068d82c4c2
commit f9b9d085c4
4 changed files with 63 additions and 119 deletions
Show Stats Download Patch File Download Diff File

18
source/de/anomic/kelondro/kelondroIntBytesMap.java
Unescape View File

@ -24,6 +24,8 @@
package de.anomic.kelondro;
import java.util.Random;
public class kelondroIntBytesMap extends kelondroRowBufferedSet {
public kelondroIntBytesMap(int payloadSize, int initSize) {
@ -65,4 +67,20 @@ public class kelondroIntBytesMap extends kelondroRowBufferedSet {
return indexentry.getColBytes(1);
}
public static void main(String[] args) {
long start = System.currentTimeMillis();
kelondroIntBytesMap c = new kelondroIntBytesMap(30, 0);
Random random = new Random(0);
int x;
for (int i = 0; i < 100000; i++) {
//x = random.nextInt(100000);
x = i;
c.putb(x, new byte[30]);
//if (c.getb(x) == null) System.out.println("consistency error at " + i + " with key " + x);
if (i % 10000 == 0) System.out.println(i + " entries. ");
}
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
}
}

6
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -832,16 +832,12 @@ public class kelondroRecords {
// generate cache entry
//byte[] cacheEntry = new byte[headchunksize];
//System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize);
Handle cacheHandle = new Handle(this.handle.index);
// store the cache entry
boolean upd = false;
upd = (cacheHeaders.putb(cacheHandle.index, headChunk) != null);
upd = (cacheHeaders.putb(this.handle.index, headChunk) != null);
if (upd) writeDouble++; else writeUnique++;
// delete the cache entry buffer
//cacheEntry = null;
cacheHandle = null;
//System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : ""));
//printCache();
}

101
source/de/anomic/kelondro/kelondroRowBufferedSet.java
Unescape View File

@ -24,29 +24,28 @@
package de.anomic.kelondro;
import java.util.HashMap;
import java.util.Map;
import java.util.Iterator;
import java.util.Random;
import java.util.TreeMap;
public class kelondroRowBufferedSet extends kelondroRowSet {
private static final long memBlockLimit = 2000000; // do not fill cache further if the amount of available memory is less that this
private static final int bufferFlushLimit = 10000;
private static final int bufferFlushMinimum = 1000;
private final boolean useRowCollection = true;
private final boolean useRowCollection = false;
private kelondroProfile profile;
private HashMap buffer;
private TreeMap buffer;
public kelondroRowBufferedSet(kelondroRow rowdef) {
super(rowdef);
buffer = new HashMap();
buffer = new TreeMap();
profile = new kelondroProfile();
}
public kelondroRowBufferedSet(kelondroRow rowdef, int objectCount) {
super(rowdef, objectCount);
buffer = new HashMap();
buffer = new TreeMap();
profile = new kelondroProfile();
}
@ -142,15 +141,16 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
long handle = profile.startWrite();
byte[] key = newentry.getColBytes(super.sortColumn);
kelondroRow.Entry oldentry = null;
Integer intk = new Integer((int) kelondroNaturalOrder.decodeLong(key));
synchronized (buffer) {
if (useRowCollection) {
oldentry = (kelondroRow.Entry) buffer.get(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
oldentry = (kelondroRow.Entry) buffer.get(intk);
if (oldentry == null) {
// try the collection
oldentry = super.get(key);
if (oldentry == null) {
// this was not anywhere
buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
buffer.put(intk, newentry);
if (((buffer.size() > bufferFlushMinimum) && (kelondroRecords.availableMemory() > memBlockLimit)) ||
(buffer.size() > bufferFlushLimit)) flush();
} else {
@ -160,10 +160,10 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
} else {
// the entry is already in buffer
// simply replace old entry
buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
buffer.put(intk, newentry);
}
} else {
oldentry = (kelondroRow.Entry) buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
oldentry = (kelondroRow.Entry) buffer.put(intk, newentry);
}
}
profile.stopWrite(handle);
@ -174,15 +174,11 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
long handle = profile.startDelete();
kelondroRow.Entry oldentry = null;
synchronized (buffer) {
if (useRowCollection) {
oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
if (oldentry == null) {
if ((oldentry == null) && (useRowCollection)) {
// try the collection
oldentry = super.removeShift(key);
}
} else {
oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
}
}
profile.stopDelete(handle);
return oldentry;
@ -192,15 +188,11 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
long handle = profile.startDelete();
kelondroRow.Entry oldentry = null;
synchronized (buffer) {
if (useRowCollection) {
oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
if (oldentry == null) {
if ((oldentry == null) && (useRowCollection)) {
// try the collection
return super.removeMarked(key);
}
} else {
oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
}
}
profile.stopDelete(handle);
return oldentry;
@ -219,73 +211,4 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
return profile;
}
public static void main(String[] args) {
String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" };
kelondroRowBufferedSet c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3}));
c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
for (int i = 0; i < test.length; i++) c.add(test[i].getBytes());
for (int i = 0; i < test.length; i++) c.add(test[i].getBytes());
c.removeMarked("fuenf".getBytes());
c.shape();
Iterator i = c.elements();
String s;
System.out.print("INPUT-ITERATOR: ");
while (i.hasNext()) {
s = new String((byte[]) i.next()).trim();
System.out.print(s + ", ");
if (s.equals("drei")) i.remove();
}
System.out.println("");
System.out.println("INPUT-TOSTRING: " + c.toString());
c.shape();
System.out.println("SORTED : " + c.toString());
c.uniq();
System.out.println("UNIQ : " + c.toString());
c.trim();
System.out.println("TRIM : " + c.toString());
// second test
c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3}));
c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
Random rand = new Random(0);
long start = System.currentTimeMillis();
long t, d = 0;
String w;
for (long k = 0; k < 60000; k++) {
t = System.currentTimeMillis();
w = "a" + Long.toString(rand.nextLong());
c.add(w.getBytes());
if (k % 10000 == 0)
System.out.println("added " + k + " entries in " +
((t - start) / 1000) + " seconds, " +
(((t - start) > 1000) ? (k / ((t - start) / 1000)) : k) +
" entries/second, size = " + c.size());
}
System.out.println("bevore sort: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
c.shape();
System.out.println("after sort: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
c.uniq();
System.out.println("after uniq: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
System.out.println("RESULT SIZE: " + c.size());
System.out.println();
// third test
c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3}), 60000);
c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
rand = new Random(0);
start = System.currentTimeMillis();
d = 0;
for (long k = 0; k < 60000; k++) {
t = System.currentTimeMillis();
w = "a" + Long.toString(rand.nextLong());
if (c.get(w.getBytes()) == null) c.add(w.getBytes()); else d++;
if (k % 10000 == 0)
System.out.println("added " + k + " entries in " +
((t - start) / 1000) + " seconds, " +
(((t - start) > 1000) ? (k / ((t - start) / 1000)) : k) +
" entries/second, " + d + " double, size = " + c.size() +
", sum = " + (c.size() + d));
}
System.out.println("RESULT SIZE: " + c.size());
}
}

45
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -139,25 +139,6 @@ public class kelondroRowSet extends kelondroRowCollection {
}
}
/*
private void resolveMarkedRemoved1() {
//long start = System.currentTimeMillis();
//int c = removeMarker.size();
Integer idx = new Integer(sortBound);
while (removeMarker.size() > 0) {
idx = (Integer) removeMarker.last();
removeMarker.remove(idx);
chunkcount--;
if (idx.intValue() < chunkcount) {
super.swap(idx.intValue(), chunkcount, 0);
}
}
if (idx.intValue() < sortBound) sortBound = idx.intValue();
removeMarker.clear();
//System.out.println("RESOLVED " + c + " entries in " + (System.currentTimeMillis() - start) + " milliseconds");
}
*/
private void resolveMarkedRemoved() {
if (removeMarker.size() == 0) return;
Integer nxt = (Integer) removeMarker.first();
@ -295,6 +276,7 @@ public class kelondroRowSet extends kelondroRowCollection {
}
public static void main(String[] args) {
/*
String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" };
kelondroRowSet c = new kelondroRowSet(new kelondroRow(new int[]{10, 3}));
c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
@ -318,7 +300,9 @@ public class kelondroRowSet extends kelondroRowCollection {
System.out.println("UNIQ : " + c.toString());
c.trim();
System.out.println("TRIM : " + c.toString());
*/
/*
// second test
c = new kelondroRowSet(new kelondroRow(new int[]{10, 3}));
c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
@ -362,5 +346,28 @@ public class kelondroRowSet extends kelondroRowCollection {
", sum = " + (c.size() + d));
}
System.out.println("RESULT SIZE: " + c.size());
*/
long start = System.currentTimeMillis();
kelondroRowSet c = new kelondroRowSet(new kelondroRow(new int[]{12, 12}), 0);
Random random = new Random(0);
byte[] key;
for (int i = 0; i < 100000; i++) {
key = randomHash(random);
c.put(c.rowdef.newEntry(new byte[][]{key, key}));
if (i % 1000 == 0) System.out.println(i + " entries. ");
}
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
}
public static byte[] randomHash(final long r0, final long r1) {
// a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
// so we construct a generic Hash using two long values
return (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 11).substring(5) +
kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 11).substring(5)).getBytes();
}
public static byte[] randomHash(Random r) {
return randomHash(r.nextLong(), r.nextLong());
}
}

Write Preview
Loading…
Cancel
Save