boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added some profiling to kelondro caching classes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2239 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent dd560e4b2f
commit 41c4641612
8 changed files with 252 additions and 60 deletions
Show Stats Download Patch File Download Diff File

16
source/dbtest.java
Unescape View File

@ -15,6 +15,7 @@ import java.util.Iterator;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroProfile;
import de.anomic.kelondro.kelondroSplittedTree;
import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroRow;
@ -177,6 +178,7 @@ public class dbtest {
}
if (dbe.equals("kelondroSplittedTree")) {
File tablepath = new File(tablename).getParentFile();
tablename = new File(tablename).getName();
table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder,
buffer,
8,
@ -211,11 +213,21 @@ public class dbtest {
long randomstart = Long.parseLong(args[4]);
Random random = new Random(randomstart);
byte[] key;
kelondroProfile ioProfileAcc = new kelondroProfile();
kelondroProfile cacheProfileAcc = new kelondroProfile();
kelondroProfile[] profiles;
for (int i = 0; i < count; i++) {
key = randomHash(random);
table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2}));
if (i % 500 == 0) {
if (i % 1000 == 0) {
System.out.println(i + " entries. " + ((table instanceof kelondroTree) ? ((kelondroTree) table).cacheNodeStatusString() : ""));
if (table instanceof kelondroTree) {
profiles = ((kelondroTree) table).profiles();
System.out.println("Cache Delta: " + kelondroProfile.delta(profiles[0], cacheProfileAcc).toString());
System.out.println("IO Delta: " + kelondroProfile.delta(profiles[1], ioProfileAcc).toString());
cacheProfileAcc = (kelondroProfile) profiles[0].clone();
ioProfileAcc = (kelondroProfile) profiles[1].clone();
}
}
}
}
@ -250,7 +262,7 @@ public class dbtest {
entry = table.get(key);
if (entry == null) System.out.println("missing value for entry " + new String(key)); else
if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)));
if (i % 500 == 0) {
if (i % 1000 == 0) {
System.out.println(i + " entries processed so far.");
}
}

18
source/de/anomic/kelondro/kelondroAbstractIOChunks.java
Unescape View File

@ -51,6 +51,12 @@ public abstract class kelondroAbstractIOChunks {
return name;
}
// profiling support
protected kelondroProfile profile = new kelondroProfile();
public kelondroProfile profile() {
return profile;
}
// pseudo-native methods:
abstract public int read(long pos, byte[] b, int off, int len) throws IOException;
abstract public void write(long pos, byte[] b, int off, int len) throws IOException;
@ -59,6 +65,7 @@ public abstract class kelondroAbstractIOChunks {
// derived methods:
public void readFully(long pos, byte[] b, int off, int len) throws IOException {
long handle = profile.startRead();
if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len);
if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len);
while (len > 0) {
@ -68,6 +75,7 @@ public abstract class kelondroAbstractIOChunks {
off += r;
len -= r;
}
profile.stopRead(handle);
}
public byte readByte(long pos) throws IOException {
@ -77,7 +85,9 @@ public abstract class kelondroAbstractIOChunks {
}
public void writeByte(long pos, final int v) throws IOException {
long handle = profile.startWrite();
this.write(pos, new byte[]{(byte) (v & 0xFF)});
profile.stopWrite(handle);
}
public short readShort(long pos) throws IOException {
@ -87,7 +97,9 @@ public abstract class kelondroAbstractIOChunks {
}
public void writeShort(long pos, final int v) throws IOException {
long handle = profile.startWrite();
this.write(pos, new byte[]{(byte) ((v >>> 8) & 0xFF), (byte) ((v >>> 0) & 0xFF)});
profile.stopWrite(handle);
}
public int readInt(long pos) throws IOException {
@ -97,12 +109,14 @@ public abstract class kelondroAbstractIOChunks {
}
public void writeInt(long pos, final int v) throws IOException {
long handle = profile.startWrite();
this.write(pos, new byte[]{
(byte) ((v >>> 24) & 0xFF),
(byte) ((v >>> 16) & 0xFF),
(byte) ((v >>> 8) & 0xFF),
(byte) ((v >>> 0) & 0xFF)
});
profile.stopWrite(handle);
}
public long readLong(long pos) throws IOException {
@ -112,6 +126,7 @@ public abstract class kelondroAbstractIOChunks {
}
public void writeLong(long pos, final long v) throws IOException {
long handle = profile.startWrite();
this.write(pos, new byte[]{
(byte) ((v >>> 56) & 0xFF),
(byte) ((v >>> 48) & 0xFF),
@ -122,10 +137,13 @@ public abstract class kelondroAbstractIOChunks {
(byte) ((v >>> 8) & 0xFF),
(byte) ((v >>> 0) & 0xFF)
});
profile.stopWrite(handle);
}
public void write(long pos, final byte[] b) throws IOException {
long handle = profile.startWrite();
this.write(pos, b, 0, b.length);
profile.stopWrite(handle);
}
}

1
source/de/anomic/kelondro/kelondroIOChunks.java
Unescape View File

@ -70,4 +70,5 @@ public interface kelondroIOChunks {
public void write(long pos, byte[] b) throws IOException;
public kelondroProfile profile();
}

2
source/de/anomic/kelondro/kelondroIntBytesMap.java
Unescape View File

@ -24,7 +24,7 @@
package de.anomic.kelondro;
public class kelondroIntBytesMap extends kelondroRowBufferedSet {
public class kelondroIntBytesMap extends kelondroRowSet {
public kelondroIntBytesMap(int payloadSize, int initSize) {
super(new kelondroRow(new int[]{4, payloadSize}), initSize);

115
source/de/anomic/kelondro/kelondroProfile.java
Unescape View File

@ -0,0 +1,115 @@
// kelondroProfile.java
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 23.06.2006 on http://www.anomic.de
//
// This is a part of the kelondro database,
// which is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
public class kelondroProfile implements Cloneable {
private long accRead;
private long accWrite;
private long accDelete;
public kelondroProfile() {
accRead = 0;
accWrite = 0;
accDelete = 0;
}
public long timeRead() {
return accRead;
}
public long timeWrite() {
return accWrite;
}
public long timeDelete() {
return accDelete;
}
public void timeReset() {
accRead = 0;
accWrite = 0;
accDelete = 0;
}
protected long startRead() {
return System.currentTimeMillis();
}
protected void stopRead(long handle) {
accRead += System.currentTimeMillis() - handle;
}
protected long startWrite() {
return System.currentTimeMillis();
}
protected void stopWrite(long handle) {
accWrite += System.currentTimeMillis() - handle;
}
protected long startDelete() {
return System.currentTimeMillis();
}
protected void stopDelete(long handle) {
accDelete += System.currentTimeMillis() - handle;
}
public Object clone() {
kelondroProfile clone = new kelondroProfile();
clone.accRead = this.accRead;
clone.accWrite = this.accWrite;
clone.accDelete = this.accDelete;
return clone;
}
public String toString() {
return "read=" + accRead + ", write=" + accWrite + ", delete=" + accDelete;
}
public static kelondroProfile consolidate(kelondroProfile[] profiles) {
for (int i = 1; i < profiles.length; i++) consolidate(profiles[0], profiles[i]);
return profiles[0];
}
public static kelondroProfile consolidate(kelondroProfile profile1, kelondroProfile profile2) {
profile1.accRead += profile2.accRead;
profile1.accWrite += profile2.accWrite;
profile1.accDelete += profile2.accDelete;
return profile1;
}
public static kelondroProfile delta(kelondroProfile newer, kelondroProfile older) {
kelondroProfile result = new kelondroProfile();
result.accRead = newer.accRead - older.accRead;
result.accWrite = newer.accWrite - older.accWrite;
result.accDelete = newer.accDelete - older.accDelete;
return result;
}
}

11
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -1371,4 +1371,15 @@ public class kelondroRecords {
}
}
public kelondroProfile[] profiles() {
return new kelondroProfile[]{
(cacheHeaders == null) ? new kelondroProfile() :
kelondroProfile.consolidate(new kelondroProfile[]{
cacheHeaders[0].profile(),
cacheHeaders[1].profile(),
cacheHeaders[2].profile()
}),
entryFile.profile()
};
}
}

78
source/de/anomic/kelondro/kelondroRowBufferedSet.java
Unescape View File

@ -24,7 +24,7 @@
package de.anomic.kelondro;
import java.util.TreeMap;
import java.util.HashMap;
import java.util.Map;
import java.util.Iterator;
import java.util.Random;
@ -32,19 +32,22 @@ import java.util.Random;
public class kelondroRowBufferedSet extends kelondroRowSet {
private static final long memBlockLimit = 2000000; // do not fill cache further if the amount of available memory is less that this
private static final int bufferFlushLimit = 100000;
private static final int bufferFlushLimit = 10000;
private static final int bufferFlushMinimum = 1000;
private final boolean useRowCollection = true;
private TreeMap buffer; // this must be a TreeSet bacause HashMap does not work with byte[]
private kelondroProfile profile;
private HashMap buffer;
public kelondroRowBufferedSet(kelondroRow rowdef) {
super(rowdef);
buffer = new TreeMap(kelondroNaturalOrder.naturalOrder);
buffer = new HashMap();
profile = new kelondroProfile();
}
public kelondroRowBufferedSet(kelondroRow rowdef, int objectCount) {
super(rowdef, objectCount);
buffer = new TreeMap(kelondroNaturalOrder.naturalOrder);
buffer = new HashMap();
profile = new kelondroProfile();
}
private final void flush() {
@ -125,90 +128,97 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
}
public kelondroRow.Entry get(byte[] key) {
long handle = profile.startRead();
kelondroRow.Entry entry = null;
synchronized (buffer) {
if (useRowCollection) {
kelondroRow.Entry entry = (kelondroRow.Entry) buffer.get(key);
if (entry != null) return entry;
return super.get(key);
} else {
return (kelondroRow.Entry) buffer.get(key);
}
entry = (kelondroRow.Entry) buffer.get(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
if ((entry == null) && (useRowCollection)) entry = super.get(key);
}
profile.stopRead(handle);
return entry;
}
public kelondroRow.Entry put(kelondroRow.Entry newentry) {
long handle = profile.startWrite();
byte[] key = newentry.getColBytes(super.sortColumn);
kelondroRow.Entry oldentry = null;
synchronized (buffer) {
if (useRowCollection) {
kelondroRow.Entry oldentry = (kelondroRow.Entry) buffer.get(key);
oldentry = (kelondroRow.Entry) buffer.get(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
if (oldentry == null) {
// try the collection
oldentry = super.get(key);
if (oldentry == null) {
// this was not anywhere
buffer.put(key, newentry);
buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
if (((buffer.size() > bufferFlushMinimum) && (kelondroRecords.availableMemory() > memBlockLimit)) ||
(buffer.size() > bufferFlushLimit)) flush();
return null;
} else {
// replace old entry
super.put(newentry);
return oldentry;
}
} else {
// the entry is already in buffer
// simply replace old entry
buffer.put(key, newentry);
return oldentry;
buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
}
} else {
return (kelondroRow.Entry) buffer.put(key, newentry);
oldentry = (kelondroRow.Entry) buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry);
}
}
profile.stopWrite(handle);
return oldentry;
}
public kelondroRow.Entry removeShift(byte[] a) {
public kelondroRow.Entry removeShift(byte[] key) {
long handle = profile.startDelete();
kelondroRow.Entry oldentry = null;
synchronized (buffer) {
if (useRowCollection) {
kelondroRow.Entry oldentry = (kelondroRow.Entry) buffer.remove(a);
oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
if (oldentry == null) {
// try the collection
return super.removeShift(a);
} else {
// the entry was in buffer
return oldentry;
oldentry = super.removeShift(key);
}
} else {
return (kelondroRow.Entry) buffer.remove(a); // test
oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
}
}
profile.stopDelete(handle);
return oldentry;
}
public kelondroRow.Entry removeMarked(byte[] a) {
public kelondroRow.Entry removeMarked(byte[] key) {
long handle = profile.startDelete();
kelondroRow.Entry oldentry = null;
synchronized (buffer) {
if (useRowCollection) {
kelondroRow.Entry oldentry = (kelondroRow.Entry) buffer.remove(a);
oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
if (oldentry == null) {
// try the collection
return super.removeMarked(a);
} else {
// the entry was in buffer
return oldentry;
return super.removeMarked(key);
}
} else {
return (kelondroRow.Entry) buffer.remove(a); // test
oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key)));
}
}
profile.stopDelete(handle);
return oldentry;
}
public void removeMarkedAll(kelondroRowCollection c) {
// this can be enhanced
long handle = profile.startDelete();
synchronized (buffer) {
flush();
super.removeMarkedAll(c);
}
profile.stopDelete(handle);
}
public kelondroProfile profile() {
return profile;
}
public static void main(String[] args) {
String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" };
kelondroRowBufferedSet c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3}));

71
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -32,17 +32,20 @@ public class kelondroRowSet extends kelondroRowCollection {
private static final int collectionReSortLimit = 90;
private static final int removeMaxSize = 100;
private kelondroProfile profile;
private TreeSet removeMarker;
public kelondroRowSet(kelondroRow rowdef) {
super(rowdef);
this.removeMarker = new TreeSet();
this.profile = new kelondroProfile();
}
public kelondroRowSet(kelondroRow rowdef, int objectCount) {
super(rowdef, objectCount);
this.removeMarker = new TreeSet();
this.profile = new kelondroProfile();
}
public kelondroRow.Entry get(byte[] key) {
@ -50,33 +53,34 @@ public class kelondroRowSet extends kelondroRowCollection {
}
private kelondroRow.Entry get(byte[] key, int astart, int alength) {
long handle = profile.startRead();
kelondroRow.Entry entry = null;
synchronized (chunkcache) {
int index = find(key, astart, alength);
if ((index < 0) || (isMarkedRemoved(index))){
return null;
} else {
return get(index);
}
if ((index >= 0) && (!(isMarkedRemoved(index)))) entry = get(index);
}
profile.stopRead(handle);
return entry;
}
public kelondroRow.Entry put(kelondroRow.Entry entry) {
long handle = profile.startWrite();
int index = -1;
kelondroRow.Entry oldentry = null;
synchronized (chunkcache) {
index = find(entry.bytes(), super.rowdef.colstart[super.sortColumn], super.rowdef.width(super.sortColumn));
if (isMarkedRemoved(index)) {
set(index, entry);
removeMarker.remove(new Integer(index));
return null;
} else if (index < 0) {
add(entry);
return null;
} else {
kelondroRow.Entry oldentry = get(index);
oldentry = get(index);
set(index, entry);
return oldentry;
}
}
profile.stopWrite(handle);
return oldentry;
}
public int size() {
@ -88,26 +92,39 @@ public class kelondroRowSet extends kelondroRowCollection {
}
private kelondroRow.Entry removeMarked(byte[] a, int astart, int alength) {
// the byte[] a may be shorter than the chunksize
if (chunkcount == 0) return null;
kelondroRow.Entry b = null;
long handle = profile.startDelete();
// check if it is contained in chunkcache
kelondroRow.Entry entry = null;
synchronized(chunkcache) {
int p = find(a, astart, alength);
if (p < 0) return null;
b = get(p);
if (p < 0) {
// the entry is not there
profile.stopDelete(handle);
return null;
}
// there is an entry
entry = get(p);
if (p < sortBound) {
removeMarker.add(new Integer(p));
} else {
super.swap(p, --chunkcount, 0);
}
// check case when complete chunkcache is marked as deleted
if (removeMarker.size() == chunkcount) {
this.clear();
removeMarker.clear();
}
}
if (removeMarker.size() == chunkcount) {
chunkcount = 0;
sortBound = 0;
removeMarker.clear();
}
// check if removeMarker is full
if (removeMarker.size() >= removeMaxSize) resolveMarkedRemoved();
return b;
profile.stopDelete(handle);
return entry;
}
private boolean isMarkedRemoved(int index) {
@ -167,27 +184,31 @@ public class kelondroRowSet extends kelondroRowCollection {
private kelondroRow.Entry removeShift(byte[] a, int astart, int alength) {
// the byte[] a may be shorter than the chunksize
if (chunkcount == 0) return null;
kelondroRow.Entry b = null;
long handle = profile.startDelete();
kelondroRow.Entry entry = null;
synchronized(chunkcache) {
int p = find(a, astart, alength);
if (p < 0) return null;
b = get(p);
entry = get(p);
if (p < sortBound) {
removeShift(p);
} else {
super.swap(p, --chunkcount, 0);
}
}
return b;
profile.stopDelete(handle);
return entry;
}
public void removeMarkedAll(kelondroRowCollection c) {
long handle = profile.startDelete();
Iterator i = c.elements();
byte[] b;
while (i.hasNext()) {
b = (byte[]) i.next();
removeMarked(b, 0, b.length);
}
profile.stopDelete(handle);
}
public kelondroOrder getOrdering() {
@ -269,6 +290,10 @@ public class kelondroRowSet extends kelondroRowCollection {
return true;
}
public kelondroProfile profile() {
return profile;
}
public static void main(String[] args) {
String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" };
kelondroRowSet c = new kelondroRowSet(new kelondroRow(new int[]{10, 3}));

Write Preview
Loading…
Cancel
Save