From 41c46416122612e324c3195879283aa11b0eb816 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 23 Jun 2006 12:49:42 +0000 Subject: [PATCH] added some profiling to kelondro caching classes git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2239 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/dbtest.java | 16 ++- .../kelondro/kelondroAbstractIOChunks.java | 18 +++ .../de/anomic/kelondro/kelondroIOChunks.java | 1 + .../anomic/kelondro/kelondroIntBytesMap.java | 2 +- .../de/anomic/kelondro/kelondroProfile.java | 115 ++++++++++++++++++ .../de/anomic/kelondro/kelondroRecords.java | 11 ++ .../kelondro/kelondroRowBufferedSet.java | 78 ++++++------ source/de/anomic/kelondro/kelondroRowSet.java | 71 +++++++---- 8 files changed, 252 insertions(+), 60 deletions(-) create mode 100644 source/de/anomic/kelondro/kelondroProfile.java diff --git a/source/dbtest.java b/source/dbtest.java index 92c33801b..20db3908f 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -15,6 +15,7 @@ import java.util.Iterator; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroFlexTable; import de.anomic.kelondro.kelondroIndex; +import de.anomic.kelondro.kelondroProfile; import de.anomic.kelondro.kelondroSplittedTree; import de.anomic.kelondro.kelondroTree; import de.anomic.kelondro.kelondroRow; @@ -177,6 +178,7 @@ public class dbtest { } if (dbe.equals("kelondroSplittedTree")) { File tablepath = new File(tablename).getParentFile(); + tablename = new File(tablename).getName(); table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder, buffer, 8, @@ -211,11 +213,21 @@ public class dbtest { long randomstart = Long.parseLong(args[4]); Random random = new Random(randomstart); byte[] key; + kelondroProfile ioProfileAcc = new kelondroProfile(); + kelondroProfile cacheProfileAcc = new kelondroProfile(); + kelondroProfile[] profiles; for (int i = 0; i < count; i++) { key = randomHash(random); table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2})); - if (i % 500 == 0) { + if (i % 1000 == 0) { System.out.println(i + " entries. " + ((table instanceof kelondroTree) ? ((kelondroTree) table).cacheNodeStatusString() : "")); + if (table instanceof kelondroTree) { + profiles = ((kelondroTree) table).profiles(); + System.out.println("Cache Delta: " + kelondroProfile.delta(profiles[0], cacheProfileAcc).toString()); + System.out.println("IO Delta: " + kelondroProfile.delta(profiles[1], ioProfileAcc).toString()); + cacheProfileAcc = (kelondroProfile) profiles[0].clone(); + ioProfileAcc = (kelondroProfile) profiles[1].clone(); + } } } } @@ -250,7 +262,7 @@ public class dbtest { entry = table.get(key); if (entry == null) System.out.println("missing value for entry " + new String(key)); else if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1))); - if (i % 500 == 0) { + if (i % 1000 == 0) { System.out.println(i + " entries processed so far."); } } diff --git a/source/de/anomic/kelondro/kelondroAbstractIOChunks.java b/source/de/anomic/kelondro/kelondroAbstractIOChunks.java index ee5820d86..d66fc388b 100644 --- a/source/de/anomic/kelondro/kelondroAbstractIOChunks.java +++ b/source/de/anomic/kelondro/kelondroAbstractIOChunks.java @@ -51,6 +51,12 @@ public abstract class kelondroAbstractIOChunks { return name; } + // profiling support + protected kelondroProfile profile = new kelondroProfile(); + public kelondroProfile profile() { + return profile; + } + // pseudo-native methods: abstract public int read(long pos, byte[] b, int off, int len) throws IOException; abstract public void write(long pos, byte[] b, int off, int len) throws IOException; @@ -59,6 +65,7 @@ public abstract class kelondroAbstractIOChunks { // derived methods: public void readFully(long pos, byte[] b, int off, int len) throws IOException { + long handle = profile.startRead(); if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len); if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len); while (len > 0) { @@ -68,6 +75,7 @@ public abstract class kelondroAbstractIOChunks { off += r; len -= r; } + profile.stopRead(handle); } public byte readByte(long pos) throws IOException { @@ -77,7 +85,9 @@ public abstract class kelondroAbstractIOChunks { } public void writeByte(long pos, final int v) throws IOException { + long handle = profile.startWrite(); this.write(pos, new byte[]{(byte) (v & 0xFF)}); + profile.stopWrite(handle); } public short readShort(long pos) throws IOException { @@ -87,7 +97,9 @@ public abstract class kelondroAbstractIOChunks { } public void writeShort(long pos, final int v) throws IOException { + long handle = profile.startWrite(); this.write(pos, new byte[]{(byte) ((v >>> 8) & 0xFF), (byte) ((v >>> 0) & 0xFF)}); + profile.stopWrite(handle); } public int readInt(long pos) throws IOException { @@ -97,12 +109,14 @@ public abstract class kelondroAbstractIOChunks { } public void writeInt(long pos, final int v) throws IOException { + long handle = profile.startWrite(); this.write(pos, new byte[]{ (byte) ((v >>> 24) & 0xFF), (byte) ((v >>> 16) & 0xFF), (byte) ((v >>> 8) & 0xFF), (byte) ((v >>> 0) & 0xFF) }); + profile.stopWrite(handle); } public long readLong(long pos) throws IOException { @@ -112,6 +126,7 @@ public abstract class kelondroAbstractIOChunks { } public void writeLong(long pos, final long v) throws IOException { + long handle = profile.startWrite(); this.write(pos, new byte[]{ (byte) ((v >>> 56) & 0xFF), (byte) ((v >>> 48) & 0xFF), @@ -122,10 +137,13 @@ public abstract class kelondroAbstractIOChunks { (byte) ((v >>> 8) & 0xFF), (byte) ((v >>> 0) & 0xFF) }); + profile.stopWrite(handle); } public void write(long pos, final byte[] b) throws IOException { + long handle = profile.startWrite(); this.write(pos, b, 0, b.length); + profile.stopWrite(handle); } } \ No newline at end of file diff --git a/source/de/anomic/kelondro/kelondroIOChunks.java b/source/de/anomic/kelondro/kelondroIOChunks.java index 14809edf1..ba0029b7c 100644 --- a/source/de/anomic/kelondro/kelondroIOChunks.java +++ b/source/de/anomic/kelondro/kelondroIOChunks.java @@ -70,4 +70,5 @@ public interface kelondroIOChunks { public void write(long pos, byte[] b) throws IOException; + public kelondroProfile profile(); } diff --git a/source/de/anomic/kelondro/kelondroIntBytesMap.java b/source/de/anomic/kelondro/kelondroIntBytesMap.java index 1bb582e01..ef05f3e49 100644 --- a/source/de/anomic/kelondro/kelondroIntBytesMap.java +++ b/source/de/anomic/kelondro/kelondroIntBytesMap.java @@ -24,7 +24,7 @@ package de.anomic.kelondro; -public class kelondroIntBytesMap extends kelondroRowBufferedSet { +public class kelondroIntBytesMap extends kelondroRowSet { public kelondroIntBytesMap(int payloadSize, int initSize) { super(new kelondroRow(new int[]{4, payloadSize}), initSize); diff --git a/source/de/anomic/kelondro/kelondroProfile.java b/source/de/anomic/kelondro/kelondroProfile.java new file mode 100644 index 000000000..8c43f2e62 --- /dev/null +++ b/source/de/anomic/kelondro/kelondroProfile.java @@ -0,0 +1,115 @@ +// kelondroProfile.java +// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany +// first published 23.06.2006 on http://www.anomic.de +// +// This is a part of the kelondro database, +// which is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +public class kelondroProfile implements Cloneable { + + private long accRead; + private long accWrite; + private long accDelete; + + public kelondroProfile() { + accRead = 0; + accWrite = 0; + accDelete = 0; + } + + public long timeRead() { + return accRead; + } + + public long timeWrite() { + return accWrite; + } + + public long timeDelete() { + return accDelete; + } + + public void timeReset() { + accRead = 0; + accWrite = 0; + accDelete = 0; + } + + protected long startRead() { + return System.currentTimeMillis(); + } + + protected void stopRead(long handle) { + accRead += System.currentTimeMillis() - handle; + } + + protected long startWrite() { + return System.currentTimeMillis(); + } + + protected void stopWrite(long handle) { + accWrite += System.currentTimeMillis() - handle; + } + + protected long startDelete() { + return System.currentTimeMillis(); + } + + protected void stopDelete(long handle) { + accDelete += System.currentTimeMillis() - handle; + } + + public Object clone() { + kelondroProfile clone = new kelondroProfile(); + clone.accRead = this.accRead; + clone.accWrite = this.accWrite; + clone.accDelete = this.accDelete; + return clone; + } + + public String toString() { + return "read=" + accRead + ", write=" + accWrite + ", delete=" + accDelete; + } + + public static kelondroProfile consolidate(kelondroProfile[] profiles) { + for (int i = 1; i < profiles.length; i++) consolidate(profiles[0], profiles[i]); + return profiles[0]; + } + + public static kelondroProfile consolidate(kelondroProfile profile1, kelondroProfile profile2) { + profile1.accRead += profile2.accRead; + profile1.accWrite += profile2.accWrite; + profile1.accDelete += profile2.accDelete; + return profile1; + } + + public static kelondroProfile delta(kelondroProfile newer, kelondroProfile older) { + kelondroProfile result = new kelondroProfile(); + result.accRead = newer.accRead - older.accRead; + result.accWrite = newer.accWrite - older.accWrite; + result.accDelete = newer.accDelete - older.accDelete; + return result; + } +} diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 6a7842bbe..4d9b3146d 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -1371,4 +1371,15 @@ public class kelondroRecords { } } + public kelondroProfile[] profiles() { + return new kelondroProfile[]{ + (cacheHeaders == null) ? new kelondroProfile() : + kelondroProfile.consolidate(new kelondroProfile[]{ + cacheHeaders[0].profile(), + cacheHeaders[1].profile(), + cacheHeaders[2].profile() + }), + entryFile.profile() + }; + } } diff --git a/source/de/anomic/kelondro/kelondroRowBufferedSet.java b/source/de/anomic/kelondro/kelondroRowBufferedSet.java index 679c4f933..883fd48be 100644 --- a/source/de/anomic/kelondro/kelondroRowBufferedSet.java +++ b/source/de/anomic/kelondro/kelondroRowBufferedSet.java @@ -24,7 +24,7 @@ package de.anomic.kelondro; -import java.util.TreeMap; +import java.util.HashMap; import java.util.Map; import java.util.Iterator; import java.util.Random; @@ -32,19 +32,22 @@ import java.util.Random; public class kelondroRowBufferedSet extends kelondroRowSet { private static final long memBlockLimit = 2000000; // do not fill cache further if the amount of available memory is less that this - private static final int bufferFlushLimit = 100000; + private static final int bufferFlushLimit = 10000; private static final int bufferFlushMinimum = 1000; private final boolean useRowCollection = true; - private TreeMap buffer; // this must be a TreeSet bacause HashMap does not work with byte[] + private kelondroProfile profile; + private HashMap buffer; public kelondroRowBufferedSet(kelondroRow rowdef) { super(rowdef); - buffer = new TreeMap(kelondroNaturalOrder.naturalOrder); + buffer = new HashMap(); + profile = new kelondroProfile(); } public kelondroRowBufferedSet(kelondroRow rowdef, int objectCount) { super(rowdef, objectCount); - buffer = new TreeMap(kelondroNaturalOrder.naturalOrder); + buffer = new HashMap(); + profile = new kelondroProfile(); } private final void flush() { @@ -125,90 +128,97 @@ public class kelondroRowBufferedSet extends kelondroRowSet { } public kelondroRow.Entry get(byte[] key) { + long handle = profile.startRead(); + kelondroRow.Entry entry = null; synchronized (buffer) { - if (useRowCollection) { - kelondroRow.Entry entry = (kelondroRow.Entry) buffer.get(key); - if (entry != null) return entry; - return super.get(key); - } else { - return (kelondroRow.Entry) buffer.get(key); - } + entry = (kelondroRow.Entry) buffer.get(new Integer((int) kelondroNaturalOrder.decodeLong(key))); + if ((entry == null) && (useRowCollection)) entry = super.get(key); } + profile.stopRead(handle); + return entry; } public kelondroRow.Entry put(kelondroRow.Entry newentry) { + long handle = profile.startWrite(); byte[] key = newentry.getColBytes(super.sortColumn); + kelondroRow.Entry oldentry = null; synchronized (buffer) { if (useRowCollection) { - kelondroRow.Entry oldentry = (kelondroRow.Entry) buffer.get(key); + oldentry = (kelondroRow.Entry) buffer.get(new Integer((int) kelondroNaturalOrder.decodeLong(key))); if (oldentry == null) { // try the collection oldentry = super.get(key); if (oldentry == null) { // this was not anywhere - buffer.put(key, newentry); + buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry); if (((buffer.size() > bufferFlushMinimum) && (kelondroRecords.availableMemory() > memBlockLimit)) || (buffer.size() > bufferFlushLimit)) flush(); - return null; } else { // replace old entry super.put(newentry); - return oldentry; } } else { // the entry is already in buffer // simply replace old entry - buffer.put(key, newentry); - return oldentry; + buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry); } } else { - return (kelondroRow.Entry) buffer.put(key, newentry); + oldentry = (kelondroRow.Entry) buffer.put(new Integer((int) kelondroNaturalOrder.decodeLong(key)), newentry); } } + profile.stopWrite(handle); + return oldentry; } - public kelondroRow.Entry removeShift(byte[] a) { + public kelondroRow.Entry removeShift(byte[] key) { + long handle = profile.startDelete(); + kelondroRow.Entry oldentry = null; synchronized (buffer) { if (useRowCollection) { - kelondroRow.Entry oldentry = (kelondroRow.Entry) buffer.remove(a); + oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); if (oldentry == null) { // try the collection - return super.removeShift(a); - } else { - // the entry was in buffer - return oldentry; + oldentry = super.removeShift(key); } } else { - return (kelondroRow.Entry) buffer.remove(a); // test + oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); } } + profile.stopDelete(handle); + return oldentry; } - public kelondroRow.Entry removeMarked(byte[] a) { + public kelondroRow.Entry removeMarked(byte[] key) { + long handle = profile.startDelete(); + kelondroRow.Entry oldentry = null; synchronized (buffer) { if (useRowCollection) { - kelondroRow.Entry oldentry = (kelondroRow.Entry) buffer.remove(a); + oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); if (oldentry == null) { // try the collection - return super.removeMarked(a); - } else { - // the entry was in buffer - return oldentry; + return super.removeMarked(key); } } else { - return (kelondroRow.Entry) buffer.remove(a); // test + oldentry = (kelondroRow.Entry) buffer.remove(new Integer((int) kelondroNaturalOrder.decodeLong(key))); } } + profile.stopDelete(handle); + return oldentry; } public void removeMarkedAll(kelondroRowCollection c) { - // this can be enhanced + long handle = profile.startDelete(); synchronized (buffer) { flush(); super.removeMarkedAll(c); } + profile.stopDelete(handle); } + public kelondroProfile profile() { + return profile; + } + public static void main(String[] args) { String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" }; kelondroRowBufferedSet c = new kelondroRowBufferedSet(new kelondroRow(new int[]{10, 3})); diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index 8ee362a22..f97b63878 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -32,17 +32,20 @@ public class kelondroRowSet extends kelondroRowCollection { private static final int collectionReSortLimit = 90; private static final int removeMaxSize = 100; - + + private kelondroProfile profile; private TreeSet removeMarker; public kelondroRowSet(kelondroRow rowdef) { super(rowdef); this.removeMarker = new TreeSet(); + this.profile = new kelondroProfile(); } public kelondroRowSet(kelondroRow rowdef, int objectCount) { super(rowdef, objectCount); this.removeMarker = new TreeSet(); + this.profile = new kelondroProfile(); } public kelondroRow.Entry get(byte[] key) { @@ -50,33 +53,34 @@ public class kelondroRowSet extends kelondroRowCollection { } private kelondroRow.Entry get(byte[] key, int astart, int alength) { + long handle = profile.startRead(); + kelondroRow.Entry entry = null; synchronized (chunkcache) { int index = find(key, astart, alength); - if ((index < 0) || (isMarkedRemoved(index))){ - return null; - } else { - return get(index); - } + if ((index >= 0) && (!(isMarkedRemoved(index)))) entry = get(index); } + profile.stopRead(handle); + return entry; } public kelondroRow.Entry put(kelondroRow.Entry entry) { + long handle = profile.startWrite(); int index = -1; + kelondroRow.Entry oldentry = null; synchronized (chunkcache) { index = find(entry.bytes(), super.rowdef.colstart[super.sortColumn], super.rowdef.width(super.sortColumn)); if (isMarkedRemoved(index)) { set(index, entry); removeMarker.remove(new Integer(index)); - return null; } else if (index < 0) { add(entry); - return null; } else { - kelondroRow.Entry oldentry = get(index); + oldentry = get(index); set(index, entry); - return oldentry; } } + profile.stopWrite(handle); + return oldentry; } public int size() { @@ -88,26 +92,39 @@ public class kelondroRowSet extends kelondroRowCollection { } private kelondroRow.Entry removeMarked(byte[] a, int astart, int alength) { - // the byte[] a may be shorter than the chunksize if (chunkcount == 0) return null; - kelondroRow.Entry b = null; + long handle = profile.startDelete(); + + // check if it is contained in chunkcache + kelondroRow.Entry entry = null; synchronized(chunkcache) { int p = find(a, astart, alength); - if (p < 0) return null; - b = get(p); + if (p < 0) { + // the entry is not there + profile.stopDelete(handle); + return null; + } + + // there is an entry + entry = get(p); if (p < sortBound) { removeMarker.add(new Integer(p)); } else { super.swap(p, --chunkcount, 0); } + + // check case when complete chunkcache is marked as deleted + if (removeMarker.size() == chunkcount) { + this.clear(); + removeMarker.clear(); + } } - if (removeMarker.size() == chunkcount) { - chunkcount = 0; - sortBound = 0; - removeMarker.clear(); - } + + // check if removeMarker is full if (removeMarker.size() >= removeMaxSize) resolveMarkedRemoved(); - return b; + + profile.stopDelete(handle); + return entry; } private boolean isMarkedRemoved(int index) { @@ -167,27 +184,31 @@ public class kelondroRowSet extends kelondroRowCollection { private kelondroRow.Entry removeShift(byte[] a, int astart, int alength) { // the byte[] a may be shorter than the chunksize if (chunkcount == 0) return null; - kelondroRow.Entry b = null; + long handle = profile.startDelete(); + kelondroRow.Entry entry = null; synchronized(chunkcache) { int p = find(a, astart, alength); if (p < 0) return null; - b = get(p); + entry = get(p); if (p < sortBound) { removeShift(p); } else { super.swap(p, --chunkcount, 0); } } - return b; + profile.stopDelete(handle); + return entry; } public void removeMarkedAll(kelondroRowCollection c) { + long handle = profile.startDelete(); Iterator i = c.elements(); byte[] b; while (i.hasNext()) { b = (byte[]) i.next(); removeMarked(b, 0, b.length); } + profile.stopDelete(handle); } public kelondroOrder getOrdering() { @@ -269,6 +290,10 @@ public class kelondroRowSet extends kelondroRowCollection { return true; } + public kelondroProfile profile() { + return profile; + } + public static void main(String[] args) { String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" }; kelondroRowSet c = new kelondroRowSet(new kelondroRow(new int[]{10, 3}));