removed active profiling in kelondroRowSet

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5433 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent be4c458951
commit 07fc115e90

Binary file not shown.

@ -17,7 +17,6 @@ import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroEcoTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroIntBytesMap;
import de.anomic.kelondro.kelondroProfile;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRowSet;
import de.anomic.kelondro.kelondroSQLTable;
@ -306,22 +305,12 @@ public class dbtest {
final long randomstart = Long.parseLong(args[5]);
final Random random = new Random(randomstart);
byte[] key;
kelondroProfile ioProfileAcc = new kelondroProfile();
kelondroProfile cacheProfileAcc = new kelondroProfile();
kelondroProfile[] profiles;
for (int i = 0; i < count; i++) {
key = randomHash(random);
table_test.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
if (table_reference != null) table_reference.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
if (i % 1000 == 0) {
System.out.println(i + " entries. ");
if (table_test instanceof kelondroTree) {
profiles = ((kelondroTree) table_test).profiles();
System.out.println("Cache Delta: " + kelondroProfile.delta(profiles[0], cacheProfileAcc).toString());
System.out.println("IO Delta: " + kelondroProfile.delta(profiles[1], ioProfileAcc).toString());
cacheProfileAcc = profiles[0].clone();
ioProfileAcc = profiles[1].clone();
}
}
}
}

@ -131,7 +131,7 @@ public class CrawlEntry extends serverProcessorJob {
// create new entry and store it into database
assert url != null;
assert initiator != null;
assert profileHandle.length() == yacySeedDB.commonHashLength : profileHandle + " != " + yacySeedDB.commonHashLength;
assert profileHandle == null || profileHandle.length() == yacySeedDB.commonHashLength : profileHandle + " != " + yacySeedDB.commonHashLength;
this.initiator = initiator;
this.url = url;
this.refhash = (referrerhash == null) ? "" : referrerhash;

@ -32,12 +32,6 @@ public abstract class kelondroAbstractIOChunks implements kelondroIOChunks {
return name;
}
// profiling support
protected kelondroProfile profile = new kelondroProfile();
public kelondroProfile profile() {
return profile;
}
// pseudo-native methods:
abstract public long length() throws IOException;
abstract public void write(long pos, byte[] b, int off, int len) throws IOException;
@ -51,9 +45,7 @@ public abstract class kelondroAbstractIOChunks implements kelondroIOChunks {
}
public synchronized void writeByte(final long pos, final int v) throws IOException {
final long handle = profile.startWrite();
this.write(pos, new byte[]{(byte) (v & 0xFF)});
profile.stopWrite(handle);
}
public synchronized short readShort(final long pos) throws IOException {
@ -63,9 +55,7 @@ public abstract class kelondroAbstractIOChunks implements kelondroIOChunks {
}
public synchronized void writeShort(final long pos, final int v) throws IOException {
final long handle = profile.startWrite();
this.write(pos, new byte[]{(byte) ((v >>> 8) & 0xFF), (byte) ((v >>> 0) & 0xFF)});
profile.stopWrite(handle);
}
public synchronized int readInt(final long pos) throws IOException {
@ -75,14 +65,12 @@ public abstract class kelondroAbstractIOChunks implements kelondroIOChunks {
}
public synchronized void writeInt(final long pos, final int v) throws IOException {
final long handle = profile.startWrite();
this.write(pos, new byte[]{
(byte) ((v >>> 24) & 0xFF),
(byte) ((v >>> 16) & 0xFF),
(byte) ((v >>> 8) & 0xFF),
(byte) ((v >>> 0) & 0xFF)
});
profile.stopWrite(handle);
}
public synchronized long readLong(final long pos) throws IOException {
@ -92,7 +80,6 @@ public abstract class kelondroAbstractIOChunks implements kelondroIOChunks {
}
public synchronized void writeLong(final long pos, final long v) throws IOException {
final long handle = profile.startWrite();
this.write(pos, new byte[]{
(byte) ((v >>> 56) & 0xFF),
(byte) ((v >>> 48) & 0xFF),
@ -103,13 +90,10 @@ public abstract class kelondroAbstractIOChunks implements kelondroIOChunks {
(byte) ((v >>> 8) & 0xFF),
(byte) ((v >>> 0) & 0xFF)
});
profile.stopWrite(handle);
}
public synchronized void write(final long pos, final byte[] b) throws IOException {
final long handle = profile.startWrite();
this.write(pos, b, 0, b.length);
profile.stopWrite(handle);
}
}

@ -127,10 +127,6 @@ public class kelondroBytesIntMap {
return index.rows(up, firstKey);
}
public kelondroProfile profile() {
return index.profile();
}
public synchronized void close() {
index.close();
index = null;

@ -180,10 +180,6 @@ public class kelondroBytesLongMap {
return index.rows(up, firstKey);
}
public kelondroProfile profile() {
return index.profile();
}
public synchronized void close() {
index.close();
index = null;

@ -93,10 +93,6 @@ public class kelondroCache implements kelondroIndex {
return 0;
}
public kelondroProfile profile() {
return index.profile(); // TODO: implement own profile and merge with global
}
public static void setCacheGrowStati(final long memStopGrowNew, final long memStartShrinkNew) {
memStopGrow = memStopGrowNew;
memStartShrink = memStartShrinkNew;

@ -245,18 +245,6 @@ public class kelondroCachedRecords extends kelondroAbstractRecords implements ke
this.cacheHeaders = null;
}
public kelondroProfile[] profiles() {
return new kelondroProfile[]{
(cacheHeaders == null) ? new kelondroProfile() :
cacheHeaders.profile(),
entryFile.profile()
};
}
public kelondroProfile profile() {
return kelondroProfile.consolidate(profiles());
}
public void print() throws IOException {
super.print();

@ -503,7 +503,7 @@ public class kelondroCollectionIndex {
} else {
// merge with the old collection
// attention! this modifies the indexrow entry which must be written with index.put(indexrow) afterwards!
kelondroRowCollection collection = container;
kelondroRowSet collection = container;
// read old information
final int oldchunksize = (int) indexrow.getColLong(idx_col_chunksize); // needed only for migration
@ -515,7 +515,7 @@ public class kelondroCollectionIndex {
// load the old collection and join it
try {
kelondroRowCollection krc = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false);
kelondroRowSet krc = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false);
//System.out.println("***DEBUG kelondroCollectionIndex.merge before merge*** krc.size = " + krc.size() + ", krc.sortbound = " + krc.sortBound + ", collection.size = " + collection.size() + ", collection.sortbound = " + collection.sortBound);
collection = collection.merge(krc);
//System.out.println("***DEBUG kelondroCollectionIndex.merge after merge*** collection.size = " + collection.size() + ", collection.sortbound = " + collection.sortBound);

@ -366,10 +366,6 @@ public class kelondroEcoTable implements kelondroIndex {
return index.keys(up, firstKey);
}
public kelondroProfile profile() {
return null;
}
public synchronized Entry put(final Entry row) throws IOException {
assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));

@ -389,23 +389,11 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
public kelondroProfile profile() {
return index.profile();
}
public static final Iterator<String> filenames() {
// iterates string objects; all file names from record tracker
return tableTracker.keySet().iterator();
}
public static final kelondroProfile profileStats(final String filename) {
// returns a map for each file in the tracker;
// the map represents properties for each record oobjects,
// i.e. for cache memory allocation
final kelondroFlexTable theFlexTable = tableTracker.get(filename);
return theFlexTable.profile();
}
public static final Map<String, String> memoryStats(final String filename) {
// returns a map for each file in the tracker;
// the map represents properties for each record objects,

@ -54,7 +54,5 @@ public interface kelondroIOChunks {
public void write(long pos, byte[] b) throws IOException;
public kelondroProfile profile();
public void deleteOnExit();
}

@ -40,7 +40,6 @@ public interface kelondroIndex {
public String filename(); // returns a unique identified for this index; can be a real or artificial file name
public int size();
public kelondroProfile profile();
public kelondroRow row();
public boolean has(byte[] key); // use this only if there is no get in case that has returns true
public kelondroRow.Entry get(byte[] key) throws IOException;

@ -210,12 +210,6 @@ public class kelondroIntBytesMap {
}
}
public kelondroProfile profile() {
if (index0 == null) return index1.profile();
if (index1 == null) return index0.profile();
return kelondroProfile.consolidate(index0.profile(), index1.profile());
}
public static void main(final String[] args) {
boolean assertEnabled = false;
assert assertEnabled = true;

@ -1,115 +0,0 @@
// kelondroProfile.java
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 23.06.2006 on http://www.anomic.de
//
// This is a part of the kelondro database,
// which is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
public class kelondroProfile implements Cloneable {
private long accRead;
private long accWrite;
private long accDelete;
public kelondroProfile() {
accRead = 0;
accWrite = 0;
accDelete = 0;
}
public long timeRead() {
return accRead;
}
public long timeWrite() {
return accWrite;
}
public long timeDelete() {
return accDelete;
}
public void timeReset() {
accRead = 0;
accWrite = 0;
accDelete = 0;
}
protected long startRead() {
return System.currentTimeMillis();
}
protected void stopRead(final long handle) {
accRead += System.currentTimeMillis() - handle;
}
protected long startWrite() {
return System.currentTimeMillis();
}
protected void stopWrite(final long handle) {
accWrite += System.currentTimeMillis() - handle;
}
protected long startDelete() {
return System.currentTimeMillis();
}
protected void stopDelete(final long handle) {
accDelete += System.currentTimeMillis() - handle;
}
public kelondroProfile clone() {
final kelondroProfile clone = new kelondroProfile();
clone.accRead = this.accRead;
clone.accWrite = this.accWrite;
clone.accDelete = this.accDelete;
return clone;
}
public String toString() {
return "read=" + accRead + ", write=" + accWrite + ", delete=" + accDelete;
}
public static kelondroProfile consolidate(final kelondroProfile[] profiles) {
for (int i = 1; i < profiles.length; i++) consolidate(profiles[0], profiles[i]);
return profiles[0];
}
public static kelondroProfile consolidate(final kelondroProfile profile1, final kelondroProfile profile2) {
profile1.accRead += profile2.accRead;
profile1.accWrite += profile2.accWrite;
profile1.accDelete += profile2.accDelete;
return profile1;
}
public static kelondroProfile delta(final kelondroProfile newer, final kelondroProfile older) {
final kelondroProfile result = new kelondroProfile();
result.accRead = newer.accRead - older.accRead;
result.accWrite = newer.accWrite - older.accWrite;
result.accDelete = newer.accDelete - older.accDelete;
return result;
}
}

@ -214,12 +214,6 @@ public class kelondroRAMIndex implements kelondroIndex {
true);
}
public kelondroProfile profile() {
if (index0 == null) return index1.profile();
if (index1 == null) return index0.profile();
return kelondroProfile.consolidate(index0.profile(), index1.profile());
}
public synchronized void close() {
if (index0 != null) index0.close();
if (index1 != null) index1.close();

@ -183,7 +183,7 @@ public class kelondroRowCollection implements Iterable<kelondroRow.Entry> {
return this.rowdef;
}
private final void ensureSize(final int elements) {
protected final void ensureSize(final int elements) {
final int needed = elements * rowdef.objectsize;
if (chunkcache.length >= needed) return;
byte[] newChunkcache = new byte[(int) (needed * growfactor)]; // increase space
@ -312,7 +312,7 @@ public class kelondroRowCollection implements Iterable<kelondroRow.Entry> {
this.lastTimeWrote = System.currentTimeMillis();
}
private final void addSorted(final byte[] a, final int astart, final int alength) {
protected final void addSorted(final byte[] a, final int astart, final int alength) {
assert (a != null);
assert (astart >= 0) && (astart < a.length) : " astart = " + astart;
assert (!(serverLog.allZero(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
@ -838,56 +838,6 @@ public class kelondroRowCollection implements Iterable<kelondroRow.Entry> {
return true;
}
/**
* merge this row collection with another row collection.
* the current collection is not altered in any way, the returned collection is a new collection with copied content.
* The resulting collection is sorted and does not contain any doubles, which are also removed during the merge
* @param c
* @return
*/
public kelondroRowCollection merge(kelondroRowCollection c) {
assert this.rowdef == c.rowdef;
kelondroRowCollection r = new kelondroRowCollection(this.rowdef, this.size() + c.size());
this.sort();
c.sort();
int ti = 0, ci = 0;
int tp, cp;
int o;
final int pkl = this.rowdef.primaryKeyLength;
while (ti < this.size() && ci < c.size()) {
tp = ti * this.rowdef.objectsize;
cp = ci * this.rowdef.objectsize;
o = this.rowdef.objectOrder.compare(this.chunkcache, tp, pkl, c.chunkcache, cp, pkl);
if (o == 0) {
r.addSorted(this.chunkcache, tp, this.rowdef.objectsize);
ti++;
ci++;
continue;
}
if (o < 0) {
r.addSorted(this.chunkcache, tp, this.rowdef.objectsize);
ti++;
continue;
}
if (o > 0) {
r.addSorted(c.chunkcache, cp, this.rowdef.objectsize);
ci++;
continue;
}
}
while (ti < this.size()) {
tp = ti * this.rowdef.objectsize;
r.addSorted(this.chunkcache, tp, this.rowdef.objectsize);
ti++;
}
while (ci < c.size()) {
cp = ci * this.rowdef.objectsize;
r.addSorted(c.chunkcache, cp, this.rowdef.objectsize);
ci++;
}
return r;
}
public synchronized String toString() {
final StringBuilder s = new StringBuilder();
final Iterator<kelondroRow.Entry> i = iterator();

@ -37,23 +37,18 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
private static final int collectionReSortLimit = 400;
private kelondroProfile profile;
public kelondroRowSet(final kelondroRowSet rs) {
super(rs);
this.profile = rs.profile;
}
public kelondroRowSet(final kelondroRow rowdef, final int objectCount, final byte[] cache, final int sortBound) {
super(rowdef, objectCount, cache, sortBound);
assert rowdef.objectOrder != null;
this.profile = new kelondroProfile();
}
public kelondroRowSet(final kelondroRow rowdef, final int objectCount) {
super(rowdef, objectCount);
assert rowdef.objectOrder != null;
this.profile = new kelondroProfile();
}
/**
@ -65,7 +60,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
public kelondroRowSet(final kelondroRow rowdef, final kelondroRow.Entry exportedCollectionRowEnvironment) {
super(rowdef, exportedCollectionRowEnvironment);
assert rowdef.objectOrder != null;
this.profile = new kelondroProfile();
}
public void setOrdering(final kelondroByteOrder newOrder, final int newColumn) {
@ -102,13 +96,10 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
public void reset() {
super.reset();
this.profile = new kelondroProfile();
}
public synchronized boolean has(final byte[] key) {
final long handle = profile.startRead();
final int index = find(key, 0, key.length);
profile.stopRead(handle);
return index >= 0;
}
@ -117,10 +108,8 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
private kelondroRow.Entry get(final byte[] key, final int astart, final int alength) {
final long handle = profile.startRead();
final int index = find(key, astart, alength);
final kelondroRow.Entry entry = (index >= 0) ? get(index, true) : null;
profile.stopRead(handle);
return entry;
}
@ -136,8 +125,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
public synchronized kelondroRow.Entry put(final kelondroRow.Entry entry) {
assert (entry != null);
assert (entry.getPrimaryKeyBytes() != null);
//assert (!(serverLog.allZero(entry.getColBytes(super.sortColumn))));
final long handle = profile.startWrite();
int index = -1;
kelondroRow.Entry oldentry = null;
// when reaching a specific amount of un-sorted entries, re-sort all
@ -151,19 +138,15 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
oldentry = get(index, true);
set(index, entry);
}
profile.stopWrite(handle);
return oldentry;
}
private synchronized kelondroRow.Entry remove(final byte[] a, final int start, final int length) {
final int index = find(a, start, length);
if (index < 0) return null;
//System.out.println("remove: chunk found at index position (before remove) " + index + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize() * index, length + 10) + ", searchkey=" + serverLog.arrayList(a, start, length));
final kelondroRow.Entry entry = super.get(index, true);
super.removeRow(index, true);
//System.out.println("remove: chunk found at index position (after remove) " + index + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize() * index, length) + ", searchkey=" + serverLog.arrayList(a, start, length));
int findagainindex = 0;
//System.out.println("kelondroRowSet.remove");
assert (findagainindex = find(a, start, length)) < 0 : "remove: chunk found again at index position (after remove) " + findagainindex + ", index(before) = " + index + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize * findagainindex, length) + ", searchkey=" + serverLog.arrayList(a, start, length); // check if the remove worked
return entry;
}
@ -260,7 +243,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return -1;
}
public int binaryPosition(final byte[] key, final int astart, final int alength) {
private int binaryPosition(final byte[] key, final int astart, final int alength) {
// returns the exact position of the key if the key exists,
// or a position of an entry that is greater than the key if the
// key does not exist
@ -278,10 +261,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return l;
}
public kelondroProfile profile() {
return profile;
}
public synchronized Iterator<byte[]> keys() {
sort();
return super.keys();
@ -393,6 +372,84 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
}
/**
* merge this row collection with another row collection.
* The resulting collection is sorted and does not contain any doubles, which are also removed during the merge.
* The new collection may be a copy of one of the old one, or can be an alteration of one of the input collections
* After this merge, none of the input collections should be used, because they can be altered
* @param c
* @return
*/
public kelondroRowSet merge(kelondroRowSet c) {
/*
if (this.isSorted() && this.size() >= c.size()) {
return mergeInsert(this, c);
}*/
return mergeEnum(this, c);
}
/*
private static kelondroRowSet mergeInsert(kelondroRowSet sorted, kelondroRowCollection small) {
assert sorted.rowdef == small.rowdef;
assert sorted.isSorted();
assert small.size() <= sorted.size();
sorted.ensureSize(sorted.size() + small.size());
for (int i = 0; i < small.size(); i++) {
}
return sorted;
}
*/
/**
* merge this row collection with another row collection using an simultanous iteration of the input collections
* the current collection is not altered in any way, the returned collection is a new collection with copied content.
* @param c
* @return
*/
protected static kelondroRowSet mergeEnum(kelondroRowCollection c0, kelondroRowCollection c1) {
assert c0.rowdef == c1.rowdef;
kelondroRowSet r = new kelondroRowSet(c0.rowdef, c0.size() + c1.size());
c0.sort();
c1.sort();
int c0i = 0, c1i = 0;
int c0p, c1p;
int o;
final int objectsize = c0.rowdef.objectsize;
while (c0i < c0.size() && c1i < c1.size()) {
c0p = c0i * objectsize;
c1p = c1i * objectsize;
o = c0.rowdef.objectOrder.compare(
c0.chunkcache, c0p, c0.rowdef.primaryKeyLength,
c1.chunkcache, c1p, c0.rowdef.primaryKeyLength);
if (o == 0) {
r.addSorted(c0.chunkcache, c0p, objectsize);
c0i++;
c1i++;
continue;
}
if (o < 0) {
r.addSorted(c0.chunkcache, c0p, objectsize);
c0i++;
continue;
}
if (o > 0) {
r.addSorted(c1.chunkcache, c1p, objectsize);
c1i++;
continue;
}
}
while (c0i < c0.size()) {
r.addSorted(c0.chunkcache, c0i * objectsize, objectsize);
c0i++;
}
while (c1i < c1.size()) {
r.addSorted(c1.chunkcache, c1i * objectsize, objectsize);
c1i++;
}
return r;
}
public static void main(final String[] args) {
// sort/uniq-test
/*

@ -274,10 +274,6 @@ public class kelondroSQLTable implements kelondroIndex {
return 0;
}
public kelondroProfile profile() {
return new kelondroProfile();
}
public final int cacheObjectChunkSize() {
// dummy method
return -1;

@ -185,14 +185,6 @@ public class kelondroSplitTable implements kelondroIndex {
return s;
}
public synchronized kelondroProfile profile() {
final kelondroProfile[] profiles = new kelondroProfile[tables.size()];
final Iterator<kelondroIndex> i = tables.values().iterator();
int c = 0;
while (i.hasNext()) profiles[c++] = (i.next()).profile();
return kelondroProfile.consolidate(profiles);
}
public int writeBufferSize() {
int s = 0;
for (final kelondroIndex index : tables.values()) {

Loading…
Cancel
Save