- removed migration code - removed BLOBTree after the removal of the BLOBTree, a lot of dead code appeared: - removed dead code that was needed for BLOBTree Some more classes may have not much use any more after the removal of BLOBTree, but still have some component that are needed elsewhere. Additional Refactoring steps are needed to clean up dependencies and then more code may appear that is unused and can be removed as well. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6150 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
16 years ago
parent
d1083a6913
commit
c5122d6836
@ -1,116 +0,0 @@
|
||||
// URLFetcherStack.java
|
||||
// -------------------------------------
|
||||
// part of YACY
|
||||
//
|
||||
// (C) 2007 by Franz Brausze
|
||||
//
|
||||
// last change: $LastChangedDate: $ by $LastChangedBy: $
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.data;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
|
||||
import de.anomic.kelondro.index.Row;
|
||||
import de.anomic.kelondro.index.Row.EntryIndex;
|
||||
import de.anomic.kelondro.order.Base64Order;
|
||||
import de.anomic.kelondro.table.Stack;
|
||||
import de.anomic.kelondro.util.kelondroException;
|
||||
import de.anomic.yacy.yacyURL;
|
||||
import de.anomic.yacy.logging.Log;
|
||||
|
||||
public class URLFetcherStack {
|
||||
|
||||
public static final String DBFILE = "urlRemote2.stack";
|
||||
|
||||
private static final Row rowdef = new Row(
|
||||
"String urlstring-256",
|
||||
Base64Order.enhancedCoder );
|
||||
private final Stack db;
|
||||
private final Log log;
|
||||
|
||||
private int popped = 0;
|
||||
private int pushed = 0;
|
||||
|
||||
public URLFetcherStack(final File path) throws IOException {
|
||||
this.db = new Stack(new File(path, DBFILE), rowdef);
|
||||
this.log = new Log("URLFETCHERSTACK");
|
||||
}
|
||||
|
||||
public int getPopped() { return this.popped; }
|
||||
public int getPushed() { return this.pushed; }
|
||||
public void clearStat() { this.popped = 0; this.pushed = 0; }
|
||||
|
||||
protected void finalize() throws Throwable {
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
public boolean push(final yacyURL url) {
|
||||
try {
|
||||
this.db.push(this.db.row().newEntry(
|
||||
new byte[][] { url.toNormalform(true, true).getBytes("UTF-8") }
|
||||
));
|
||||
this.pushed++;
|
||||
return true;
|
||||
} catch (final IOException e) {
|
||||
this.log.logSevere("error storing entry", e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public yacyURL pop() {
|
||||
try {
|
||||
final Row.Entry r = this.db.pop();
|
||||
if (r == null) return null;
|
||||
final String url = r.getColString(0, null);
|
||||
try {
|
||||
this.popped++;
|
||||
return new yacyURL(url, null);
|
||||
} catch (final MalformedURLException e) {
|
||||
this.log.logSevere("found invalid URL-entry: " + url);
|
||||
return null;
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
this.log.logSevere("error retrieving entry", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public String[] top(final int count) {
|
||||
try {
|
||||
final ArrayList<String> ar = new ArrayList<String>();
|
||||
final Iterator<EntryIndex> it = db.contentRows(500);
|
||||
Row.EntryIndex ei;
|
||||
for (int i=0; i<count && it.hasNext(); i++) {
|
||||
ei = it.next();
|
||||
if (ei == null) continue;
|
||||
ar.add(ei.getColString(0, null));
|
||||
}
|
||||
return ar.toArray(new String[ar.size()]);
|
||||
} catch (final kelondroException e) {
|
||||
this.log.logSevere("error retrieving entry", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return this.db.size();
|
||||
}
|
||||
}
|
||||
@ -1,430 +0,0 @@
|
||||
// kelondroBLOBTree.java
|
||||
// (C) 2004 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 09.02.2004 (as "kelondroDyn.java") on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
/*
|
||||
This class extends the kelondroTree and adds dynamic data handling
|
||||
A dynamic content is created, by using several tree nodes and
|
||||
combining them over a set of associated keys.
|
||||
Example: a byte[] of length 1000 shall be stored in a kelondroTree
|
||||
with node size 256. The key for the entry is 'entry'.
|
||||
Then kelondroDyn stores the first part of four into the entry
|
||||
'entry00', the second into 'entry01', and so on.
|
||||
|
||||
*/
|
||||
|
||||
package de.anomic.kelondro.blob;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import de.anomic.kelondro.index.Cache;
|
||||
import de.anomic.kelondro.index.Row;
|
||||
import de.anomic.kelondro.index.ObjectIndex;
|
||||
import de.anomic.kelondro.io.AbstractRandomAccess;
|
||||
import de.anomic.kelondro.io.RandomAccessInterface;
|
||||
import de.anomic.kelondro.order.ByteOrder;
|
||||
import de.anomic.kelondro.order.CloneableIterator;
|
||||
import de.anomic.kelondro.order.RotateIterator;
|
||||
import de.anomic.kelondro.table.EcoTable;
|
||||
import de.anomic.kelondro.table.Tree;
|
||||
import de.anomic.kelondro.util.FileUtils;
|
||||
import de.anomic.kelondro.util.kelondroException;
|
||||
|
||||
public class BLOBTree {
|
||||
|
||||
private static final int counterlen = 8;
|
||||
|
||||
protected int keylen;
|
||||
private final int reclen;
|
||||
//private int segmentCount;
|
||||
private final char fillChar;
|
||||
private final ObjectIndex index;
|
||||
private ObjectBuffer buffer;
|
||||
private final Row rowdef;
|
||||
|
||||
/**
|
||||
* Deprecated Class. Please use Heap instead
|
||||
*/
|
||||
private BLOBTree(final File file, final boolean useNodeCache, final boolean useObjectCache, final int key,
|
||||
final int nodesize, final char fillChar, final ByteOrder objectOrder) {
|
||||
// creates or opens a dynamic tree
|
||||
rowdef = new Row("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder);
|
||||
ObjectIndex fbi;
|
||||
try {
|
||||
fbi = new Tree(file, useNodeCache, 0, rowdef, 1, 8);
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
FileUtils.deletedelete(file);
|
||||
throw new kelondroException(e.getMessage());
|
||||
}
|
||||
this.index = ((useObjectCache) && (!(fbi instanceof EcoTable))) ? (ObjectIndex) new Cache(fbi) : fbi;
|
||||
this.keylen = key;
|
||||
this.reclen = nodesize;
|
||||
this.fillChar = fillChar;
|
||||
//this.segmentCount = 0;
|
||||
//if (!(tree.fileExisted)) writeSegmentCount();
|
||||
buffer = new ObjectBuffer(file.toString());
|
||||
}
|
||||
|
||||
public static Heap toHeap(final File file, final boolean useNodeCache, final boolean useObjectCache, final int key,
|
||||
final int nodesize, final char fillChar, final ByteOrder objectOrder, final File blob) throws IOException {
|
||||
if (blob.exists() || !file.exists()) {
|
||||
// open the blob file and ignore the tree
|
||||
return new Heap(blob, key, objectOrder, 1024 * 64);
|
||||
}
|
||||
// open a Tree and migrate everything to a Heap
|
||||
BLOBTree tree = new BLOBTree(file, useNodeCache, useObjectCache, key, nodesize, fillChar, objectOrder);
|
||||
Heap heap = new Heap(blob, key, objectOrder, 1024 * 64);
|
||||
Iterator<byte[]> i = tree.keys(true, false);
|
||||
byte[] k, kk = new byte[key], v;
|
||||
String s;
|
||||
while (i.hasNext()) {
|
||||
k = i.next();
|
||||
//assert k.length == key : "k.length = " + k.length + ", key = " + key;
|
||||
if (k == null) continue;
|
||||
v = tree.get(k);
|
||||
if (v == null) continue;
|
||||
s = new String(v, "UTF-8").trim();
|
||||
// enlarge entry key to fit into the given key length
|
||||
if (k.length == key) {
|
||||
heap.put(k, s.getBytes("UTF-8"));
|
||||
} else {
|
||||
System.arraycopy(k, 0, kk, 0, k.length);
|
||||
for (int j = k.length; j < key; j++) kk[j] = (byte) fillChar;
|
||||
heap.put(kk, s.getBytes("UTF-8"));
|
||||
}
|
||||
}
|
||||
tree.close(false);
|
||||
return heap;
|
||||
}
|
||||
|
||||
private static String counter(final int c) {
|
||||
String s = Integer.toHexString(c);
|
||||
while (s.length() < counterlen) s = "0" + s;
|
||||
return s;
|
||||
}
|
||||
|
||||
private byte[] elementKey(String key, final int record) {
|
||||
if (key.length() > keylen) throw new RuntimeException("key len (" + key.length() + ") out of limit (" + keylen + "): '" + key + "'");
|
||||
while (key.length() < keylen) key = key + fillChar;
|
||||
key = key + counter(record);
|
||||
try {
|
||||
return key.getBytes("UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return key.getBytes();
|
||||
}
|
||||
}
|
||||
|
||||
private String origKey(final byte[] rawKey) {
|
||||
int n = keylen - 1;
|
||||
if (n >= rawKey.length) n = rawKey.length - 1;
|
||||
while ((n > 0) && (rawKey[n] == (byte) fillChar)) n--;
|
||||
try {
|
||||
return new String(rawKey, 0, n + 1, "UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return new String(rawKey, 0, n + 1);
|
||||
}
|
||||
}
|
||||
|
||||
private class keyIterator implements CloneableIterator<byte[]> {
|
||||
// the iterator iterates all keys
|
||||
CloneableIterator<Row.Entry> ri;
|
||||
String nextKey;
|
||||
|
||||
private keyIterator(final CloneableIterator<Row.Entry> iter) {
|
||||
ri = iter;
|
||||
nextKey = n();
|
||||
}
|
||||
|
||||
public keyIterator clone(final Object modifier) {
|
||||
return new keyIterator(ri.clone(modifier));
|
||||
}
|
||||
|
||||
public boolean hasNext() {
|
||||
return nextKey != null;
|
||||
}
|
||||
|
||||
public byte[] next() {
|
||||
final String result = nextKey;
|
||||
nextKey = n();
|
||||
try {
|
||||
return origKey(result.getBytes("UTF-8")).getBytes("UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return origKey(result.getBytes()).getBytes();
|
||||
}
|
||||
}
|
||||
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException("no remove in RawKeyIterator");
|
||||
}
|
||||
|
||||
private String n() {
|
||||
byte[] g;
|
||||
String k;
|
||||
String v;
|
||||
int c;
|
||||
Row.Entry nt;
|
||||
while (ri.hasNext()) {
|
||||
nt = ri.next();
|
||||
if (nt == null) return null;
|
||||
g = nt.getColBytes(0);
|
||||
if (g == null) return null;
|
||||
try {
|
||||
k = new String(g, 0, keylen, "UTF-8");
|
||||
} catch (UnsupportedEncodingException e1) {
|
||||
k = new String(g, 0, keylen);
|
||||
}
|
||||
try {
|
||||
v = new String(g, keylen, counterlen, "UTF-8");
|
||||
} catch (UnsupportedEncodingException e1) {
|
||||
v = new String(g, keylen, counterlen);
|
||||
}
|
||||
try {
|
||||
c = Integer.parseInt(v, 16);
|
||||
} catch (final NumberFormatException e) {
|
||||
c = -1;
|
||||
}
|
||||
if (c == 0) return k;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private synchronized CloneableIterator<byte[]> keys(final boolean up, final boolean rotating) throws IOException {
|
||||
// iterates only the keys of the Nodes
|
||||
// enumerated objects are of type String
|
||||
final keyIterator i = new keyIterator(index.rows(up, null));
|
||||
if (rotating) return new RotateIterator<byte[]>(i, null, index.size());
|
||||
return i;
|
||||
}
|
||||
|
||||
private byte[] getValueCached(final byte[] key) throws IOException {
|
||||
|
||||
// read from buffer
|
||||
final byte[] buffered = (byte[]) buffer.get(key);
|
||||
if (buffered != null) return buffered;
|
||||
|
||||
// read from db
|
||||
final Row.Entry result = index.get(key);
|
||||
if (result == null) return null;
|
||||
|
||||
// return result
|
||||
return result.getColBytes(1);
|
||||
}
|
||||
|
||||
private synchronized void setValueCached(final byte[] key, final byte[] value) throws IOException {
|
||||
// update storage
|
||||
synchronized (this) {
|
||||
index.put(rowdef.newEntry(new byte[][]{key, value}));
|
||||
buffer.put(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized int get(final String key, final int pos) throws IOException {
|
||||
final int reccnt = pos / reclen;
|
||||
// read within a single record
|
||||
final byte[] buf = getValueCached(elementKey(key, reccnt));
|
||||
if (buf == null) return -1;
|
||||
final int recpos = pos % reclen;
|
||||
if (buf.length <= recpos) return -1;
|
||||
return buf[recpos] & 0xFF;
|
||||
}
|
||||
|
||||
private synchronized byte[] get(final byte[] key) throws IOException {
|
||||
final RandomAccessInterface ra = getRA(new String(key, "UTF-8"));
|
||||
if (ra == null) return null;
|
||||
return ra.readFully();
|
||||
}
|
||||
|
||||
private synchronized byte[] get(final String key, final int pos, final int len) throws IOException {
|
||||
final int recpos = pos % reclen;
|
||||
final int reccnt = pos / reclen;
|
||||
byte[] segment1;
|
||||
// read first within a single record
|
||||
if ((recpos == 0) && (reclen == len)) {
|
||||
segment1 = getValueCached(elementKey(key, reccnt));
|
||||
if (segment1 == null) return null;
|
||||
} else {
|
||||
byte[] buf = getValueCached(elementKey(key, reccnt));
|
||||
if (buf == null) return null;
|
||||
if (buf.length < reclen) {
|
||||
byte[] buff = new byte[reclen];
|
||||
System.arraycopy(buf, 0, buff, 0, buf.length);
|
||||
buf = buff;
|
||||
buff = null;
|
||||
}
|
||||
// System.out.println("read:
|
||||
// buf.length="+buf.length+",recpos="+recpos+",len="+len);
|
||||
if (recpos + len <= reclen) {
|
||||
segment1 = new byte[len];
|
||||
System.arraycopy(buf, recpos, segment1, 0, len);
|
||||
} else {
|
||||
segment1 = new byte[reclen - recpos];
|
||||
System.arraycopy(buf, recpos, segment1, 0, reclen - recpos);
|
||||
}
|
||||
}
|
||||
// if this is all, return
|
||||
if (recpos + len <= reclen) return segment1;
|
||||
// read from several records
|
||||
// we combine recursively all participating records
|
||||
// we have two segments: the one in the starting record, and the remaining
|
||||
// segment 1 in record <reccnt> : start = recpos, length = reclen - recpos
|
||||
// segment 2 in record <reccnt>+1: start = 0, length = len - reclen + recpos
|
||||
// recursively step further
|
||||
final byte[] segment2 = get(key, pos + segment1.length, len - segment1.length);
|
||||
if (segment2 == null) return segment1;
|
||||
// now combine the two segments into the result
|
||||
final byte[] result = new byte[segment1.length + segment2.length];
|
||||
System.arraycopy(segment1, 0, result, 0, segment1.length);
|
||||
System.arraycopy(segment2, 0, result, segment1.length, segment2.length);
|
||||
return result;
|
||||
}
|
||||
|
||||
private synchronized void put(final String key, final int pos, final byte[] b, final int off, final int len) throws IOException {
|
||||
final int recpos = pos % reclen;
|
||||
final int reccnt = pos / reclen;
|
||||
byte[] buf;
|
||||
// first write current record
|
||||
if ((recpos == 0) && (reclen == len)) {
|
||||
if (off == 0) {
|
||||
setValueCached(elementKey(key, reccnt), b);
|
||||
} else {
|
||||
buf = new byte[len];
|
||||
System.arraycopy(b, off, buf, 0, len);
|
||||
setValueCached(elementKey(key, reccnt), b);
|
||||
}
|
||||
} else {
|
||||
buf = getValueCached(elementKey(key, reccnt));
|
||||
if (buf == null) {
|
||||
buf = new byte[reclen];
|
||||
} else if (buf.length < reclen) {
|
||||
byte[] buff = new byte[reclen];
|
||||
System.arraycopy(buf, 0, buff, 0, buf.length);
|
||||
buf = buff;
|
||||
}
|
||||
// System.out.println("write:
|
||||
// b.length="+b.length+",off="+off+",len="+(reclen-recpos));
|
||||
if (len < (reclen - recpos))
|
||||
System.arraycopy(b, off, buf, recpos, len);
|
||||
else
|
||||
System.arraycopy(b, off, buf, recpos, reclen - recpos);
|
||||
setValueCached(elementKey(key, reccnt), buf);
|
||||
}
|
||||
// if more records are necessary, write to them also recursively
|
||||
if (recpos + len > reclen) {
|
||||
put(key, pos + reclen - recpos, b, off + reclen - recpos, len - reclen + recpos);
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void put(final String key, final int pos, final int b) throws IOException {
|
||||
final int recpos = pos % reclen;
|
||||
final int reccnt = pos / reclen;
|
||||
byte[] buf;
|
||||
// first write current record
|
||||
buf = getValueCached(elementKey(key, reccnt));
|
||||
if (buf == null) {
|
||||
buf = new byte[reclen];
|
||||
} else if (buf.length < reclen) {
|
||||
byte[] buff = new byte[reclen];
|
||||
System.arraycopy(buf, 0, buff, 0, buf.length);
|
||||
buf = buff;
|
||||
}
|
||||
buf[recpos] = (byte) b;
|
||||
setValueCached(elementKey(key, reccnt), buf);
|
||||
}
|
||||
|
||||
private synchronized RandomAccessInterface getRA(final String filekey) {
|
||||
// this returns always a RARecord, even if no existed bevore
|
||||
//return new kelondroBufferedRA(new RARecord(filekey), 512, 0);
|
||||
return new RARecord(filekey);
|
||||
}
|
||||
|
||||
private class RARecord extends AbstractRandomAccess implements RandomAccessInterface {
|
||||
|
||||
int seekpos = 0;
|
||||
int compLength = -1;
|
||||
|
||||
String filekey;
|
||||
|
||||
private RARecord(final String filekey) {
|
||||
this.filekey = filekey;
|
||||
}
|
||||
|
||||
public long length() throws IOException {
|
||||
if (compLength >= 0) return compLength;
|
||||
int p = 0;
|
||||
while (get(filekey, p, reclen) != null) p+= reclen;
|
||||
compLength = p-1;
|
||||
return p-1;
|
||||
}
|
||||
|
||||
public void setLength(long length) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public long available() throws IOException {
|
||||
return length() - seekpos;
|
||||
}
|
||||
|
||||
public int read() throws IOException {
|
||||
return get(filekey, seekpos++);
|
||||
}
|
||||
|
||||
public void write(final int i) throws IOException {
|
||||
put(filekey, seekpos++, i);
|
||||
}
|
||||
|
||||
public void readFully(final byte[] b, final int off, final int len) throws IOException {
|
||||
int l = Math.min(b.length - off, len);
|
||||
final byte[] buf = get(filekey, seekpos, l);
|
||||
if (buf == null) throw new IOException("record at off " + off + ", len " + len + " not found");
|
||||
l = Math.min(buf.length, l);
|
||||
System.arraycopy(buf, 0, b, off, l);
|
||||
seekpos += l;
|
||||
}
|
||||
|
||||
public void write(final byte[] b, final int off, final int len) throws IOException {
|
||||
put(filekey, seekpos, b, off, len);
|
||||
seekpos += len;
|
||||
}
|
||||
|
||||
public void seek(final long pos) throws IOException {
|
||||
seekpos = (int) pos;
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
// no need to do anything here
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private synchronized void close(boolean writeIDX) {
|
||||
index.close();
|
||||
}
|
||||
}
|
||||
@ -1,573 +0,0 @@
|
||||
// kelondroCachedRecords.java
|
||||
// (C) 2003 - 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 2003 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2009-01-30 15:33:00 +0000 (Fr, 30 Jan 2009) $
|
||||
// $LastChangedRevision: 5540 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro.table;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.kelondro.index.Row;
|
||||
import de.anomic.kelondro.index.ObjectArrayCache;
|
||||
import de.anomic.kelondro.io.RandomAccessRecords;
|
||||
import de.anomic.kelondro.util.MemoryControl;
|
||||
import de.anomic.kelondro.util.kelondroException;
|
||||
|
||||
public class CachedRecords extends AbstractRecords implements RandomAccessRecords {
|
||||
|
||||
// static supervision objects: recognize and coordinate all activites
|
||||
private static final TreeMap<String, CachedRecords> recordTracker = new TreeMap<String, CachedRecords>();
|
||||
private static final long memStopGrow = 40 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available
|
||||
private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available
|
||||
|
||||
// caching buffer
|
||||
private ObjectArrayCache cacheHeaders; // the cache; holds overhead values and key element
|
||||
private int readHit;
|
||||
private int readMiss;
|
||||
private int writeUnique;
|
||||
private int writeDouble;
|
||||
private int cacheDelete;
|
||||
private int cacheFlush;
|
||||
|
||||
|
||||
public CachedRecords(
|
||||
final File file, final boolean useNodeCache, final long preloadTime,
|
||||
final short ohbytec, final short ohhandlec,
|
||||
final Row rowdef, final int FHandles, final int txtProps, final int txtPropWidth) throws IOException {
|
||||
super(file, useNodeCache, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth);
|
||||
initCache(useNodeCache, preloadTime);
|
||||
if (useNodeCache) recordTracker.put(this.filename, this);
|
||||
}
|
||||
|
||||
private void initCache(final boolean useNodeCache, final long preloadTime) {
|
||||
if (useNodeCache) {
|
||||
this.cacheHeaders = new ObjectArrayCache(this.headchunksize, 0);
|
||||
} else {
|
||||
this.cacheHeaders = null;
|
||||
}
|
||||
this.readHit = 0;
|
||||
this.readMiss = 0;
|
||||
this.writeUnique = 0;
|
||||
this.writeDouble = 0;
|
||||
this.cacheDelete = 0;
|
||||
this.cacheFlush = 0;
|
||||
// pre-load node cache
|
||||
if ((preloadTime > 0) && (useNodeCache)) {
|
||||
final long stop = System.currentTimeMillis() + preloadTime;
|
||||
int count = 0;
|
||||
try {
|
||||
final Iterator<Node> i = contentNodes(preloadTime);
|
||||
CacheNode n;
|
||||
while ((System.currentTimeMillis() < stop) && (cacheGrowStatus() == 2) && (i.hasNext())) {
|
||||
n = (CacheNode) i.next();
|
||||
cacheHeaders.addb(n.handle().index, n.headChunk);
|
||||
count++;
|
||||
}
|
||||
cacheHeaders.flush();
|
||||
logFine("preloaded " + count + " records into cache");
|
||||
} catch (final kelondroException e) {
|
||||
// the contentNodes iterator had a time-out; we don't do a preload
|
||||
logFine("could not preload records: " + e.getMessage());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private int cacheGrowStatus() {
|
||||
final long available = MemoryControl.available();
|
||||
if ((cacheHeaders != null) && (available - 2 * 1024 * 1024 < cacheHeaders.memoryNeededForGrow())) return 0;
|
||||
return cacheGrowStatus(available, memStopGrow, memStartShrink);
|
||||
}
|
||||
|
||||
public static final int cacheGrowStatus(final long available, final long stopGrow, final long startShrink) {
|
||||
// returns either 0, 1 or 2:
|
||||
// 0: cache is not allowed to grow, but shall shrink
|
||||
// 1: cache is allowed to grow, but need not to shrink
|
||||
// 2: cache is allowed to grow and must not shrink
|
||||
if (available > stopGrow) return 2;
|
||||
if (available > startShrink) {
|
||||
MemoryControl.gc(60000, "kelendroCacheRecords.cacheGrowStatus(...) 1"); // thq
|
||||
return 1;
|
||||
}
|
||||
MemoryControl.gc(30000, "kelendroCacheRecords.cacheGrowStatus(...) 0"); // thq
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected synchronized void deleteNode(final RecordHandle handle) throws IOException {
|
||||
if (cacheHeaders == null) {
|
||||
super.deleteNode(handle);
|
||||
} else synchronized (cacheHeaders) {
|
||||
if (cacheHeaders.size() == 0) {
|
||||
super.deleteNode(handle);
|
||||
} else {
|
||||
cacheHeaders.removeb(handle.index);
|
||||
cacheDelete++;
|
||||
super.deleteNode(handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void printCache() {
|
||||
if (cacheHeaders == null) {
|
||||
System.out.println("### file report: " + size() + " entries");
|
||||
for (int i = 0; i < USAGE.allCount(); i++) {
|
||||
// print from file to compare
|
||||
System.out.print("#F " + i + ": ");
|
||||
try {
|
||||
for (int j = 0; j < headchunksize; j++)
|
||||
System.out.print(Integer.toHexString(0xff & entryFile.readByte(j + seekpos(new RecordHandle(i)))) + " ");
|
||||
} catch (final IOException e) {}
|
||||
|
||||
System.out.println();
|
||||
}
|
||||
} else {
|
||||
System.out.println("### cache report: " + cacheHeaders.size() + " entries");
|
||||
|
||||
final Iterator<Row.Entry> i = cacheHeaders.rows();
|
||||
Row.Entry entry;
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
|
||||
// print from cache
|
||||
System.out.print("#C ");
|
||||
printChunk(entry);
|
||||
System.out.println();
|
||||
|
||||
// print from file to compare
|
||||
/*
|
||||
System.out.print("#F " + cp + " " + ((Handle) entry.getKey()).index + ": ");
|
||||
try {
|
||||
for (int j = 0; j < headchunksize; j++)
|
||||
System.out.print(entryFile.readByte(j + seekpos((Handle) entry.getKey())) + ",");
|
||||
} catch (IOException e) {}
|
||||
*/
|
||||
System.out.println();
|
||||
}
|
||||
}
|
||||
System.out.println("### end report");
|
||||
}
|
||||
|
||||
public synchronized void close() {
|
||||
if (cacheHeaders == null) {
|
||||
if (recordTracker.get(this.filename) != null) {
|
||||
theLogger.severe("close(): file '" + this.filename + "' was tracked with record tracker, but it should not.");
|
||||
}
|
||||
} else {
|
||||
if (recordTracker.remove(this.filename) == null) {
|
||||
theLogger.severe("close(): file '" + this.filename + "' was not tracked with record tracker.");
|
||||
}
|
||||
}
|
||||
super.close();
|
||||
this.cacheHeaders = null;
|
||||
}
|
||||
|
||||
public void print() throws IOException {
|
||||
super.print();
|
||||
|
||||
// print also all records
|
||||
System.out.println("CACHE");
|
||||
printCache();
|
||||
System.out.println("--");
|
||||
System.out.println("NODES");
|
||||
final Iterator<Node> i = new contentNodeIterator(-1);
|
||||
Node n;
|
||||
while (i.hasNext()) {
|
||||
n = i.next();
|
||||
System.out.println("NODE: " + n.toString());
|
||||
}
|
||||
}
|
||||
|
||||
public Node newNode(final RecordHandle handle, final byte[] bulk, final int offset) throws IOException {
|
||||
return new CacheNode(handle, bulk, offset);
|
||||
}
|
||||
|
||||
public final class CacheNode implements Node {
|
||||
// an Node holds all information of one row of data. This includes the key to the entry
|
||||
// which is stored as entry element at position 0
|
||||
// an Node object can be created in two ways:
|
||||
// 1. instantiation with an index number. After creation the Object does not hold any
|
||||
// value information until such is retrieved using the getValue() method
|
||||
// 2. instantiation with a value array. the values are not directly written into the
|
||||
// file. Expanding the tree structure is then done using the save() method. at any
|
||||
// time it is possible to verify the save state using the saved() predicate.
|
||||
// Therefore an entry object has three modes:
|
||||
// a: holding an index information only (saved() = true)
|
||||
// b: holding value information only (saved() = false)
|
||||
// c: holding index and value information at the same time (saved() = true)
|
||||
// which can be the result of one of the two processes as follow:
|
||||
// (i) created with index and after using the getValue() method, or
|
||||
// (ii) created with values and after calling the save() method
|
||||
// the method will therefore throw an IllegalStateException when the following
|
||||
// process step is performed:
|
||||
// - create the Node with index and call then the save() method
|
||||
// this case can be decided with
|
||||
// ((index != NUL) && (values == null))
|
||||
// The save() method represents the insert function for the tree. Balancing functions
|
||||
// are applied automatically. While balancing, the Node does never change its index key,
|
||||
// but its parent/child keys.
|
||||
//private byte[] ohBytes = null; // the overhead bytes, OHBYTEC values
|
||||
//private Handle[] ohHandle= null; // the overhead handles, OHHANDLEC values
|
||||
//private byte[][] values = null; // an array of byte[] nodes is the value vector
|
||||
private RecordHandle handle = null; // index of the entry, by default NUL means undefined
|
||||
byte[] headChunk = null; // contains ohBytes, ohHandles and the key value
|
||||
private byte[] tailChunk = null; // contains all values except the key value
|
||||
private boolean headChanged = false;
|
||||
private boolean tailChanged = false;
|
||||
|
||||
public CacheNode(final byte[] rowinstance) throws IOException {
|
||||
// this initializer is used to create nodes from bulk-read byte arrays
|
||||
assert ((rowinstance == null) || (rowinstance.length == ROW.objectsize)) : "bulkchunk.length = " + (rowinstance == null ? "null" : rowinstance.length) + ", ROW.width(0) = " + ROW.width(0);
|
||||
this.handle = new RecordHandle(USAGE.allocatePayload(rowinstance));
|
||||
|
||||
// create empty chunks
|
||||
this.headChunk = new byte[headchunksize];
|
||||
this.tailChunk = new byte[tailchunksize];
|
||||
|
||||
// write content to chunks
|
||||
if (rowinstance == null) {
|
||||
for (int i = headchunksize - 1; i >= 0; i--) this.headChunk[i] = (byte) 0xff;
|
||||
for (int i = tailchunksize - 1; i >= 0; i--) this.tailChunk[i] = (byte) 0xff;
|
||||
} else {
|
||||
for (int i = overhead - 1; i >= 0; i--) this.headChunk[i] = (byte) 0xff;
|
||||
System.arraycopy(rowinstance, 0, this.headChunk, overhead, ROW.width(0));
|
||||
System.arraycopy(rowinstance, ROW.width(0), this.tailChunk, 0, tailchunksize);
|
||||
}
|
||||
|
||||
if (cacheHeaders != null) synchronized (cacheHeaders) {
|
||||
updateNodeCache();
|
||||
}
|
||||
|
||||
// mark chunks as changed
|
||||
// if the head/tail chunks come from a file system read, setChanged should be false
|
||||
// if the chunks come from a overwrite attempt, it should be true
|
||||
this.headChanged = false; // we wrote the head already during allocate
|
||||
this.tailChanged = false; // we write the tail already during allocate
|
||||
}
|
||||
|
||||
public CacheNode(final RecordHandle handle, final byte[] bulkchunk, final int offset) throws IOException {
|
||||
// this initializer is used to create nodes from bulk-read byte arrays
|
||||
// if write is true, then the chunk in bulkchunk is written to the file
|
||||
// othervise it is considered equal to what is stored in the file
|
||||
// (that is ensured during pre-loaded enumeration)
|
||||
this.handle = handle;
|
||||
boolean changed;
|
||||
if (handle.index >= USAGE.allCount()) {
|
||||
// this causes only a write action if we create a node beyond the end of the file
|
||||
USAGE.allocateRecord(handle.index, bulkchunk, offset);
|
||||
changed = false; // we have already wrote the record, so it is considered as unchanged
|
||||
} else {
|
||||
changed = true;
|
||||
}
|
||||
assert ((bulkchunk == null) || (bulkchunk.length - offset >= recordsize)) : "bulkchunk.length = " + (bulkchunk == null ? "null" : bulkchunk.length) + ", offset = " + offset + ", recordsize = " + recordsize;
|
||||
|
||||
// create empty chunks
|
||||
this.headChunk = new byte[headchunksize];
|
||||
this.tailChunk = new byte[tailchunksize];
|
||||
|
||||
// write content to chunks
|
||||
if (bulkchunk != null) {
|
||||
System.arraycopy(bulkchunk, offset, this.headChunk, 0, headchunksize);
|
||||
System.arraycopy(bulkchunk, offset + headchunksize, this.tailChunk, 0, tailchunksize);
|
||||
}
|
||||
|
||||
// mark chunks as changed
|
||||
this.headChanged = changed;
|
||||
this.tailChanged = changed;
|
||||
}
|
||||
|
||||
public CacheNode(final RecordHandle handle, final boolean fillTail) throws IOException {
|
||||
this(handle, null, 0, fillTail);
|
||||
}
|
||||
|
||||
public CacheNode(final RecordHandle handle, final CacheNode parentNode, final int referenceInParent, boolean fillTail) throws IOException {
|
||||
// this creates an entry with an pre-reserved entry position.
|
||||
// values can be written using the setValues() method,
|
||||
// but we expect that values are already there in the file.
|
||||
assert (handle != null): "node handle is null";
|
||||
assert (handle.index >= 0): "node handle too low: " + handle.index;
|
||||
//assert (handle.index < USAGE.allCount()) : "node handle too high: " + handle.index + ", USEDC=" + USAGE.USEDC + ", FREEC=" + USAGE.FREEC;
|
||||
|
||||
// the parentNode can be given if an auto-fix in the following case is wanted
|
||||
if (handle == null) throw new kelondroException(filename, "INTERNAL ERROR: node handle is null.");
|
||||
if (handle.index >= USAGE.allCount()) {
|
||||
if (parentNode == null) throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index " + handle.index + " exceeds size. No auto-fix node was submitted. This is a serious failure.");
|
||||
try {
|
||||
parentNode.setOHHandle(referenceInParent, null);
|
||||
parentNode.commit();
|
||||
logWarning("INTERNAL ERROR, Node/init in " + filename + ": node handle index " + handle.index + " exceeds size. The bad node has been auto-fixed");
|
||||
} catch (final IOException ee) {
|
||||
throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index " + handle.index + " exceeds size. It was tried to fix the bad node, but failed with an IOException: " + ee.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// use given handle
|
||||
this.handle = new RecordHandle(handle.index);
|
||||
|
||||
// check for memory availability when fillTail is requested
|
||||
if ((fillTail) && (tailchunksize > 10000)) fillTail = false; // this is a fail-safe 'short version' of a memory check
|
||||
|
||||
// init the content
|
||||
// create chunks; read them from file or cache
|
||||
this.tailChunk = null;
|
||||
if (cacheHeaders == null) {
|
||||
if (fillTail) {
|
||||
// read complete record
|
||||
byte[] chunkbuffer = new byte[recordsize];
|
||||
entryFile.readFully(seekpos(this.handle), chunkbuffer, 0, recordsize);
|
||||
this.headChunk = new byte[headchunksize];
|
||||
this.tailChunk = new byte[tailchunksize];
|
||||
System.arraycopy(chunkbuffer, 0, this.headChunk, 0, headchunksize);
|
||||
System.arraycopy(chunkbuffer, headchunksize, this.tailChunk, 0, tailchunksize);
|
||||
} else {
|
||||
// read overhead and key
|
||||
this.headChunk = new byte[headchunksize];
|
||||
this.tailChunk = null;
|
||||
entryFile.readFully(seekpos(this.handle), this.headChunk, 0, headchunksize);
|
||||
}
|
||||
} else synchronized(cacheHeaders) {
|
||||
byte[] cacheEntry = null;
|
||||
cacheEntry = cacheHeaders.getb(this.handle.index);
|
||||
if (cacheEntry == null) {
|
||||
// cache miss, we read overhead and key from file
|
||||
readMiss++;
|
||||
if (fillTail) {
|
||||
// read complete record
|
||||
byte[] chunkbuffer = new byte[recordsize];
|
||||
entryFile.readFully(seekpos(this.handle), chunkbuffer, 0, recordsize);
|
||||
this.headChunk = new byte[headchunksize];
|
||||
this.tailChunk = new byte[tailchunksize];
|
||||
System.arraycopy(chunkbuffer, 0, this.headChunk, 0, headchunksize);
|
||||
System.arraycopy(chunkbuffer, headchunksize, this.tailChunk, 0, tailchunksize);
|
||||
} else {
|
||||
// read overhead and key
|
||||
this.headChunk = new byte[headchunksize];
|
||||
this.tailChunk = null;
|
||||
entryFile.readFully(seekpos(this.handle), this.headChunk, 0, headchunksize);
|
||||
}
|
||||
|
||||
// if space left in cache, copy these value to the cache
|
||||
updateNodeCache();
|
||||
} else {
|
||||
readHit++;
|
||||
this.headChunk = cacheEntry;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void setValue(final byte[] value, final int valueoffset, int valuewidth, final byte[] targetarray, int targetoffset) {
|
||||
if (value == null) {
|
||||
while (valuewidth-- > 0) targetarray[targetoffset++] = 0;
|
||||
} else {
|
||||
assert ((valueoffset >= 0) && (valueoffset < value.length)) : "valueoffset = " + valueoffset;
|
||||
assert ((valueoffset + valuewidth <= value.length)) : "valueoffset = " + valueoffset + ", valuewidth = " + valuewidth + ", value.length = " + value.length;
|
||||
assert ((targetoffset >= 0) && (targetoffset < targetarray.length)) : "targetoffset = " + targetoffset;
|
||||
assert ((targetoffset + valuewidth <= targetarray.length)) : "targetoffset = " + targetoffset + ", valuewidth = " + valuewidth + ", targetarray.length = " + targetarray.length;
|
||||
System.arraycopy(value, valueoffset, targetarray, targetoffset, Math.min(value.length, valuewidth)); // error?
|
||||
while (valuewidth-- > value.length) targetarray[targetoffset + valuewidth] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public RecordHandle handle() {
|
||||
// if this entry has an index, return it
|
||||
if (this.handle.index == RecordHandle.NUL) throw new kelondroException(filename, "the entry has no index assigned");
|
||||
return this.handle;
|
||||
}
|
||||
|
||||
public void setOHByte(final int i, final byte b) {
|
||||
if (i >= OHBYTEC) throw new IllegalArgumentException("setOHByte: wrong index " + i);
|
||||
if (this.handle.index == RecordHandle.NUL) throw new kelondroException(filename, "setOHByte: no handle assigned");
|
||||
this.headChunk[i] = b;
|
||||
this.headChanged = true;
|
||||
}
|
||||
|
||||
public void setOHHandle(final int i, final RecordHandle otherhandle) {
|
||||
assert (i < OHHANDLEC): "setOHHandle: wrong array size " + i;
|
||||
assert (this.handle.index != RecordHandle.NUL): "setOHHandle: no handle assigned ind file" + filename;
|
||||
if (otherhandle == null) {
|
||||
NUL2bytes(this.headChunk, OHBYTEC + 4 * i);
|
||||
} else {
|
||||
if (otherhandle.index >= USAGE.allCount()) throw new kelondroException(filename, "INTERNAL ERROR, setOHHandles: handle " + i + " exceeds file size (" + handle.index + " >= " + USAGE.allCount() + ")");
|
||||
int2bytes(otherhandle.index, this.headChunk, OHBYTEC + 4 * i);
|
||||
}
|
||||
this.headChanged = true;
|
||||
}
|
||||
|
||||
public byte getOHByte(final int i) {
|
||||
if (i >= OHBYTEC) throw new IllegalArgumentException("getOHByte: wrong index " + i);
|
||||
if (this.handle.index == RecordHandle.NUL) throw new kelondroException(filename, "Cannot load OH values");
|
||||
return this.headChunk[i];
|
||||
}
|
||||
|
||||
public RecordHandle getOHHandle(final int i) {
|
||||
if (this.handle.index == RecordHandle.NUL) throw new kelondroException(filename, "Cannot load OH values");
|
||||
assert (i < OHHANDLEC): "handle index out of bounds: " + i + " in file " + filename;
|
||||
final int h = bytes2int(this.headChunk, OHBYTEC + 4 * i);
|
||||
return (h == RecordHandle.NUL) ? null : new RecordHandle(h);
|
||||
}
|
||||
|
||||
public synchronized void setValueRow(final byte[] row) throws IOException {
|
||||
// if the index is defined, then write values directly to the file, else only to the object
|
||||
if ((row != null) && (row.length != ROW.objectsize)) throw new IOException("setValueRow with wrong (" + row.length + ") row length instead correct: " + ROW.objectsize);
|
||||
|
||||
// set values
|
||||
if (this.handle.index != RecordHandle.NUL) {
|
||||
setValue(row, 0, ROW.width(0), headChunk, overhead);
|
||||
if (ROW.columns() > 1) setValue(row, ROW.width(0), tailchunksize, tailChunk, 0);
|
||||
}
|
||||
this.headChanged = true;
|
||||
this.tailChanged = true;
|
||||
}
|
||||
|
||||
public synchronized boolean valid() {
|
||||
// returns true if the key starts with non-zero byte
|
||||
// this may help to detect deleted entries
|
||||
return (headChunk[overhead] != 0) && ((headChunk[overhead] != -128) || (headChunk[overhead + 1] != 0));
|
||||
}
|
||||
|
||||
public synchronized byte[] getKey() {
|
||||
// read key
|
||||
return trimCopy(headChunk, overhead, ROW.width(0));
|
||||
}
|
||||
|
||||
public synchronized byte[] getValueRow() throws IOException {
|
||||
|
||||
if (this.tailChunk == null) {
|
||||
// load all values from the database file
|
||||
this.tailChunk = new byte[tailchunksize];
|
||||
// read values
|
||||
entryFile.readFully(seekpos(this.handle) + headchunksize, this.tailChunk, 0, this.tailChunk.length);
|
||||
}
|
||||
|
||||
// create return value
|
||||
final byte[] row = new byte[ROW.objectsize];
|
||||
|
||||
// read key
|
||||
System.arraycopy(headChunk, overhead, row, 0, ROW.width(0));
|
||||
|
||||
// read remaining values
|
||||
System.arraycopy(tailChunk, 0, row, ROW.width(0), tailchunksize);
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
public synchronized void commit() throws IOException {
|
||||
// this must be called after all write operations to the node are
|
||||
// finished
|
||||
|
||||
// place the data to the file
|
||||
|
||||
if (this.headChunk == null) {
|
||||
// there is nothing to save
|
||||
throw new kelondroException(filename, "no values to save (header missing)");
|
||||
}
|
||||
|
||||
final boolean doCommit = this.headChanged || this.tailChanged;
|
||||
|
||||
// save head
|
||||
synchronized (entryFile) {
|
||||
if (this.headChanged) {
|
||||
//System.out.println("WRITEH(" + filename + ", " + seekpos(this.handle) + ", " + this.headChunk.length + ")");
|
||||
assert (headChunk == null) || (headChunk.length == headchunksize);
|
||||
entryFile.write(seekpos(this.handle), (this.headChunk == null) ? new byte[headchunksize] : this.headChunk);
|
||||
updateNodeCache();
|
||||
this.headChanged = false;
|
||||
}
|
||||
|
||||
// save tail
|
||||
if ((this.tailChunk != null) && (this.tailChanged)) {
|
||||
//System.out.println("WRITET(" + filename + ", " + (seekpos(this.handle) + headchunksize) + ", " + this.tailChunk.length + ")");
|
||||
assert (tailChunk == null) || (tailChunk.length == tailchunksize);
|
||||
entryFile.write(seekpos(this.handle) + headchunksize, (this.tailChunk == null) ? new byte[tailchunksize] : this.tailChunk);
|
||||
this.tailChanged = false;
|
||||
}
|
||||
|
||||
if (doCommit) entryFile.commit();
|
||||
}
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
if (this.handle.index == RecordHandle.NUL) return "NULL";
|
||||
String s = Integer.toHexString(this.handle.index);
|
||||
RecordHandle h;
|
||||
while (s.length() < 4) s = "0" + s;
|
||||
try {
|
||||
for (int i = 0; i < OHBYTEC; i++) s = s + ":b" + getOHByte(i);
|
||||
for (int i = 0; i < OHHANDLEC; i++) {
|
||||
h = getOHHandle(i);
|
||||
if (h == null) s = s + ":hNULL"; else s = s + ":h" + h.toString();
|
||||
}
|
||||
final Row.Entry content = row().newEntry(getValueRow());
|
||||
for (int i = 0; i < row().columns(); i++) s = s + ":" + ((content.empty(i)) ? "NULL" : content.getColString(i, "UTF-8").trim());
|
||||
} catch (final IOException e) {
|
||||
s = s + ":***LOAD ERROR***:" + e.getMessage();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
private boolean cacheSpace() {
|
||||
// check for space in cache
|
||||
// should be only called within a synchronized(cacheHeaders) environment
|
||||
// returns true if it is allowed to add another entry to the cache
|
||||
// returns false if the cache is considered to be full
|
||||
if (cacheHeaders == null) return false; // no caching
|
||||
if (cacheHeaders.size() == 0) return true; // nothing there to flush
|
||||
if (cacheGrowStatus() == 2) return true; // no need to flush cache space
|
||||
|
||||
// just delete any of the entries
|
||||
if (cacheGrowStatus() <= 1) synchronized (cacheHeaders) {
|
||||
cacheHeaders.removeoneb();
|
||||
cacheFlush++;
|
||||
}
|
||||
return cacheGrowStatus() > 0;
|
||||
}
|
||||
|
||||
private void updateNodeCache() {
|
||||
if (this.handle == null) return; // wrong access
|
||||
if (this.headChunk == null) return; // nothing there to cache
|
||||
if (cacheHeaders == null) return; // we do not use the cache
|
||||
if (cacheSpace()) synchronized (cacheHeaders) {
|
||||
// generate cache entry
|
||||
//byte[] cacheEntry = new byte[headchunksize];
|
||||
//System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize);
|
||||
|
||||
// store the cache entry
|
||||
boolean upd = false;
|
||||
upd = (cacheHeaders.putb(this.handle.index, headChunk) != null);
|
||||
if (upd) writeDouble++; else writeUnique++;
|
||||
|
||||
//System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : ""));
|
||||
//printCache();
|
||||
} else {
|
||||
// there shall be no entry in the cache. If one exists, we remove it
|
||||
boolean rem = false;
|
||||
rem = (cacheHeaders.removeb(this.handle.index) != null);
|
||||
if (rem) cacheDelete++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,227 +0,0 @@
|
||||
// kelondroArray.java
|
||||
// ------------------
|
||||
// part of the Kelondro Database
|
||||
// (C) by Michael Peter Christen; mc@yacy.net
|
||||
// first published on http://www.anomic.de
|
||||
// Frankfurt, Germany, 2005
|
||||
// last major change: 20.06.2005
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
/*
|
||||
This class extends the kelondroRecords and adds a array structure
|
||||
*/
|
||||
|
||||
package de.anomic.kelondro.table;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.kelondro.index.ObjectArray;
|
||||
import de.anomic.kelondro.index.Row;
|
||||
import de.anomic.kelondro.io.RandomAccessInterface;
|
||||
import de.anomic.kelondro.order.NaturalOrder;
|
||||
import de.anomic.kelondro.util.FileUtils;
|
||||
|
||||
public class FixedWidthArray extends FullRecords implements ObjectArray {
|
||||
|
||||
// define the Over-Head-Array
|
||||
private static short thisOHBytes = 0; // our record definition does not need extra bytes
|
||||
private static short thisOHHandles = 0; // and no handles
|
||||
|
||||
public FixedWidthArray(final File file, final Row rowdef, final int intprops) throws IOException {
|
||||
// this creates a new array
|
||||
//super(file, true, -1, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
|
||||
super(file, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
|
||||
if (!(super.fileExisted)) {
|
||||
for (int i = 0; i < intprops; i++) {
|
||||
setHandle(i, new RecordHandle(RecordHandle.NUL));
|
||||
}
|
||||
// store column description
|
||||
for (int i = 0; i < rowdef.columns(); i++) {
|
||||
try {super.setText(i, rowdef.column(i).toString().getBytes());} catch (final IOException e) {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public FixedWidthArray(final RandomAccessInterface ra, final String filename, final Row rowdef, final int intprops) throws IOException {
|
||||
// this creates a new array
|
||||
//super(ra, filename, true, -1, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, false);
|
||||
super(ra, filename, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, false);
|
||||
for (int i = 0; i < intprops; i++) {
|
||||
setHandle(i, new RecordHandle(0));
|
||||
}
|
||||
// store column description
|
||||
for (int i = 0; i < rowdef.columns(); i++) {
|
||||
try {super.setText(i, rowdef.column(i).toString().getBytes());} catch (final IOException e) {}
|
||||
}
|
||||
}
|
||||
|
||||
public static FixedWidthArray open(final File file, final Row rowdef, final int intprops) {
|
||||
try {
|
||||
return new FixedWidthArray(file, rowdef, intprops);
|
||||
} catch (final IOException e) {
|
||||
FileUtils.deletedelete(file);
|
||||
try {
|
||||
return new FixedWidthArray(file, rowdef, intprops);
|
||||
} catch (final IOException ee) {
|
||||
e.printStackTrace();
|
||||
ee.printStackTrace();
|
||||
System.exit(-1);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void set(final int index, final Row.Entry rowentry) throws IOException {
|
||||
// this writes a row without reading the row from the file system first
|
||||
|
||||
// create a node at position index with rowentry
|
||||
final RecordHandle h = new RecordHandle(index);
|
||||
(new EcoNode(h, (rowentry == null) ? null : rowentry.bytes(), 0)).commit();
|
||||
// attention! this newNode call wants that the OH bytes are passed within the bulkchunk
|
||||
// field. Here, only the rowentry.bytes() raw payload is passed. This is valid, because
|
||||
// the OHbytes and OHhandles are zero.
|
||||
}
|
||||
|
||||
public synchronized void setMultiple(final TreeMap<Integer, Row.Entry> rows) throws IOException {
|
||||
final Iterator<Map.Entry<Integer, Row.Entry>> i = rows.entrySet().iterator();
|
||||
Map.Entry<Integer, Row.Entry> entry;
|
||||
int k;
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
k = entry.getKey().intValue();
|
||||
set(k, entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized Row.Entry getIfValid(final int index) throws IOException {
|
||||
final byte[] b = (new EcoNode(new RecordHandle(index))).getValueRow();
|
||||
if (b[0] == 0) return null;
|
||||
if ((b[0] == -128) && (b[1] == 0)) return null;
|
||||
return row().newEntry(b);
|
||||
}
|
||||
|
||||
public synchronized Row.Entry get(final int index) throws IOException {
|
||||
return row().newEntry(new EcoNode(new RecordHandle(index)).getValueRow());
|
||||
}
|
||||
|
||||
protected synchronized int seti(final int index, final int value) throws IOException {
|
||||
final int before = getHandle(index).hashCode();
|
||||
setHandle(index, new RecordHandle(value));
|
||||
return before;
|
||||
}
|
||||
|
||||
protected synchronized int geti(final int index) {
|
||||
return getHandle(index).hashCode();
|
||||
}
|
||||
|
||||
public synchronized int add(final Row.Entry rowentry) throws IOException {
|
||||
// adds a new rowentry, but re-uses a previously as-deleted marked entry
|
||||
final Node n = new EcoNode(rowentry.bytes());
|
||||
n.commit();
|
||||
return n.handle().hashCode();
|
||||
}
|
||||
|
||||
public synchronized void remove(final int index) throws IOException {
|
||||
assert (index < (super.free() + super.size())) : "remove: index " + index + " out of bounds " + (super.free() + super.size());
|
||||
|
||||
// get the node at position index
|
||||
final RecordHandle h = new RecordHandle(index);
|
||||
final Node n = new EcoNode(h);
|
||||
|
||||
// erase the row
|
||||
n.setValueRow(null);
|
||||
n.commit();
|
||||
|
||||
// mark row as deleted so it can be re-used
|
||||
deleteNode(h);
|
||||
}
|
||||
|
||||
public void print() throws IOException {
|
||||
System.out.println("PRINTOUT of table, length=" + size());
|
||||
Row.Entry row;
|
||||
for (int i = 0; i < (super.free() + super.size()); i++) {
|
||||
System.out.print("row " + i + ": ");
|
||||
row = get(i);
|
||||
for (int j = 0; j < row.columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
|
||||
System.out.println();
|
||||
}
|
||||
System.out.println("EndOfTable");
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
//File f = new File("d:\\\\mc\\privat\\fixtest.db");
|
||||
final File f = new File("/Users/admin/fixtest.db");
|
||||
final Row rowdef = new Row("byte[] a-12, byte[] b-4", NaturalOrder.naturalOrder);
|
||||
try {
|
||||
System.out.println("erster Test");
|
||||
f.delete();
|
||||
FixedWidthArray k = new FixedWidthArray(f, rowdef, 6);
|
||||
k.set(3, k.row().newEntry(new byte[][]{
|
||||
"test123".getBytes(), "abcd".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{
|
||||
"test456".getBytes(), "efgh".getBytes()}));
|
||||
k.close();
|
||||
|
||||
k = new FixedWidthArray(f, rowdef, 6);
|
||||
System.out.println(k.get(2).toString());
|
||||
System.out.println(k.get(3).toString());
|
||||
k.close();
|
||||
|
||||
System.out.println("zweiter Test");
|
||||
f.delete();
|
||||
k = new FixedWidthArray(f, rowdef, 6);
|
||||
k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()}));
|
||||
k.remove(0);
|
||||
|
||||
k.add(k.row().newEntry(new byte[][]{"c".getBytes(), "xxxx".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{"d".getBytes(), "xxxx".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{"e".getBytes(), "xxxx".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{"f".getBytes(), "xxxx".getBytes()}));
|
||||
k.remove(0);
|
||||
k.remove(1);
|
||||
|
||||
k.print();
|
||||
k.print();
|
||||
k.close();
|
||||
|
||||
|
||||
System.out.println("dritter Test");
|
||||
f.delete();
|
||||
k = new FixedWidthArray(f, rowdef, 6);
|
||||
for (int i = 1; i <= 200; i = i * 2) {
|
||||
for (int j = 0; j < i*2; j++) {
|
||||
k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()}));
|
||||
}
|
||||
for (int j = 0; j < i; j++) {
|
||||
k.remove(j);
|
||||
}
|
||||
}
|
||||
k.print();
|
||||
k.print();
|
||||
k.close();
|
||||
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,430 +0,0 @@
|
||||
// kelondroFlexWidthArray.java
|
||||
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 01.06.2006 on http://www.anomic.de
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
|
||||
package de.anomic.kelondro.table;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.kelondro.index.ObjectArray;
|
||||
import de.anomic.kelondro.index.Column;
|
||||
import de.anomic.kelondro.index.Row;
|
||||
import de.anomic.kelondro.order.NaturalOrder;
|
||||
import de.anomic.kelondro.util.FileUtils;
|
||||
import de.anomic.kelondro.util.kelondroException;
|
||||
import de.anomic.yacy.logging.Log;
|
||||
|
||||
public class FlexWidthArray implements ObjectArray {
|
||||
|
||||
protected FixedWidthArray[] col;
|
||||
protected Row rowdef;
|
||||
protected File path;
|
||||
protected String tablename;
|
||||
protected String filename;
|
||||
|
||||
public FlexWidthArray(final File path, final String tablename, final Row rowdef, final boolean resetOnFail) {
|
||||
this.path = path;
|
||||
this.rowdef = rowdef;
|
||||
this.tablename = tablename;
|
||||
try {
|
||||
init();
|
||||
} catch (final IOException e) {
|
||||
if (resetOnFail) {
|
||||
Log.logSevere("kelondroFlexWidthArray", "IOException during initialization of " + new File(path, tablename).toString() + ": reset");
|
||||
delete(path, tablename);
|
||||
try {
|
||||
init();
|
||||
} catch (final IOException e1) {
|
||||
e1.printStackTrace();
|
||||
throw new kelondroException("IOException during initialization of " + new File(path, tablename).toString() + ": cannot reset: " + e1.getMessage());
|
||||
}
|
||||
} else {
|
||||
throw new kelondroException("IOException during initialization of " + new File(path, tablename).toString() + ": not allowed to reset: " + e.getMessage());
|
||||
}
|
||||
} catch (final kelondroException e) {
|
||||
if (resetOnFail) {
|
||||
Log.logSevere("kelondroFlexWidthArray", "kelondroException during initialization of " + new File(path, tablename).toString() + ": reset");
|
||||
delete(path, tablename);
|
||||
try {
|
||||
init();
|
||||
} catch (final IOException e1) {
|
||||
e1.printStackTrace();
|
||||
throw new kelondroException("kelondroException during initialization of " + new File(path, tablename).toString() + ": cannot reset: " + e1.getMessage());
|
||||
}
|
||||
} else {
|
||||
throw new kelondroException("kelondroException during initialization of " + new File(path, tablename).toString() + ": not allowed to reset: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void init() throws IOException {
|
||||
|
||||
// initialize columns
|
||||
col = new FixedWidthArray[rowdef.columns()];
|
||||
String check = "";
|
||||
for (int i = 0; i < rowdef.columns(); i++) {
|
||||
col[i] = null;
|
||||
check += '_';
|
||||
}
|
||||
|
||||
// check if table directory exists
|
||||
final File tabledir = new File(path, tablename);
|
||||
if (tabledir.exists()) {
|
||||
if (!(tabledir.isDirectory())) throw new IOException("path " + tabledir.toString() + " must be a directory");
|
||||
} else {
|
||||
tabledir.mkdirs();
|
||||
tabledir.mkdir();
|
||||
}
|
||||
this.filename = tabledir.getCanonicalPath();
|
||||
|
||||
// save/check property file for this array
|
||||
/*
|
||||
final File propfile = new File(tabledir, "properties");
|
||||
Map<String, String> props = new HashMap<String, String>();
|
||||
if (propfile.exists()) {
|
||||
props = serverFileUtils.loadMap(propfile);
|
||||
final String stored_rowdef = props.get("rowdef");
|
||||
if ((stored_rowdef != null) && (!(rowdef.subsumes(new Row(stored_rowdef, rowdef.objectOrder, 0))))) {
|
||||
System.out.println("FATAL ERROR: stored rowdef '" + stored_rowdef + "' does not match with new rowdef '" +
|
||||
rowdef + "' for flex table '" + path + "', table " + tablename);
|
||||
System.exit(-1);
|
||||
}
|
||||
}
|
||||
props.put("rowdef", rowdef.toString());
|
||||
serverFileUtils.saveMap(propfile, props, "FlexWidthArray properties");
|
||||
*/
|
||||
|
||||
// open existing files
|
||||
final String[] files = tabledir.list();
|
||||
for (int i = 0; i < files.length; i++) {
|
||||
if ((files[i].startsWith("col.") && (files[i].endsWith(".list")))) {
|
||||
final int colstart = Integer.parseInt(files[i].substring(4, 7));
|
||||
final int colend = (files[i].charAt(7) == '-') ? Integer.parseInt(files[i].substring(8, 11)) : colstart;
|
||||
|
||||
final Column columns[] = new Column[colend - colstart + 1];
|
||||
for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.column(j);
|
||||
col[colstart] = new FixedWidthArray(new File(tabledir, files[i]), new Row(columns, (colstart == 0) ? rowdef.objectOrder : NaturalOrder.naturalOrder), 16);
|
||||
for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// check if all columns are there
|
||||
int p, q;
|
||||
while ((p = check.indexOf('_')) >= 0) {
|
||||
q = p;
|
||||
if (p != 0) {
|
||||
while ((q <= check.length() - 1) && (check.charAt(q) == '_')) q++;
|
||||
q--;
|
||||
}
|
||||
// create new array file
|
||||
final Column[] columns = new Column[q - p + 1];
|
||||
for (int j = p; j <= q; j++) {
|
||||
columns[j - p] = rowdef.column(j);
|
||||
check = check.substring(0, j) + "X" + check.substring(j + 1);
|
||||
}
|
||||
col[p] = new FixedWidthArray(new File(tabledir, colfilename(p, q)), new Row(columns, (p == 0) ? rowdef.objectOrder : NaturalOrder.naturalOrder), 16);
|
||||
}
|
||||
}
|
||||
|
||||
public final String filename() {
|
||||
return this.filename;
|
||||
}
|
||||
|
||||
public static int staticsize(final File path, final String tablename) {
|
||||
|
||||
// check if table directory exists
|
||||
final File tabledir = new File(path, tablename);
|
||||
if (tabledir.exists()) {
|
||||
if (!(tabledir.isDirectory())) return 0;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// open existing files
|
||||
final File file = new File(tabledir, "col.000.list");
|
||||
return AbstractRecords.staticsize(file);
|
||||
}
|
||||
|
||||
public static void delete(final File path, final String tablename) {
|
||||
final File tabledir = new File(path, tablename);
|
||||
if (!(tabledir.exists())) return;
|
||||
if ((!(tabledir.isDirectory()))) {
|
||||
FileUtils.deletedelete(tabledir);
|
||||
return;
|
||||
}
|
||||
|
||||
final String[] files = tabledir.list();
|
||||
for (int i = 0; i < files.length; i++) {
|
||||
FileUtils.deletedelete(new File(tabledir, files[i]));
|
||||
}
|
||||
|
||||
FileUtils.deletedelete(tabledir);
|
||||
}
|
||||
|
||||
public void reset() throws IOException {
|
||||
this.close();
|
||||
delete(path, tablename);
|
||||
this.init();
|
||||
}
|
||||
|
||||
public synchronized void close() {
|
||||
if (col != null) {
|
||||
for (int i = 0; i < col.length; i++) {
|
||||
if (col[i] != null) {
|
||||
// a column can be null, this is normal
|
||||
col[i].close();
|
||||
col[i] = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected static final String colfilename(final int start, final int end) {
|
||||
String f = Integer.toString(end);
|
||||
while (f.length() < 3) f = "0" + f;
|
||||
if (start == end) return "col." + f + ".list";
|
||||
f = Integer.toString(start) + "-" + f;
|
||||
while (f.length() < 7) f = "0" + f;
|
||||
return "col." + f + ".list";
|
||||
}
|
||||
|
||||
|
||||
public Row row() {
|
||||
return rowdef;
|
||||
}
|
||||
|
||||
public int size() {
|
||||
//assert ((rowdef.columns() == 1) || (col[0].size() == col[1].size())) : "col[0].size() = " + col[0].size() + ", col[1].size() = " + col[1].size() + ", file = " + filename;
|
||||
return col[0].size();
|
||||
}
|
||||
|
||||
public synchronized void setMultiple(final TreeMap<Integer, Row.Entry> entries) throws IOException {
|
||||
// a R/W head path-optimized option to write a set of entries
|
||||
Iterator<Map.Entry<Integer, Row.Entry>> i;
|
||||
Map.Entry<Integer, Row.Entry> entry;
|
||||
Row.Entry rowentry, e;
|
||||
int c = 0, index;
|
||||
// go across each file
|
||||
while (c < rowdef.columns()) {
|
||||
i = entries.entrySet().iterator();
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
index = entry.getKey().intValue();
|
||||
rowentry = entry.getValue();
|
||||
assert rowentry.objectsize() == this.rowdef.objectsize;
|
||||
|
||||
e = col[c].row().newEntry(rowentry.bytes(), rowdef.colstart[c], false);
|
||||
col[c].set(index, e);
|
||||
}
|
||||
c = c + col[c].row().columns();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void set(final int index, final Row.Entry rowentry) throws IOException {
|
||||
assert rowentry.objectsize() == this.rowdef.objectsize;
|
||||
int c = 0;
|
||||
Row.Entry e;
|
||||
final byte[] reb = rowentry.bytes();
|
||||
while (c < rowdef.columns()) {
|
||||
e = col[c].row().newEntry(reb, rowdef.colstart[c], false);
|
||||
col[c].set(index, e);
|
||||
c = c + col[c].row().columns();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized int add(final Row.Entry rowentry) throws IOException {
|
||||
assert rowentry.objectsize() == this.rowdef.objectsize;
|
||||
int index = -1;
|
||||
final byte[] reb = rowentry.bytes();
|
||||
index = col[0].add(col[0].row().newEntry(reb, 0, false));
|
||||
int c = col[0].row().columns();
|
||||
|
||||
while (c < rowdef.columns()) {
|
||||
col[c].set(index, col[c].row().newEntry(reb, rowdef.colstart[c], false));
|
||||
c = c + col[c].row().columns();
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected synchronized TreeMap<Integer, byte[]> addMultiple(final List<Row.Entry> rows) throws IOException {
|
||||
// result is a Integer/byte[] relation
|
||||
// of newly added rows (index, key)
|
||||
final TreeMap<Integer, byte[]> indexref = new TreeMap<Integer, byte[]>();
|
||||
Iterator<Row.Entry> i;
|
||||
Row.Entry rowentry;
|
||||
// prepare storage for other columns
|
||||
final TreeMap<Integer, Row.Entry>[] colm = new TreeMap[col.length];
|
||||
for (int j = 0; j < col.length; j++) {
|
||||
if (col[j] == null) colm[j] = null; else colm[j] = new TreeMap<Integer, Row.Entry>();
|
||||
}
|
||||
i = rows.iterator();
|
||||
while (i.hasNext()) {
|
||||
rowentry = i.next();
|
||||
assert rowentry.objectsize() == this.rowdef.objectsize;
|
||||
|
||||
Row.Entry e;
|
||||
int index = -1;
|
||||
final byte[] reb = rowentry.bytes();
|
||||
e = col[0].row().newEntry(reb, 0, false);
|
||||
index = col[0].add(e);
|
||||
int c = col[0].row().columns();
|
||||
|
||||
while (c < rowdef.columns()) {
|
||||
e = col[c].row().newEntry(reb, rowdef.colstart[c], false);
|
||||
// remember write to column, but do not write directly
|
||||
colm[c].put(Integer.valueOf(index), e); // col[c].set(index,e);
|
||||
c = c + col[c].row().columns();
|
||||
}
|
||||
indexref.put(Integer.valueOf(index), rowentry.getColBytes(0));
|
||||
}
|
||||
// write the other columns
|
||||
for (int j = 1; j < col.length; j++) {
|
||||
if (col[j] != null) col[j].setMultiple(colm[j]);
|
||||
}
|
||||
// return references to entries with key
|
||||
return indexref;
|
||||
}
|
||||
|
||||
public synchronized Row.Entry get(final int index) throws IOException {
|
||||
Row.Entry e = col[0].getIfValid(index);
|
||||
//assert e != null;
|
||||
if (e == null) return null; // probably a deleted entry
|
||||
final Row.Entry p = rowdef.newEntry();
|
||||
p.setCol(0, e.getColBytes(0));
|
||||
int r = col[0].row().columns();
|
||||
while (r < rowdef.columns()) {
|
||||
e = col[r].get(index);
|
||||
for (int i = 0; i < col[r].row().columns(); i++) {
|
||||
p.setCol(r + i, e.getColBytes(i));
|
||||
}
|
||||
r = r + col[r].row().columns();
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
public synchronized Row.Entry getOmitCol0(final int index, final byte[] col0) throws IOException {
|
||||
assert col[0].row().columns() == 1;
|
||||
final Row.Entry p = rowdef.newEntry();
|
||||
Row.Entry e;
|
||||
p.setCol(0, col0);
|
||||
int r = 1;
|
||||
while (r < rowdef.columns()) {
|
||||
e = col[r].get(index);
|
||||
for (int i = 0; i < col[r].row().columns(); i++) {
|
||||
p.setCol(r + i, e.getColBytes(i));
|
||||
}
|
||||
r = r + col[r].row().columns();
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
public synchronized void remove(final int index) throws IOException {
|
||||
int r = 0;
|
||||
|
||||
// remove only from the first column
|
||||
col[0].remove(index);
|
||||
r = r + col[r].row().columns();
|
||||
|
||||
// the other columns will be blanked out only
|
||||
while (r < rowdef.columns()) {
|
||||
col[r].set(index, null);
|
||||
r = r + col[r].row().columns();
|
||||
}
|
||||
}
|
||||
|
||||
public void print() throws IOException {
|
||||
System.out.println("PRINTOUT of table, length=" + size());
|
||||
Row.Entry row;
|
||||
for (int i = 0; i < (col[0].free() + col[0].size()); i++) {
|
||||
System.out.print("row " + i + ": ");
|
||||
row = get(i);
|
||||
System.out.println(row.toString());
|
||||
//for (int j = 0; j < row().columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
|
||||
//System.out.println();
|
||||
}
|
||||
System.out.println("EndOfTable");
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
//File f = new File("d:\\\\mc\\privat\\fixtest.db");
|
||||
final File f = new File("/Users/admin/");
|
||||
final Row rowdef = new Row("byte[] a-12, byte[] b-4", NaturalOrder.naturalOrder);
|
||||
final String testname = "flextest";
|
||||
try {
|
||||
System.out.println("erster Test");
|
||||
FlexWidthArray.delete(f, testname);
|
||||
FlexWidthArray k = new FlexWidthArray(f, "flextest", rowdef, true);
|
||||
k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()}));
|
||||
k.remove(0);
|
||||
|
||||
k.add(k.row().newEntry(new byte[][]{"c".getBytes(), "xxxx".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{"d".getBytes(), "xxxx".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{"e".getBytes(), "xxxx".getBytes()}));
|
||||
k.add(k.row().newEntry(new byte[][]{"f".getBytes(), "xxxx".getBytes()}));
|
||||
k.remove(0);
|
||||
k.remove(1);
|
||||
|
||||
k.print();
|
||||
k.col[0].print();
|
||||
k.col[1].print();
|
||||
k.close();
|
||||
|
||||
|
||||
System.out.println("zweiter Test");
|
||||
FlexWidthArray.delete(f, testname);
|
||||
//k = kelondroFlexWidthArray.open(f, "flextest", rowdef);
|
||||
for (int i = 1; i <= 20; i = i * 2) {
|
||||
System.out.println("LOOP: " + i);
|
||||
k = new FlexWidthArray(f, "flextest", rowdef, true);
|
||||
for (int j = 0; j < i*2; j++) {
|
||||
k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()}));
|
||||
}
|
||||
k.close();
|
||||
k = new FlexWidthArray(f, "flextest", rowdef, true);
|
||||
for (int j = 0; j < i; j++) {
|
||||
k.remove(i*2 - j - 1);
|
||||
}
|
||||
k.close();
|
||||
}
|
||||
k = new FlexWidthArray(f, "flextest", rowdef, true);
|
||||
k.print();
|
||||
k.col[0].print();
|
||||
k.close();
|
||||
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteOnExit() {
|
||||
for (int i = 0; i < this.col.length; i++) this.col[i].deleteOnExit();
|
||||
}
|
||||
}
|
||||
@ -1,253 +0,0 @@
|
||||
// kelondroHashtable.java
|
||||
// ------------------
|
||||
// part of the Kelondro Database
|
||||
// (C) by Michael Peter Christen; mc@yacy.net
|
||||
// first published on http://www.anomic.de
|
||||
// Frankfurt, Germany, 2005
|
||||
// last major change: 21.06.2005
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
/*
|
||||
|
||||
we implement a hashtable based on folded binary trees
|
||||
each hight in these binary trees represents one step of rehasing
|
||||
the re-hashing is realised by extending the number of relevant bits in the given hash
|
||||
We construct the binary tree as follows
|
||||
- there exists no root node
|
||||
- at height-1 are 2 nodes, and can be accessed by using only the least significant bit of the hash
|
||||
- at height-2 are 4 nodes, addresses by (hash & 3) - mapping the 2 lsb of the hash
|
||||
- at height-3 are 8 nodes, addresses by (hash & 7)
|
||||
- .. and so on.
|
||||
The number of nodes N(k) that are needed for a tree of height-k is
|
||||
|
||||
N(k) = 2**k + N(k-1) = 2**(k + 1) - 2 [where k > 0]
|
||||
|
||||
We fold this tree by putting all heights of the tree in a sequence
|
||||
|
||||
Computation of the position (the index) of a node:
|
||||
given:
|
||||
hash h, with k significant bits (representing a height-k): h|k
|
||||
then the position of a node node(h,k) is
|
||||
|
||||
node(h,k) = N(k - 1) + h|k [where k > 0]
|
||||
|
||||
We use these nodes to sequentially store a hash h at position node(h, 1), and
|
||||
if that fails on node(h, 2), node(h, 3) and so on.
|
||||
|
||||
This is highly inefficient for the most heights k = 1, ..., (?)
|
||||
The 'inefficient-border' depends on the number of elements that we want to store.
|
||||
|
||||
We therefore introduce an offset o which is the number of bits that are not used
|
||||
at the beginning of (re-)hashing. But even if these o re-hasing steps are not done,
|
||||
all bits of the hash are relevant.
|
||||
Now the number of nodes N(k) that are needed is computed by N(k,o):
|
||||
|
||||
N(k,o) = N(k) - N(o) = 2**(k + 1) - 2**(o + 1) [where k > o, o >= 0]
|
||||
|
||||
When o=0 then this is equivalent to N(k).
|
||||
|
||||
The node-formula must be adopted as well
|
||||
|
||||
node(h,k,o) = N(k - 1, o) + h|k [where k > o, o >= 0]
|
||||
|
||||
So if you set an offset o, this leads to a minimum number of nodes
|
||||
at level k=o+1: node(0,o + 1,o) = N(o, o) = 0 (position of the first entry)
|
||||
|
||||
Computatiion of the maxlen 'maxk', the maximum height of the tree for a given
|
||||
number of maximum entries 'maxsize' in the hashtable:
|
||||
maxk shall be computed in such a way, that N(k,o) <= maxsize, for any o or k
|
||||
This means paricualary, that
|
||||
|
||||
node(h,k,o) < maxsize
|
||||
|
||||
for h|k we must consider the worst case:
|
||||
|
||||
h|k (by maxk) = 2**k - 1
|
||||
|
||||
therefore
|
||||
|
||||
node(h,maxk,o) < maxsize
|
||||
N(maxk - 1, o) + h|maxk < maxsize [where maxk > o, o >= 0]
|
||||
2**maxk - 2**(o + 1) + 2**maxk - 1 < maxsize [where maxk > o, o >= 0]
|
||||
2**maxk - 2**(o + 1) + 2**maxk < maxsize + 1 [where maxk > o, o >= 0]
|
||||
2**maxk + 2**maxk < maxsize + 2**(o + 1) + 1 [where maxk > o, o >= 0]
|
||||
2**(maxk+1) < maxsize + 2**(o + 1) + 1 [where maxk > o, o >= 0]
|
||||
maxk < log2(maxsize + 2**(o + 1) + 1) [where maxk > o, o >= 0]
|
||||
|
||||
setting maxk to
|
||||
|
||||
maxk = log2(maxsize)
|
||||
|
||||
will make this relation true in any case, even if maxk = log2(maxsize) + 1
|
||||
would also be correct in some cases
|
||||
|
||||
Now we can use the following formula to create the folded binary hash tree:
|
||||
|
||||
node(h,k,o) = 2**k - 2**(o + 1) + h|k
|
||||
|
||||
to compute the node index and
|
||||
|
||||
maxk = log2(maxsize)
|
||||
|
||||
to compute the upper limit of re-hashing
|
||||
|
||||
|
||||
*/
|
||||
|
||||
package de.anomic.kelondro.table;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import de.anomic.kelondro.index.Column;
|
||||
import de.anomic.kelondro.index.Row;
|
||||
import de.anomic.kelondro.order.Base64Order;
|
||||
import de.anomic.kelondro.util.SetTools;
|
||||
|
||||
public class Hashtable {
|
||||
|
||||
private final FixedWidthArray hashArray;
|
||||
protected int offset;
|
||||
protected int maxk;
|
||||
private int maxrehash;
|
||||
private Row.Entry dummyRow;
|
||||
|
||||
private static final byte[] dummyKey = Base64Order.enhancedCoder.encodeLong(0, 5).getBytes();
|
||||
|
||||
public Hashtable(final File file, final Row rowdef, final int offset, final int maxsize, final int maxrehash) throws IOException {
|
||||
// this creates a new hashtable
|
||||
// the key element is not part of the columns array
|
||||
// this is unlike the kelondroTree, where the key is part of a row
|
||||
// the offset is a number of bits that is omitted in the folded tree hierarchy
|
||||
// a good number for offset is 8
|
||||
// the maxsize number is the maximum number of elements in the hashtable
|
||||
// this number is needed to omit grow of the table in case of re-hashing
|
||||
// the maxsize is re-computed to a virtual folding height and will result in a tablesize
|
||||
// less than the given maxsize. The actual maxsize can be retrieved by maxsize()
|
||||
final boolean fileExisted = file.exists();
|
||||
this.hashArray = new FixedWidthArray(file, extCol(rowdef), 6);
|
||||
if (fileExisted) {
|
||||
this.offset = hashArray.geti(0);
|
||||
this.maxk = hashArray.geti(1);
|
||||
this.maxrehash = hashArray.geti(2);
|
||||
} else {
|
||||
this.offset = offset;
|
||||
this.maxk = SetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
|
||||
if (this.maxk >= SetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
|
||||
this.maxrehash = maxrehash;
|
||||
dummyRow = this.hashArray.row().newEntry();
|
||||
dummyRow.setCol(0, dummyKey);
|
||||
//for (int i = 0; i < hashArray.row().columns(); i++)
|
||||
hashArray.seti(0, this.offset);
|
||||
hashArray.seti(1, this.maxk);
|
||||
hashArray.seti(2, this.maxrehash);
|
||||
}
|
||||
}
|
||||
|
||||
private Row extCol(final Row rowdef) {
|
||||
final Column[] newCol = new Column[rowdef.columns() + 1];
|
||||
newCol[0] = new Column("Cardinal key-4 {b256}");
|
||||
for (int i = 0; i < rowdef.columns(); i++) newCol[i + 1] = rowdef.column(i);
|
||||
return new Row(newCol, rowdef.objectOrder);
|
||||
}
|
||||
|
||||
public static int power2(int x) {
|
||||
int p = 1;
|
||||
while (x > 0) {p = p << 1; x--;}
|
||||
return p;
|
||||
}
|
||||
|
||||
public synchronized byte[][] get(final int key) throws IOException {
|
||||
final Object[] search = search(new Hash(key));
|
||||
if (search[1] == null) return null;
|
||||
final byte[][] row = (byte[][]) search[1];
|
||||
final byte[][] result = new byte[row.length - 1][];
|
||||
System.arraycopy(row, 1, result, 0, row.length - 1);
|
||||
return result;
|
||||
}
|
||||
|
||||
public synchronized Row.Entry put(final int key, final Row.Entry rowentry) throws IOException {
|
||||
final Hash hash = new Hash(key);
|
||||
|
||||
// find row
|
||||
final Object[] search = search(hash);
|
||||
Row.Entry oldhkrow;
|
||||
final int rowNumber = ((Integer) search[0]).intValue();
|
||||
if (search[1] == null) {
|
||||
oldhkrow = null;
|
||||
} else {
|
||||
oldhkrow = (Row.Entry) search[1];
|
||||
}
|
||||
|
||||
// make space
|
||||
while (rowNumber >= hashArray.size()) hashArray.set(hashArray.size(), dummyRow);
|
||||
|
||||
// write row
|
||||
final Row.Entry newhkrow = hashArray.row().newEntry();
|
||||
newhkrow.setCol(0, hash.key());
|
||||
newhkrow.setCol(1, rowentry.bytes());
|
||||
hashArray.set(rowNumber, newhkrow);
|
||||
return (oldhkrow == null ? null : hashArray.row().newEntry(oldhkrow.getColBytes(1)));
|
||||
}
|
||||
|
||||
private Object[] search(final Hash hash) throws IOException {
|
||||
Row.Entry hkrow;
|
||||
int rowKey;
|
||||
int rowNumber;
|
||||
do {
|
||||
rowNumber = hash.node();
|
||||
if (rowNumber >= hashArray.size()) return new Object[]{Integer.valueOf(rowNumber), null};
|
||||
hkrow = hashArray.get(rowNumber);
|
||||
rowKey = (int) hkrow.getColLong(0);
|
||||
if (rowKey == 0) return new Object[]{Integer.valueOf(rowNumber), null};
|
||||
hash.rehash();
|
||||
} while (rowKey != hash.key());
|
||||
return new Object[]{Integer.valueOf(rowNumber), hkrow};
|
||||
}
|
||||
|
||||
|
||||
private class Hash {
|
||||
int key;
|
||||
int hash;
|
||||
int depth;
|
||||
public Hash(final int key) {
|
||||
this.key = key;
|
||||
this.hash = key;
|
||||
this.depth = offset + 1;
|
||||
}
|
||||
public int key() {
|
||||
return key;
|
||||
}
|
||||
private int hash() {
|
||||
return hash & (power2(depth) - 1); // apply mask
|
||||
}
|
||||
public int depth() {
|
||||
return depth;
|
||||
}
|
||||
public void rehash() {
|
||||
depth++;
|
||||
if (depth > maxk) {
|
||||
depth = offset + 1;
|
||||
hash = (int) ((5 * (long) hash - 7) / 3 + 13);
|
||||
}
|
||||
}
|
||||
public int node() {
|
||||
// node(h,k,o) = 2**k - 2**(o + 1) + h|k
|
||||
return power2(depth) - power2(offset + 1) + hash();
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue