fine-tuning of cache usage from SVN 5386 and a bug fix for overflow in available() method

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5387 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 1779c3c507
commit 5b94498643

@ -51,17 +51,19 @@ abstract class kelondroAbstractRA implements kelondroRA {
// pseudo-native methods:
abstract public void readFully(byte[] b, int off, int len) throws IOException;
abstract public long length() throws IOException;
abstract public int available() throws IOException;
abstract public void setLength(long length) throws IOException;
abstract public long available() throws IOException;
abstract public void write(byte[] b, int off, int len) throws IOException;
abstract public void seek(long pos) throws IOException;
abstract public void close() throws IOException;
// derived methods:
public byte[] readFully() throws IOException {
int a = this.available();
long a = this.available();
if (a <= 0) return null;
final byte[] buffer = new byte[a];
this.readFully(buffer, 0, a);
if (a > Integer.MAX_VALUE) throw new IOException("available too large for a single array");
final byte[] buffer = new byte[(int) a];
this.readFully(buffer, 0, (int) a);
return buffer;
}

@ -970,7 +970,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
try {
nn = next00();
} catch (final IOException e) {
serverLog.logSevere("kelondroCachedRecords", filename + " failed with " + e.getMessage(), e);
serverLog.logSevere("kelondroAbstractRecords", filename + " failed with " + e.getMessage(), e);
return null;
}
byte[] key = null;
@ -1001,9 +1001,9 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
bulkstart = pos.index;
final int maxlength = Math.min(USAGE.allCount() - bulkstart, bulksize);
if (((POS_NODES) + ((long) bulkstart) * ((long) recordsize)) < 0)
serverLog.logSevere("kelondroCachedRecords", "DEBUG: negative offset. POS_NODES = " + POS_NODES + ", bulkstart = " + bulkstart + ", recordsize = " + recordsize);
serverLog.logSevere("kelondroAbstractRecords", "DEBUG: negative offset. POS_NODES = " + POS_NODES + ", bulkstart = " + bulkstart + ", recordsize = " + recordsize);
if ((maxlength * recordsize) < 0)
serverLog.logSevere("kelondroCachedRecords", "DEBUG: negative length. maxlength = " + maxlength + ", recordsize = " + recordsize);
serverLog.logSevere("kelondroAbstractRecords", "DEBUG: negative length. maxlength = " + maxlength + ", recordsize = " + recordsize);
entryFile.readFully((POS_NODES) + ((long) bulkstart) * ((long) recordsize), bulk, 0, maxlength * recordsize);
}
/* POS_NODES = 302, bulkstart = 3277, recordsize = 655386

@ -45,7 +45,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
private TreeMap<Long, Integer> free; // list of {size, seek} pairs denoting space and position of free records
private final File heapFile; // the file of the heap
private final kelondroByteOrder ordering; // the ordering on keys
private kelondroFileRA file; // a random access to the file
private kelondroCachedFileRA file; // a random access to the file
private HashMap<String, byte[]> buffer; // a write buffer to limit IO to the file; attention: Maps cannot use byte[] as key
private int buffersize; // bytes that are buffered in buffer
private int buffermax; // maximum size of the buffer
@ -89,7 +89,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
this.free = new TreeMap<Long, Integer>();
this.buffer = new HashMap<String, byte[]>();
this.buffersize = 0;
this.file = new kelondroFileRA(heapFile);
this.file = new kelondroCachedFileRA(heapFile);
byte[] key = new byte[keylength];
int reclen;
long seek = 0;
@ -356,7 +356,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
e.printStackTrace();
}
this.heapFile.delete();
this.file = new kelondroFileRA(heapFile);
this.file = new kelondroCachedFileRA(heapFile);
}
/**

@ -417,8 +417,12 @@ public class kelondroBLOBTree implements kelondroBLOB {
return p-1;
}
public int available() throws IOException {
return (int) (length() - seekpos);
public void setLength(long length) throws IOException {
throw new UnsupportedOperationException();
}
public long available() throws IOException {
return length() - seekpos;
}
public int read() throws IOException {

@ -45,7 +45,7 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
return this.sbb;
}
public int available() throws IOException {
public long available() throws IOException {
return Integer.MAX_VALUE - sbb.length();
}
@ -57,6 +57,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
return sbb.length();
}
public void setLength(long length) throws IOException {
sbb.resize((int) length);
}
public int read() throws IOException {
return 0xff & sbb.byteAt((int) pos++);
}

@ -0,0 +1,134 @@
// kelondroCachedFileRA.java
// -----------------------
// part of The Kelondro Database
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://yacy.net
// Frankfurt, Germany, 2004-2008
// last major change: 09.12.2008
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
public final class kelondroCachedFileRA extends kelondroAbstractRA implements kelondroRA {
private RandomAccessFile RAFile;
private byte[] cache;
private long cachestart;
private int cachelen;
public kelondroCachedFileRA(final File file) throws IOException, FileNotFoundException {
this.name = file.getName();
this.file = file;
RAFile = new RandomAccessFile(file, "rw");
cache = new byte[8192];
cachestart = 0;
cachelen = 0;
}
public synchronized long length() throws IOException {
return this.RAFile.length();
}
public synchronized void setLength(long length) throws IOException {
cachelen = 0;
RAFile.setLength(length);
}
public synchronized long available() throws IOException {
return this.length() - RAFile.getFilePointer();
}
public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException {
long seek = RAFile.getFilePointer();
if (cache != null && cachestart <= seek && cachelen - seek + cachestart >= len) {
// read from cache
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": CACHE HIT at " + seek);
System.arraycopy(cache, (int) (seek - cachestart), b, off, len);
RAFile.seek(seek + len);
return;
}
if (cache == null || cache.length < len) {
// cannot fill cache here
RAFile.readFully(b, off, len);
return;
}
// we fill the cache here
int available = (int) (this.RAFile.length() - seek);
if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len);
if (cachestart + cachelen == seek && cache.length - cachelen >= len) {
RAFile.readFully(cache, cachelen, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes");
System.arraycopy(cache, cachelen, b, off, len);
cachelen += len;
} else {
// fill the cache as much as possible
int m = Math.min(available, cache.length);
RAFile.readFully(cache, 0, m);
cachestart = seek;
cachelen = m;
if (m != len) RAFile.seek(seek + len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": replace fill " + len + " bytes");
System.arraycopy(cache, 0, b, off, len);
}
}
public synchronized void write(final byte[] b, final int off, final int len) throws IOException {
//assert len > 0;
// write to file
if (this.cache.length > 512) this.cache = new byte[512]; // the large cache is only useful during an initialization phase
long seekpos = this.RAFile.getFilePointer();
if (this.cachelen + len <= this.cache.length && this.cachestart + this.cachelen == seekpos) {
// append to cache
System.arraycopy(b, off, this.cache, this.cachelen, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write append " + len + " bytes");
this.cachelen += len;
} else if (len <= this.cache.length) {
// copy to cache
System.arraycopy(b, off, this.cache, 0, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write copy " + len + " bytes");
this.cachelen = len;
this.cachestart = seekpos;
} else {
// delete cache
this.cachelen = 0;
}
RAFile.write(b, off, len);
}
public synchronized void seek(final long pos) throws IOException {
RAFile.seek(pos);
}
public synchronized void close() throws IOException {
if (RAFile != null) RAFile.close();
this.cache = null;
this.RAFile = null;
}
protected void finalize() throws Throwable {
if (RAFile != null) {
this.close();
}
super.finalize();
}
}

@ -45,8 +45,12 @@ public final class kelondroChannelRA extends kelondroAbstractRA implements kelon
return channel.size();
}
public int available() throws IOException {
return (int) (channel.size() - channel.position());
public void setLength(long length) throws IOException {
channel.truncate(length);
}
public long available() throws IOException {
return channel.size() - channel.position();
}
public final void readFully(final byte[] b, final int off, final int len) throws IOException {

@ -92,7 +92,7 @@ public class kelondroEcoFS {
// open an existing table file
try {
raf = new RandomAccessFile(tablefile, "rw");
raf = new RandomAccessFile(tablefile,"rw");
} catch (final FileNotFoundException e) {
// should never happen
e.printStackTrace();

@ -31,17 +31,11 @@ import java.util.Map;
public final class kelondroFileRA extends kelondroAbstractRA implements kelondroRA {
private RandomAccessFile RAFile;
private byte[] cache;
private long cachestart;
private int cachelen;
public kelondroFileRA(final File file) throws IOException, FileNotFoundException {
this.name = file.getName();
this.file = file;
RAFile = new RandomAccessFile(file, "rw");
cache = new byte[8192];
cachestart = 0;
cachelen = 0;
}
public synchronized long length() throws IOException {
@ -49,69 +43,18 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro
}
public synchronized void setLength(long length) throws IOException {
cachelen = 0;
RAFile.setLength(length);
}
public synchronized int available() throws IOException {
return (int) (this.length() - RAFile.getFilePointer());
public synchronized long available() throws IOException {
return this.length() - RAFile.getFilePointer();
}
public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException {
long seek = RAFile.getFilePointer();
if (cache != null && cachestart <= seek && cachelen - seek + cachestart >= len) {
// read from cache
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": CACHE HIT at " + seek);
System.arraycopy(cache, (int) (seek - cachestart), b, off, len);
RAFile.seek(seek + len);
return;
}
if (cache == null || cache.length < len) {
// cannot fill cache here
RAFile.readFully(b, off, len);
return;
}
// we fill the cache here
int available = (int) (this.RAFile.length() - seek);
if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len);
if (cachestart + cachelen == seek && cache.length - cachelen >= len) {
RAFile.readFully(cache, cachelen, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes");
System.arraycopy(cache, cachelen, b, off, len);
cachelen += len;
} else {
// fill the cache as much as possible
int m = Math.min(available, cache.length);
RAFile.readFully(cache, 0, m);
cachestart = seek;
cachelen = m;
if (m != len) RAFile.seek(seek + len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": replace fill " + len + " bytes");
System.arraycopy(cache, 0, b, off, len);
}
RAFile.readFully(b, off, len);
}
public synchronized void write(final byte[] b, final int off, final int len) throws IOException {
//assert len > 0;
// write to file
//if (this.cache.length > 2048) this.cache = new byte[2048]; // the large cache is only useful during an initialization phase
long seekpos = this.RAFile.getFilePointer();
if (this.cachelen + len <= this.cache.length && this.cachestart + this.cachelen == seekpos) {
// append to cache
System.arraycopy(b, off, this.cache, this.cachelen, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write append " + len + " bytes");
this.cachelen += len;
} else if (len <= this.cache.length) {
// copy to cache
System.arraycopy(b, off, this.cache, 0, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write copy " + len + " bytes");
this.cachelen = len;
this.cachestart = seekpos;
} else {
// delete cache
this.cachelen = 0;
}
RAFile.write(b, off, len);
}
@ -121,7 +64,6 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro
public synchronized void close() throws IOException {
if (RAFile != null) RAFile.close();
this.cache = null;
this.RAFile = null;
}
@ -138,7 +80,7 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro
if (fp != null) fp.mkdirs();
kelondroRA kra = null;
try {
kra = new kelondroFileRA(f);
kra = new kelondroCachedFileRA(f);
kra.writeMap(map, comment);
kra.close();
} finally {
@ -149,7 +91,7 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro
public static Map<String, String> readMap(final File f) throws IOException {
kelondroRA kra = null;
try {
kra = new kelondroFileRA(f);
kra = new kelondroCachedFileRA(f);
final Map<String, String> map = kra.readMap();
kra.close();
return map;

@ -43,7 +43,8 @@ public interface kelondroRA {
// pseudo-native methods:
public long length() throws IOException;
public int available() throws IOException;
public void setLength(long length) throws IOException;
public long available() throws IOException;
public void readFully(byte[] b, int off, int len) throws IOException;

Loading…
Cancel
Save