diff --git a/source/de/anomic/kelondro/kelondroAbstractRA.java b/source/de/anomic/kelondro/kelondroAbstractRA.java index 4b7b808ea..c28d0a5a7 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRA.java +++ b/source/de/anomic/kelondro/kelondroAbstractRA.java @@ -51,17 +51,19 @@ abstract class kelondroAbstractRA implements kelondroRA { // pseudo-native methods: abstract public void readFully(byte[] b, int off, int len) throws IOException; abstract public long length() throws IOException; - abstract public int available() throws IOException; + abstract public void setLength(long length) throws IOException; + abstract public long available() throws IOException; abstract public void write(byte[] b, int off, int len) throws IOException; abstract public void seek(long pos) throws IOException; abstract public void close() throws IOException; // derived methods: public byte[] readFully() throws IOException { - int a = this.available(); + long a = this.available(); if (a <= 0) return null; - final byte[] buffer = new byte[a]; - this.readFully(buffer, 0, a); + if (a > Integer.MAX_VALUE) throw new IOException("available too large for a single array"); + final byte[] buffer = new byte[(int) a]; + this.readFully(buffer, 0, (int) a); return buffer; } diff --git a/source/de/anomic/kelondro/kelondroAbstractRecords.java b/source/de/anomic/kelondro/kelondroAbstractRecords.java index cf3a2f102..cd9930b04 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRecords.java +++ b/source/de/anomic/kelondro/kelondroAbstractRecords.java @@ -970,7 +970,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords { try { nn = next00(); } catch (final IOException e) { - serverLog.logSevere("kelondroCachedRecords", filename + " failed with " + e.getMessage(), e); + serverLog.logSevere("kelondroAbstractRecords", filename + " failed with " + e.getMessage(), e); return null; } byte[] key = null; @@ -1001,9 +1001,9 @@ public abstract class kelondroAbstractRecords implements kelondroRecords { bulkstart = pos.index; final int maxlength = Math.min(USAGE.allCount() - bulkstart, bulksize); if (((POS_NODES) + ((long) bulkstart) * ((long) recordsize)) < 0) - serverLog.logSevere("kelondroCachedRecords", "DEBUG: negative offset. POS_NODES = " + POS_NODES + ", bulkstart = " + bulkstart + ", recordsize = " + recordsize); + serverLog.logSevere("kelondroAbstractRecords", "DEBUG: negative offset. POS_NODES = " + POS_NODES + ", bulkstart = " + bulkstart + ", recordsize = " + recordsize); if ((maxlength * recordsize) < 0) - serverLog.logSevere("kelondroCachedRecords", "DEBUG: negative length. maxlength = " + maxlength + ", recordsize = " + recordsize); + serverLog.logSevere("kelondroAbstractRecords", "DEBUG: negative length. maxlength = " + maxlength + ", recordsize = " + recordsize); entryFile.readFully((POS_NODES) + ((long) bulkstart) * ((long) recordsize), bulk, 0, maxlength * recordsize); } /* POS_NODES = 302, bulkstart = 3277, recordsize = 655386 diff --git a/source/de/anomic/kelondro/kelondroBLOBHeap.java b/source/de/anomic/kelondro/kelondroBLOBHeap.java index c8a374185..b07c0050b 100755 --- a/source/de/anomic/kelondro/kelondroBLOBHeap.java +++ b/source/de/anomic/kelondro/kelondroBLOBHeap.java @@ -45,7 +45,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB { private TreeMap free; // list of {size, seek} pairs denoting space and position of free records private final File heapFile; // the file of the heap private final kelondroByteOrder ordering; // the ordering on keys - private kelondroFileRA file; // a random access to the file + private kelondroCachedFileRA file; // a random access to the file private HashMap buffer; // a write buffer to limit IO to the file; attention: Maps cannot use byte[] as key private int buffersize; // bytes that are buffered in buffer private int buffermax; // maximum size of the buffer @@ -89,7 +89,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB { this.free = new TreeMap(); this.buffer = new HashMap(); this.buffersize = 0; - this.file = new kelondroFileRA(heapFile); + this.file = new kelondroCachedFileRA(heapFile); byte[] key = new byte[keylength]; int reclen; long seek = 0; @@ -356,7 +356,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB { e.printStackTrace(); } this.heapFile.delete(); - this.file = new kelondroFileRA(heapFile); + this.file = new kelondroCachedFileRA(heapFile); } /** diff --git a/source/de/anomic/kelondro/kelondroBLOBTree.java b/source/de/anomic/kelondro/kelondroBLOBTree.java index e4295a9a3..ce473c63e 100644 --- a/source/de/anomic/kelondro/kelondroBLOBTree.java +++ b/source/de/anomic/kelondro/kelondroBLOBTree.java @@ -417,8 +417,12 @@ public class kelondroBLOBTree implements kelondroBLOB { return p-1; } - public int available() throws IOException { - return (int) (length() - seekpos); + public void setLength(long length) throws IOException { + throw new UnsupportedOperationException(); + } + + public long available() throws IOException { + return length() - seekpos; } public int read() throws IOException { diff --git a/source/de/anomic/kelondro/kelondroBufferedRA.java b/source/de/anomic/kelondro/kelondroBufferedRA.java index d90fdecc3..9d613b2c3 100644 --- a/source/de/anomic/kelondro/kelondroBufferedRA.java +++ b/source/de/anomic/kelondro/kelondroBufferedRA.java @@ -45,7 +45,7 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA return this.sbb; } - public int available() throws IOException { + public long available() throws IOException { return Integer.MAX_VALUE - sbb.length(); } @@ -57,6 +57,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA return sbb.length(); } + public void setLength(long length) throws IOException { + sbb.resize((int) length); + } + public int read() throws IOException { return 0xff & sbb.byteAt((int) pos++); } diff --git a/source/de/anomic/kelondro/kelondroCachedFileRA.java b/source/de/anomic/kelondro/kelondroCachedFileRA.java new file mode 100644 index 000000000..0df0b8c32 --- /dev/null +++ b/source/de/anomic/kelondro/kelondroCachedFileRA.java @@ -0,0 +1,134 @@ +// kelondroCachedFileRA.java +// ----------------------- +// part of The Kelondro Database +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://yacy.net +// Frankfurt, Germany, 2004-2008 +// last major change: 09.12.2008 +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; + +public final class kelondroCachedFileRA extends kelondroAbstractRA implements kelondroRA { + + private RandomAccessFile RAFile; + private byte[] cache; + private long cachestart; + private int cachelen; + + public kelondroCachedFileRA(final File file) throws IOException, FileNotFoundException { + this.name = file.getName(); + this.file = file; + RAFile = new RandomAccessFile(file, "rw"); + cache = new byte[8192]; + cachestart = 0; + cachelen = 0; + } + + public synchronized long length() throws IOException { + return this.RAFile.length(); + } + + public synchronized void setLength(long length) throws IOException { + cachelen = 0; + RAFile.setLength(length); + } + + public synchronized long available() throws IOException { + return this.length() - RAFile.getFilePointer(); + } + + public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException { + long seek = RAFile.getFilePointer(); + if (cache != null && cachestart <= seek && cachelen - seek + cachestart >= len) { + // read from cache + //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": CACHE HIT at " + seek); + System.arraycopy(cache, (int) (seek - cachestart), b, off, len); + RAFile.seek(seek + len); + return; + } + if (cache == null || cache.length < len) { + // cannot fill cache here + RAFile.readFully(b, off, len); + return; + } + // we fill the cache here + int available = (int) (this.RAFile.length() - seek); + if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len); + if (cachestart + cachelen == seek && cache.length - cachelen >= len) { + RAFile.readFully(cache, cachelen, len); + //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes"); + System.arraycopy(cache, cachelen, b, off, len); + cachelen += len; + } else { + // fill the cache as much as possible + int m = Math.min(available, cache.length); + RAFile.readFully(cache, 0, m); + cachestart = seek; + cachelen = m; + if (m != len) RAFile.seek(seek + len); + //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": replace fill " + len + " bytes"); + System.arraycopy(cache, 0, b, off, len); + } + + } + + public synchronized void write(final byte[] b, final int off, final int len) throws IOException { + //assert len > 0; + // write to file + if (this.cache.length > 512) this.cache = new byte[512]; // the large cache is only useful during an initialization phase + long seekpos = this.RAFile.getFilePointer(); + if (this.cachelen + len <= this.cache.length && this.cachestart + this.cachelen == seekpos) { + // append to cache + System.arraycopy(b, off, this.cache, this.cachelen, len); + //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write append " + len + " bytes"); + this.cachelen += len; + } else if (len <= this.cache.length) { + // copy to cache + System.arraycopy(b, off, this.cache, 0, len); + //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write copy " + len + " bytes"); + this.cachelen = len; + this.cachestart = seekpos; + } else { + // delete cache + this.cachelen = 0; + } + RAFile.write(b, off, len); + } + + public synchronized void seek(final long pos) throws IOException { + RAFile.seek(pos); + } + + public synchronized void close() throws IOException { + if (RAFile != null) RAFile.close(); + this.cache = null; + this.RAFile = null; + } + + protected void finalize() throws Throwable { + if (RAFile != null) { + this.close(); + } + super.finalize(); + } + +} diff --git a/source/de/anomic/kelondro/kelondroChannelRA.java b/source/de/anomic/kelondro/kelondroChannelRA.java index a66006d25..374bac9cf 100644 --- a/source/de/anomic/kelondro/kelondroChannelRA.java +++ b/source/de/anomic/kelondro/kelondroChannelRA.java @@ -45,8 +45,12 @@ public final class kelondroChannelRA extends kelondroAbstractRA implements kelon return channel.size(); } - public int available() throws IOException { - return (int) (channel.size() - channel.position()); + public void setLength(long length) throws IOException { + channel.truncate(length); + } + + public long available() throws IOException { + return channel.size() - channel.position(); } public final void readFully(final byte[] b, final int off, final int len) throws IOException { diff --git a/source/de/anomic/kelondro/kelondroEcoFS.java b/source/de/anomic/kelondro/kelondroEcoFS.java index cf7333498..75f9c2d34 100644 --- a/source/de/anomic/kelondro/kelondroEcoFS.java +++ b/source/de/anomic/kelondro/kelondroEcoFS.java @@ -92,7 +92,7 @@ public class kelondroEcoFS { // open an existing table file try { - raf = new RandomAccessFile(tablefile, "rw"); + raf = new RandomAccessFile(tablefile,"rw"); } catch (final FileNotFoundException e) { // should never happen e.printStackTrace(); diff --git a/source/de/anomic/kelondro/kelondroFileRA.java b/source/de/anomic/kelondro/kelondroFileRA.java index 23f4cc4b6..f3f826d11 100644 --- a/source/de/anomic/kelondro/kelondroFileRA.java +++ b/source/de/anomic/kelondro/kelondroFileRA.java @@ -1,161 +1,103 @@ -// kelondroFileRA.java -// ----------------------- -// part of The Kelondro Database -// (C) by Michael Peter Christen; mc@yacy.net -// first published on http://yacy.net -// Frankfurt, Germany, 2004-2008 -// last major change: 09.12.2008 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.kelondro; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.Map; - -public final class kelondroFileRA extends kelondroAbstractRA implements kelondroRA { - - private RandomAccessFile RAFile; - private byte[] cache; - private long cachestart; - private int cachelen; - - public kelondroFileRA(final File file) throws IOException, FileNotFoundException { - this.name = file.getName(); - this.file = file; - RAFile = new RandomAccessFile(file, "rw"); - cache = new byte[8192]; - cachestart = 0; - cachelen = 0; - } - - public synchronized long length() throws IOException { - return this.RAFile.length(); - } - - public synchronized void setLength(long length) throws IOException { - cachelen = 0; - RAFile.setLength(length); - } - - public synchronized int available() throws IOException { - return (int) (this.length() - RAFile.getFilePointer()); - } - - public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException { - long seek = RAFile.getFilePointer(); - if (cache != null && cachestart <= seek && cachelen - seek + cachestart >= len) { - // read from cache - //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": CACHE HIT at " + seek); - System.arraycopy(cache, (int) (seek - cachestart), b, off, len); - RAFile.seek(seek + len); - return; - } - if (cache == null || cache.length < len) { - // cannot fill cache here - RAFile.readFully(b, off, len); - return; - } - // we fill the cache here - int available = (int) (this.RAFile.length() - seek); - if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len); - if (cachestart + cachelen == seek && cache.length - cachelen >= len) { - RAFile.readFully(cache, cachelen, len); - //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes"); - System.arraycopy(cache, cachelen, b, off, len); - cachelen += len; - } else { - // fill the cache as much as possible - int m = Math.min(available, cache.length); - RAFile.readFully(cache, 0, m); - cachestart = seek; - cachelen = m; - if (m != len) RAFile.seek(seek + len); - //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": replace fill " + len + " bytes"); - System.arraycopy(cache, 0, b, off, len); - } - - } - - public synchronized void write(final byte[] b, final int off, final int len) throws IOException { - //assert len > 0; - // write to file - //if (this.cache.length > 2048) this.cache = new byte[2048]; // the large cache is only useful during an initialization phase - long seekpos = this.RAFile.getFilePointer(); - if (this.cachelen + len <= this.cache.length && this.cachestart + this.cachelen == seekpos) { - // append to cache - System.arraycopy(b, off, this.cache, this.cachelen, len); - //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write append " + len + " bytes"); - this.cachelen += len; - } else if (len <= this.cache.length) { - // copy to cache - System.arraycopy(b, off, this.cache, 0, len); - //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write copy " + len + " bytes"); - this.cachelen = len; - this.cachestart = seekpos; - } else { - // delete cache - this.cachelen = 0; - } - RAFile.write(b, off, len); - } - - public synchronized void seek(final long pos) throws IOException { - RAFile.seek(pos); - } - - public synchronized void close() throws IOException { - if (RAFile != null) RAFile.close(); - this.cache = null; - this.RAFile = null; - } - - protected void finalize() throws Throwable { - if (RAFile != null) { - this.close(); - } - super.finalize(); - } - - // some static tools - public static void writeMap(final File f, final Map map, final String comment) throws IOException { - final File fp = f.getParentFile(); - if (fp != null) fp.mkdirs(); - kelondroRA kra = null; - try { - kra = new kelondroFileRA(f); - kra.writeMap(map, comment); - kra.close(); - } finally { - if (kra != null) try {kra.close();}catch(final Exception e){} - } - } - - public static Map readMap(final File f) throws IOException { - kelondroRA kra = null; - try { - kra = new kelondroFileRA(f); - final Map map = kra.readMap(); - kra.close(); - return map; - } finally { - if (kra != null) try {kra.close();}catch(final Exception e){} - } - } - -} +// kelondroFileRA.java +// ----------------------- +// part of The Kelondro Database +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://yacy.net +// Frankfurt, Germany, 2004-2008 +// last major change: 09.12.2008 +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Map; + +public final class kelondroFileRA extends kelondroAbstractRA implements kelondroRA { + + private RandomAccessFile RAFile; + + public kelondroFileRA(final File file) throws IOException, FileNotFoundException { + this.name = file.getName(); + this.file = file; + RAFile = new RandomAccessFile(file, "rw"); + } + + public synchronized long length() throws IOException { + return this.RAFile.length(); + } + + public synchronized void setLength(long length) throws IOException { + RAFile.setLength(length); + } + + public synchronized long available() throws IOException { + return this.length() - RAFile.getFilePointer(); + } + + public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException { + RAFile.readFully(b, off, len); + } + + public synchronized void write(final byte[] b, final int off, final int len) throws IOException { + RAFile.write(b, off, len); + } + + public synchronized void seek(final long pos) throws IOException { + RAFile.seek(pos); + } + + public synchronized void close() throws IOException { + if (RAFile != null) RAFile.close(); + this.RAFile = null; + } + + protected void finalize() throws Throwable { + if (RAFile != null) { + this.close(); + } + super.finalize(); + } + + // some static tools + public static void writeMap(final File f, final Map map, final String comment) throws IOException { + final File fp = f.getParentFile(); + if (fp != null) fp.mkdirs(); + kelondroRA kra = null; + try { + kra = new kelondroCachedFileRA(f); + kra.writeMap(map, comment); + kra.close(); + } finally { + if (kra != null) try {kra.close();}catch(final Exception e){} + } + } + + public static Map readMap(final File f) throws IOException { + kelondroRA kra = null; + try { + kra = new kelondroCachedFileRA(f); + final Map map = kra.readMap(); + kra.close(); + return map; + } finally { + if (kra != null) try {kra.close();}catch(final Exception e){} + } + } + +} diff --git a/source/de/anomic/kelondro/kelondroRA.java b/source/de/anomic/kelondro/kelondroRA.java index 57a52b238..c35f53231 100644 --- a/source/de/anomic/kelondro/kelondroRA.java +++ b/source/de/anomic/kelondro/kelondroRA.java @@ -43,7 +43,8 @@ public interface kelondroRA { // pseudo-native methods: public long length() throws IOException; - public int available() throws IOException; + public void setLength(long length) throws IOException; + public long available() throws IOException; public void readFully(byte[] b, int off, int len) throws IOException;