diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 1742d2572..5a943c086 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -1085,6 +1085,7 @@ public final class yacyClient { // first arg: path to application home // second arg: address of target peer // third arg: search word or file name with list of search words + // i.e. /Data/workspace1/yacy/ localhost:8080 /Data/workspace1/yacy/test/words/searchtest.words System.out.println("yacyClient Test"); File searchwordfile = new File(args[2]); List searchlines = new ArrayList(); diff --git a/source/net/yacy/kelondro/io/CachedRecords.java b/source/net/yacy/kelondro/io/CachedRecords.java new file mode 100644 index 000000000..d75008449 --- /dev/null +++ b/source/net/yacy/kelondro/io/CachedRecords.java @@ -0,0 +1,586 @@ +// Records.java +// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 14.01.2008 on http://yacy.net +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.kelondro.io; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; + +import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.FileUtils; +import net.yacy.kelondro.util.MemoryControl; + + +/** + * The Records data structure is a flat file with records of fixed length. + * The file does not contain any meta information and the first record starts + * right at file position 0. + * The access rules are in such a way that a minimum of IO operations are necessary + * Two caches provide a mirror to content in the file: a read cache and a write buffer + * The read cache contains a number of entries from the file; a mirror that moves + * whenever information outside the mirror is requested. + * The write buffer always exists only at the end of the file. It contains only records + * that have never been written to the file before. When the write buffer is flushed, + * the file grows + * The record file may also shrink when the last entry of the file is removed. + * Removal of Entries inside the file is not possible, but such entries can be erased + * by overwriting the data with zero bytes + * All access to the file is made with byte[] that are generated outside of this class + * This class only references byte[] that are handed over to methods of this class. + */ +public final class CachedRecords { + + private RandomAccessFile raf; + private final File tablefile; + /** + * number of bytes in one record + */ + protected final int recordsize; + private long cacheindex; + /** + * number of entries in buffer + */ + private int cachecount; + private int buffercount; + private byte[] cache; + private byte[] buffer; + private final byte[] zero; + + /** + * stay below hard disc cache (is that necessary?) + */ + private static final int maxReadCache = 16 * 1024; + private static final int maxWriteBuffer = 16 * 1024; + + + public CachedRecords(final File tablefile, final int recordsize) throws IOException { + this.tablefile = tablefile; + this.recordsize = recordsize; + + // initialize zero buffer + this.zero = new byte[recordsize]; + for (int i = 0; i < recordsize; i++) this.zero[i] = 0; + + // initialize table file + if (!tablefile.exists()) { + // make new file + FileOutputStream fos = null; + try { + fos = new FileOutputStream(tablefile); + } catch (final FileNotFoundException e) { + // should not happen + Log.logException(e); + } + try { if (fos != null) fos.close(); } catch (final IOException e) {} + } + + // open an existing table file + try { + raf = new RandomAccessFile(tablefile,"rw"); + } catch (final FileNotFoundException e) { + // should never happen + Log.logException(e); + } + + // initialize cache and buffer + int cachesize = Math.max(1, (maxReadCache / recordsize)) * recordsize; + int buffersize = Math.max(1, (maxWriteBuffer / recordsize)) * recordsize; + if (!MemoryControl.request(cachesize + buffersize + 1024 * 1024 * 20, true)) { + // not enough memory there, take less + long lessmem = Math.min(Math.min(maxReadCache, maxWriteBuffer) / 8, MemoryControl.available() - (1024 * 1024 * 6) / 6); + //System.out.println("newmem vorher: cachesize = " + cachesize + ", buffersize = " + buffersize + ", available = " + serverMemory.available() + ", lessmem = " + lessmem); + cachesize = Math.max(1, (int) (lessmem / recordsize)) * recordsize; + buffersize = Math.max(1, (int) (lessmem / recordsize)) * recordsize; + //System.out.println("newmem nachher: cachesize = " + cachesize + ", buffersize = " + buffersize); + } + + this.cache = new byte[cachesize]; + this.buffer = new byte[buffersize]; + this.buffercount = 0; + + // first-time read of cache + fillCache(0); + } + + /** + * @param tablefile + * @param recordsize + * @return number of records in table + */ + public final static long tableSize(final File tablefile, final long recordsize) throws IOException { + if (!tablefile.exists()) return 0; + final long size = tablefile.length(); + if (size % recordsize != 0) throw new IOException("wrong file size: file = " + tablefile + ", size = " + size + ", recordsize = " + recordsize); + return size / recordsize; + } + + public final static void fixTableSize(final File tablefile, final long recordsize) throws IOException { + if (!tablefile.exists()) return; + final long size = tablefile.length(); + long cut = size % recordsize; + if (cut > 0) { + RandomAccessFile raf = new RandomAccessFile(tablefile, "rw"); + raf.setLength(size - cut); + raf.close(); + } + } + + /** + * @return the number of records in file plus number of records in buffer + * @throws IOException + */ + public final synchronized long size() throws IOException { + return filesize() + this.buffercount; + } + + public final File filename() { + return this.tablefile; + } + + /** + * @return records in file + * @throws IOException + */ + private final long filesize() throws IOException { + return raf.length() / recordsize; + } + + /** + * checks if the index is inside the cache + * + * @param index + * @return the index offset inside the cache or -1 if the index is not in the cache + */ + private final int inCache(final long index) { + if (index >= this.cacheindex && index < this.cacheindex + this.cachecount) { + return (int) (index - this.cacheindex); + } + return -1; + } + + /** + * checks if the index is inside the buffer + * + * @param index + * @return the index offset inside the buffer or -1 if the index is not in the buffer + * @throws IOException + */ + private final int inBuffer(final long index) throws IOException { + final long fs = filesize(); + if ((index >= fs) && (index < fs + this.buffercount)) { + return (int) (index - fs); + } + return -1; + } + + /** + * load cache with copy of disc content; start with record at index + * + * if the record would overlap with the write buffer, + * its start is shifted forward until it fits + * + * @param index + * @throws IOException + */ + private final void fillCache(long index) throws IOException { + //System.out.println("*** DEBUG FillCache " + this.tablefile.getName() + " at index " + index); + // first check if the index is inside the current cache + assert inCache(index) < 0; + if (inCache(index) >= 0) return; + + // calculate new start position + final long fs = this.filesize(); + if (index + this.cache.length / this.recordsize > fs) { + index = fs - this.cache.length / this.recordsize; + } + if (index < 0) index = 0; + + // calculate number of records that shall be stored in the cache + this.cachecount = (int) Math.min(this.cache.length / this.recordsize, this.filesize() - index); + assert this.cachecount >= 0; + + // check if we need to read 0 bytes from the file + this.cacheindex = index; + if (this.cachecount == 0) return; + + // copy records from file to cache + raf.seek(this.recordsize * index); + raf.readFully(this.cache, 0, this.recordsize * this.cachecount); + } + + /** + * write buffer to end of file + */ + public final void flushBuffer() { + try { + raf.seek(raf.length()); + raf.write(this.buffer, 0, this.recordsize * this.buffercount); + } catch (final IOException e) { + Log.logException(e); + } + this.buffercount = 0; + } + + public final synchronized void close() { + flushBuffer(); + + // then close the file + if (raf != null) try { + raf.close(); + } catch (final IOException e) { + Log.logException(e); + } + this.raf = null; + this.buffer = null; + this.cache = null; + } + + /** + * @param index record which should be read + * @param b destination array + * @param start offset in b to store data + * @throws IOException + */ + public final synchronized void get(final long index, final byte[] b, final int start) throws IOException { + assert b.length - start >= this.recordsize; + if (index >= size()) throw new IndexOutOfBoundsException("kelondroEcoFS.get(" + index + ") outside bounds (" + this.size() + ")"); + // check if index is inside of cache + int p = inCache(index); + final int q = (p >= 0) ? -1 : inBuffer(index); + if (p < 0 && q < 0) { + // the index is outside of cache and buffer index. shift cache window + fillCache(index); + p = inCache(index); + assert p >= 0; + } + if (p >= 0) { + // read entry from the cache + System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize); + return; + } + if (q >= 0) { + // read entry from the buffer + System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize); + return; + } + assert false; + } + + public final synchronized void put(final long index, final byte[] b, final int start) throws IOException { + assert b.length - start >= this.recordsize; + final long s = filesize() + this.buffercount; + if (index > s) throw new IndexOutOfBoundsException("kelondroEcoFS.put(" + index + ") outside bounds (" + this.size() + ")"); + + // check if this is an empty entry + if (isClean(b , start, this.recordsize)) { + clean(index); + return; + } + + // check if index is inside of cache + final int p = inCache(index); + final int q = (p >= 0) ? -1 : inBuffer(index); + if (p >= 0) { + // write entry to the cache and to the file + System.arraycopy(b, start, this.cache, p * this.recordsize, this.recordsize); + raf.seek(index * this.recordsize); + raf.write(b, start, this.recordsize); + return; + } + if (q >= 0) { + // write entry to the buffer + System.arraycopy(b, start, this.buffer, q * this.recordsize, this.recordsize); + return; + } + if (index == s) { + // append the record to the end of the file; + + // look if there is space in the buffer + if (this.buffercount >= this.buffer.length / this.recordsize) { + assert this.buffercount == this.buffer.length / this.recordsize; + // the record does not fit in current buffer + // write buffer + flushBuffer(); + // write new entry to buffer + System.arraycopy(b, start, this.buffer, 0, this.recordsize); + this.buffercount = 1; + } else { + System.arraycopy(b, start, this.buffer, this.buffercount * this.recordsize, this.recordsize); + this.buffercount++; + } + assert this.buffercount <= this.buffer.length / this.recordsize; + } else { + // write the record directly to the file, + // do not care about the cache; this case was checked before + raf.seek(index * this.recordsize); + raf.write(b, start, this.recordsize); + } + } + + public final synchronized void add(final byte[] b, final int start) throws IOException { + // index == size() == filesize() + (long) this.buffercount + + assert b.length - start >= this.recordsize; + + // check if this is an empty entry + if (isClean(b , start, this.recordsize)) { + // it is not possible to add a clean record at the end of a EcoFS, because + // such records should cause the record to shrink + throw new IOException("add: record at end is clean"); + } + + // append the record to the end of the file; + // look if there is space in the buffer + if (this.buffercount >= this.buffer.length / this.recordsize) { + assert this.buffercount == this.buffer.length / this.recordsize; + // the record does not fit in current buffer + // write buffer + flushBuffer(); + // write new entry to buffer + System.arraycopy(b, start, this.buffer, 0, this.recordsize); + this.buffercount = 1; + } else { + System.arraycopy(b, start, this.buffer, this.buffercount * this.recordsize, this.recordsize); + this.buffercount++; + } + assert this.buffercount <= this.buffer.length / this.recordsize; + } + + private final boolean isClean(final byte[] b, final int offset, final int length) { + for (int i = 0; i < length; i++) { + if (b[i + offset] != 0) return false; + } + return true; + } + + private final boolean isClean(final long index) throws IOException { + assert index < size(); + // check if index is inside of cache + int p = inCache(index); + final int q = (p >= 0) ? -1 : inBuffer(index); + if (p < 0 && q < 0) { + // the index is outside of cache and buffer index. shift cache window + fillCache(index); + p = inCache(index); + assert p >= 0; + } + if (p >= 0) { + // check entry from the cache + return isClean(this.cache, p * this.recordsize, this.recordsize); + } + if (q >= 0) { + // check entry from the buffer + return isClean(this.buffer, q * this.recordsize, this.recordsize); + } + assert false; + return false; + } + + /** + * @see clean(long, byte[], int) + * @param index + * @throws IOException + */ + public final synchronized void clean(final long index) throws IOException { + final long s = size(); + if (index >= s) throw new IndexOutOfBoundsException("kelondroEcoFS.clean(" + index + ") outside bounds (" + s + ")"); + if (index == s - 1) { + cleanLast(); + return; + } + + // check if index is inside of cache + final int p = inCache(index); + final int q = (p >= 0) ? -1 : inBuffer(index); + if (p >= 0) { + // write zero bytes to the cache and to the file + System.arraycopy(zero, 0, this.cache, p * this.recordsize, this.recordsize); + raf.seek(index * this.recordsize); + raf.write(zero, 0, this.recordsize); + return; + } + if (q >= 0) { + // write zero to the buffer + System.arraycopy(zero, 0, this.buffer, q * this.recordsize, this.recordsize); + return; + } + + raf.seek(index * this.recordsize); + raf.write(zero, 0, this.recordsize); + } + + /** + * @see clean(long, byte[], int) + * @param b + * @param start + * @throws IOException + */ + public final synchronized void cleanLast(final byte[] b, final int start) throws IOException { + cleanLast0(b, start); + long i; + while (((i = size()) > 0) && (isClean(i - 1))) { + //System.out.println("Extra clean/1: before size = " + size()); + cleanLast0(); + //System.out.println(" after size = " + size()); + } + } + + /** + * this is like + * clean(this.size() - 1, b, start); + * + * @see clean(long, byte[], int) + * @param b + * @param start + * @throws IOException + */ + private final synchronized void cleanLast0(final byte[] b, final int start) throws IOException { + assert b.length - start >= this.recordsize; + // check if index is inside of cache + final long s = this.size(); + int p = inCache(s - 1); + final int q = (p >= 0) ? -1 : inBuffer(s - 1); + if ((p < 0) && (q < 0)) { + // the index is outside of cache and buffer index. shift cache window + fillCache(this.size() - 1); + p = inCache(this.size() - 1); + assert p >= 0; + } + if (p >= 0) { + // read entry from the cache + System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize); + // shrink cache and file + assert this.buffercount == 0; + this.raf.setLength((s - 1) * this.recordsize); + this.cachecount--; + return; + } + if (q >= 0) { + // read entry from the buffer + System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize); + // shrink buffer + assert this.buffercount > 0; + this.buffercount--; + return; + } + assert false; + } + + /** + * @see clean(long, byte[], int) + * @throws IOException + */ + public final synchronized void cleanLast() throws IOException { + cleanLast0(); + long i; + while (((i = size()) > 0) && (isClean(i - 1))) { + //System.out.println("Extra clean/0: before size = " + size()); + cleanLast0(); + //System.out.println(" after size = " + size()); + } + } + + private final synchronized void cleanLast0() throws IOException { + + // check if index is inside of cache + final long s = this.size(); + final long p = inCache(s - 1); + final long q = (p >= 0) ? -1 : inBuffer(s - 1); + if (p >= 0) { + // shrink cache and file + assert this.buffercount == 0; + this.raf.setLength((s - 1) * this.recordsize); + this.cachecount--; + return; + } + if (q >= 0) { + // shrink buffer + assert this.buffercount > 0; + this.buffercount--; + return; + } + // check if file should shrink + assert this.buffercount == 0; + this.raf.setLength((s - 1) * this.recordsize); + } + + public final void deleteOnExit() { + this.tablefile.deleteOnExit(); + } + + /** + * main - writes some data and checks the tables size (with time measureing) + * @param args + */ + public static void main(final String[] args) { + // open a file, add one entry and exit + final File f = new File(args[0]); + if (f.exists()) FileUtils.deletedelete(f); + try { + final CachedRecords t = new CachedRecords(f, 8); + final byte[] b = new byte[8]; + t.add("01234567".getBytes(), 0); + t.add("ABCDEFGH".getBytes(), 0); + t.add("abcdefgh".getBytes(), 0); + t.add("--------".getBytes(), 0); + t.add("********".getBytes(), 0); + for (int i = 0; i < 1000; i++) t.add("++++++++".getBytes(), 0); + t.add("=======0".getBytes(), 0); + t.add("=======1".getBytes(), 0); + t.add("=======2".getBytes(), 0); + t.cleanLast(b, 0); + System.out.println(new String(b)); + t.cleanLast(b, 0); + //t.clean(2, b, 0); + System.out.println(new String(b)); + t.get(1, b, 0); + System.out.println(new String(b)); + t.put(1, "AbCdEfGh".getBytes(), 0); + t.get(1, b, 0); + System.out.println(new String(b)); + t.get(3, b, 0); + System.out.println(new String(b)); + t.get(4, b, 0); + System.out.println(new String(b)); + System.out.println("size = " + t.size()); + //t.clean(t.size() - 2); + t.cleanLast(); + final long start = System.currentTimeMillis(); + long c = 0; + for (int i = 0; i < 100000; i++) { + c = t.size(); + } + System.out.println("size() needs " + ((System.currentTimeMillis() - start) / 100) + " nanoseconds"); + System.out.println("size = " + c); + + t.close(); + } catch (final IOException e) { + Log.logException(e); + } + } + +} diff --git a/source/net/yacy/kelondro/io/Records.java b/source/net/yacy/kelondro/io/Records.java index 699a13772..493580193 100644 --- a/source/net/yacy/kelondro/io/Records.java +++ b/source/net/yacy/kelondro/io/Records.java @@ -60,24 +60,20 @@ public final class Records { * number of bytes in one record */ protected final int recordsize; - private long cacheindex; /** * number of entries in buffer */ - private int cachecount; private int buffercount; - private byte[] cache; private byte[] buffer; private final byte[] zero; /** * stay below hard disc cache (is that necessary?) */ - private static final int maxReadCache = 16 * 1024; private static final int maxWriteBuffer = 16 * 1024; - public Records(final File tablefile, final int recordsize) throws IOException { + public Records(final File tablefile, final int recordsize) { this.tablefile = tablefile; this.recordsize = recordsize; @@ -106,24 +102,18 @@ public final class Records { Log.logException(e); } - // initialize cache and buffer - int cachesize = Math.max(1, (maxReadCache / recordsize)) * recordsize; + // initialize write buffer int buffersize = Math.max(1, (maxWriteBuffer / recordsize)) * recordsize; - if (!MemoryControl.request(cachesize + buffersize + 1024 * 1024 * 20, true)) { + if (!MemoryControl.request(buffersize + 1024 * 1024 * 20, true)) { // not enough memory there, take less - long lessmem = Math.min(Math.min(maxReadCache, maxWriteBuffer) / 8, MemoryControl.available() - (1024 * 1024 * 6) / 6); + long lessmem = Math.min(maxWriteBuffer / 8, MemoryControl.available() - (1024 * 1024 * 6) / 6); //System.out.println("newmem vorher: cachesize = " + cachesize + ", buffersize = " + buffersize + ", available = " + serverMemory.available() + ", lessmem = " + lessmem); - cachesize = Math.max(1, (int) (lessmem / recordsize)) * recordsize; - buffersize = Math.max(1, (int) (lessmem / recordsize)) * recordsize; + buffersize = Math.max(1, (int) (lessmem / recordsize)) * recordsize; //System.out.println("newmem nachher: cachesize = " + cachesize + ", buffersize = " + buffersize); } - this.cache = new byte[cachesize]; this.buffer = new byte[buffersize]; this.buffercount = 0; - - // first-time read of cache - fillCache(0); } /** @@ -168,19 +158,6 @@ public final class Records { private final long filesize() throws IOException { return raf.length() / recordsize; } - - /** - * checks if the index is inside the cache - * - * @param index - * @return the index offset inside the cache or -1 if the index is not in the cache - */ - private final int inCache(final long index) { - if ((index >= this.cacheindex) && (index < this.cacheindex + this.cachecount)) { - return (int) (index - this.cacheindex); - } - return -1; - } /** * checks if the index is inside the buffer @@ -191,51 +168,16 @@ public final class Records { */ private final int inBuffer(final long index) throws IOException { final long fs = filesize(); - if ((index >= fs) && (index < fs + this.buffercount)) { + if (index >= fs && index < fs + this.buffercount) { return (int) (index - fs); } return -1; } - - /** - * load cache with copy of disc content; start with record at index - * - * if the record would overlap with the write buffer, - * its start is shifted forward until it fits - * - * @param index - * @throws IOException - */ - private final void fillCache(long index) throws IOException { - //System.out.println("*** DEBUG FillCache " + this.tablefile.getName() + " at index " + index); - // first check if the index is inside the current cache - assert inCache(index) < 0; - if (inCache(index) >= 0) return; - - // calculate new start position - final long fs = this.filesize(); - if (index + this.cache.length / this.recordsize > fs) { - index = fs - this.cache.length / this.recordsize; - } - if (index < 0) index = 0; - - // calculate number of records that shall be stored in the cache - this.cachecount = (int) Math.min(this.cache.length / this.recordsize, this.filesize() - index); - assert this.cachecount >= 0; - - // check if we need to read 0 bytes from the file - this.cacheindex = index; - if (this.cachecount == 0) return; - - // copy records from file to cache - raf.seek(this.recordsize * index); - raf.readFully(this.cache, 0, this.recordsize * this.cachecount); - } /** * write buffer to end of file */ - public final void flushBuffer() { + protected final synchronized void flushBuffer() { try { raf.seek(raf.length()); raf.write(this.buffer, 0, this.recordsize * this.buffercount); @@ -256,7 +198,6 @@ public final class Records { } this.raf = null; this.buffer = null; - this.cache = null; } /** @@ -269,25 +210,15 @@ public final class Records { assert b.length - start >= this.recordsize; if (index >= size()) throw new IndexOutOfBoundsException("kelondroEcoFS.get(" + index + ") outside bounds (" + this.size() + ")"); // check if index is inside of cache - int p = inCache(index); - final int q = (p >= 0) ? -1 : inBuffer(index); - if ((p < 0) && (q < 0)) { - // the index is outside of cache and buffer index. shift cache window - fillCache(index); - p = inCache(index); - assert p >= 0; - } - if (p >= 0) { - // read entry from the cache - System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize); - return; - } - if (q >= 0) { - // read entry from the buffer - System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize); + final int q = inBuffer(index); + if (q < 0) { + // copy records from file to given buffer + raf.seek(this.recordsize * index); + raf.readFully(b, start, this.recordsize); return; } - assert false; + // read entry from the buffer + System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize); } public final synchronized void put(final long index, final byte[] b, final int start) throws IOException { @@ -302,15 +233,7 @@ public final class Records { } // check if index is inside of cache - final int p = inCache(index); - final int q = (p >= 0) ? -1 : inBuffer(index); - if (p >= 0) { - // write entry to the cache and to the file - System.arraycopy(b, start, this.cache, p * this.recordsize, this.recordsize); - raf.seek(index * this.recordsize); - raf.write(b, start, this.recordsize); - return; - } + final int q = inBuffer(index); if (q >= 0) { // write entry to the buffer System.arraycopy(b, start, this.buffer, q * this.recordsize, this.recordsize); @@ -379,79 +302,16 @@ public final class Records { private final boolean isClean(final long index) throws IOException { assert index < size(); - // check if index is inside of cache - int p = inCache(index); - final int q = (p >= 0) ? -1 : inBuffer(index); - if ((p < 0) && (q < 0)) { - // the index is outside of cache and buffer index. shift cache window - fillCache(index); - p = inCache(index); - assert p >= 0; - } - if (p >= 0) { - // check entry from the cache - return isClean(this.cache, p * this.recordsize, this.recordsize); - } + // check if index is inside of buffer + final int q = inBuffer(index); if (q >= 0) { // check entry from the buffer return isClean(this.buffer, q * this.recordsize, this.recordsize); } - assert false; - return false; - } - - /** - * removes an entry by cleaning (writing zero bytes to the file) - * - * the entry that had been at the specific place before is copied to the given array b - * if the last entry in the file was cleaned, the file shrinks by the given record - * - * this is like - * get(index, b, start); - * put(index, zero, 0); - * plus an additional check if the file should shrink - * - * @param index - * @param b content at index - * @param start offset in record to start reading - * @throws IOException - */ - public final synchronized void clean(final long index, final byte[] b, final int start) throws IOException { - assert b.length - start >= this.recordsize; - final long s = size(); - if (index >= s) throw new IndexOutOfBoundsException("kelondroEcoFS.clean(" + index + ") outside bounds (" + s + ")"); - if (index == s - 1) { - cleanLast(b, start); - return; - } - - // check if index is inside of cache - int p = inCache(index); - final int q = (p >= 0) ? -1 : inBuffer(index); - if ((p < 0) && (q < 0)) { - // the index is outside of cache and buffer index. shift cache window - fillCache(index); - p = inCache(index); - assert p >= 0; - } - if (p >= 0) { - // read entry from the cache - System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize); - - // write zero bytes to the cache and to the file - System.arraycopy(zero, 0, this.cache, p * this.recordsize, this.recordsize); - this.raf.seek(index * this.recordsize); - this.raf.write(zero, 0, this.recordsize); - return; - } - if (q >= 0) { - // read entry from the buffer - System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize); - // write zero to the buffer - System.arraycopy(zero, 0, this.buffer, q * this.recordsize, this.recordsize); - return; - } - assert false; + byte[] b = new byte[this.recordsize]; + raf.seek(index * this.recordsize); + raf.readFully(b, 0, this.recordsize); + return isClean(b, 0, this.recordsize); } /** @@ -459,7 +319,7 @@ public final class Records { * @param index * @throws IOException */ - public final synchronized void clean(final long index) throws IOException { + private final void clean(final long index) throws IOException { final long s = size(); if (index >= s) throw new IndexOutOfBoundsException("kelondroEcoFS.clean(" + index + ") outside bounds (" + s + ")"); if (index == s - 1) { @@ -468,15 +328,7 @@ public final class Records { } // check if index is inside of cache - final int p = inCache(index); - final int q = (p >= 0) ? -1 : inBuffer(index); - if (p >= 0) { - // write zero bytes to the cache and to the file - System.arraycopy(zero, 0, this.cache, p * this.recordsize, this.recordsize); - raf.seek(index * this.recordsize); - raf.write(zero, 0, this.recordsize); - return; - } + final int q = inBuffer(index); if (q >= 0) { // write zero to the buffer System.arraycopy(zero, 0, this.buffer, q * this.recordsize, this.recordsize); @@ -496,7 +348,7 @@ public final class Records { public final synchronized void cleanLast(final byte[] b, final int start) throws IOException { cleanLast0(b, start); long i; - while (((i = size()) > 0) && (isClean(i - 1))) { + while ((i = size()) > 0 && isClean(i - 1)) { //System.out.println("Extra clean/1: before size = " + size()); cleanLast0(); //System.out.println(" after size = " + size()); @@ -512,36 +364,27 @@ public final class Records { * @param start * @throws IOException */ - private final synchronized void cleanLast0(final byte[] b, final int start) throws IOException { + private final void cleanLast0(final byte[] b, final int start) throws IOException { assert b.length - start >= this.recordsize; - // check if index is inside of cache - final long s = this.size(); - int p = inCache(s - 1); - final int q = (p >= 0) ? -1 : inBuffer(s - 1); - if ((p < 0) && (q < 0)) { - // the index is outside of cache and buffer index. shift cache window - fillCache(this.size() - 1); - p = inCache(this.size() - 1); - assert p >= 0; - } - if (p >= 0) { - // read entry from the cache - System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize); - // shrink cache and file - assert this.buffercount == 0; - this.raf.setLength((s - 1) * this.recordsize); - this.cachecount--; - return; - } - if (q >= 0) { + // check if index is inside of buffer + if (this.buffercount > 0) { // read entry from the buffer - System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize); + System.arraycopy(this.buffer, (this.buffercount - 1) * this.recordsize, b, start, this.recordsize); // shrink buffer - assert this.buffercount > 0; this.buffercount--; return; } - assert false; + // read entry from the file + final long endpos = this.raf.length() - this.recordsize; + raf.seek(endpos); + raf.readFully(b, start, this.recordsize); + + // write zero bytes to the cache and to the file + this.raf.seek(endpos); + this.raf.write(zero, 0, this.recordsize); + + // shrink file + this.raf.setLength(endpos); } /** @@ -558,28 +401,16 @@ public final class Records { } } - private final synchronized void cleanLast0() throws IOException { + private final void cleanLast0() throws IOException { // check if index is inside of cache - final long s = this.size(); - final long p = inCache(s - 1); - final long q = (p >= 0) ? -1 : inBuffer(s - 1); - if (p >= 0) { - // shrink cache and file - assert this.buffercount == 0; - this.raf.setLength((s - 1) * this.recordsize); - this.cachecount--; - return; - } - if (q >= 0) { + if (this.buffercount > 0) { // shrink buffer - assert this.buffercount > 0; this.buffercount--; return; } - // check if file should shrink - assert this.buffercount == 0; - this.raf.setLength((s - 1) * this.recordsize); + // shrink file + this.raf.setLength(raf.length() - this.recordsize); } public final void deleteOnExit() { diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 5794f5ab2..4b73df1e1 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -449,14 +449,12 @@ public class Table implements Index, Iterable { } public Entry get(final byte[] key) throws IOException { - if ((file == null) || (index == null)) return null; - synchronized (this) { - assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size() + ", file = " + this.filename(); - assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size() + ", file = " + this.filename(); - } + if (file == null || index == null) return null; Entry e = get0(key); if (e != null && this.rowdef.objectOrder.equal(key, e.getPrimaryKeyBytes())) return e; synchronized (this) { + assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size() + ", file = " + this.filename(); + assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size() + ", file = " + this.filename(); e = get0(key); assert e == null || this.rowdef.objectOrder.equal(key, e.getPrimaryKeyBytes()); return e; @@ -464,7 +462,7 @@ public class Table implements Index, Iterable { } private Entry get0(final byte[] key) throws IOException { - if ((file == null) || (index == null)) return null; + if (file == null || index == null) return null; final int i = (int) index.get(key); if (i == -1) return null; final byte[] b = new byte[rowdef.objectsize]; @@ -756,7 +754,11 @@ public class Table implements Index, Iterable { final int i = (int) index.remove(lr.getPrimaryKeyBytes()); assert i < 0 || index.size() < is : "index.size() = " + index.size() + ", is = " + is; assert i >= 0; - if (table != null) table.removeOne(); + if (table != null) { + int tsb = table.size(); + table.removeOne(); + assert table.size() < tsb : "table.size() = " + table.size() + ", tsb = " + tsb; + } assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size(); return lr; @@ -791,14 +793,7 @@ public class Table implements Index, Iterable { } if (fos != null) try { fos.close(); } catch (final IOException e) {} - - // open an existing table file - try { - this.file = new BufferedRecords(new Records(f, rowdef.objectsize), this.buffersize); - } catch (final FileNotFoundException e) { - // should never happen - Log.logSevere("Table", "", e); - } + this.file = new BufferedRecords(new Records(f, rowdef.objectsize), this.buffersize); // initialize index and copy table table = (table == null) ? null : new RowSet(taildef);