From 27d00285aa9239e0420368c695795224e7260155 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 9 Sep 2009 21:28:23 +0000 Subject: [PATCH] - added a new file reader cache that may serve as full-file-copy of blob database files. This is not yet used - removed class FileWriter and replaced all usage of that class with CachedFileWriter git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6309 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../de/anomic/kelondro/blob/HeapReader.java | 17 ++- .../kelondro/io/random/CachedFileReader.java | 91 ++++++++++++++++ .../anomic/kelondro/io/random/FileWriter.java | 103 ------------------ source/de/anomic/kelondro/table/Records.java | 10 +- 4 files changed, 109 insertions(+), 112 deletions(-) create mode 100644 source/de/anomic/kelondro/io/random/CachedFileReader.java delete mode 100644 source/de/anomic/kelondro/io/random/FileWriter.java diff --git a/source/de/anomic/kelondro/blob/HeapReader.java b/source/de/anomic/kelondro/blob/HeapReader.java index a5442fc58..b2c9e4e1d 100644 --- a/source/de/anomic/kelondro/blob/HeapReader.java +++ b/source/de/anomic/kelondro/blob/HeapReader.java @@ -36,6 +36,7 @@ import java.util.concurrent.ExecutionException; import de.anomic.kelondro.index.HandleMap; import de.anomic.kelondro.io.random.CachedFileWriter; +import de.anomic.kelondro.io.random.Writer; import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.RotateIterator; @@ -47,12 +48,15 @@ public class HeapReader { public final static long keepFreeMem = 20 * 1024 * 1024; + // input values protected int keylength; // the length of the primary key - protected HandleMap index; // key/seek relation for used records - protected Gap free; // set of {seek, size} pairs denoting space and position of free records protected File heapFile; // the file of the heap protected final ByteOrder ordering; // the ordering on keys - protected CachedFileWriter file; // a random access to the file + + // computed values + protected Writer file; // a random access to the file + protected HandleMap index; // key/seek relation for used records + protected Gap free; // set of {seek, size} pairs denoting space and position of free records public HeapReader( final File heapFile, @@ -391,7 +395,12 @@ public class HeapReader { * close the BLOB table */ public synchronized void close(boolean writeIDX) { - if (file != null) file.close(); + if (file != null) + try { + file.close(); + } catch (IOException e) { + e.printStackTrace(); + } file = null; if (writeIDX && index != null && free != null && (index.size() > 3 || free.size() > 3)) { // now we can create a dump of the index and the gap information diff --git a/source/de/anomic/kelondro/io/random/CachedFileReader.java b/source/de/anomic/kelondro/io/random/CachedFileReader.java new file mode 100644 index 000000000..bf014a5ec --- /dev/null +++ b/source/de/anomic/kelondro/io/random/CachedFileReader.java @@ -0,0 +1,91 @@ +// CachedFileReader.java +// --------------------- +// (C) 2009 by Michael Peter Christen; mc@yacy.net +// first published 09.09.2009 on http://yacy.net +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +package de.anomic.kelondro.io.random; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; + +import de.anomic.kelondro.util.MemoryControl; + +public class CachedFileReader extends AbstractReader implements Reader { + + private RandomAccessFile RAFile; + private byte[] cache; + private int cachelen; + + public CachedFileReader(final File file) throws IOException, FileNotFoundException { + this.name = file.getName(); + this.file = file; + this.RAFile = new RandomAccessFile(this.file, "r"); + if (MemoryControl.available() / 10L > this.RAFile.length() && this.RAFile.length() < (long) Integer.MAX_VALUE) { + this.cache = new byte[(int) this.RAFile.length()]; + this.RAFile.seek(0); + this.RAFile.readFully(this.cache); + } else { + this.cache = null; + } + this.cachelen = 0; + } + + public synchronized long available() throws IOException { + return this.length() - RAFile.getFilePointer(); + } + + public synchronized long length() throws IOException { + return this.RAFile.length(); + } + + public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException { + long seek = RAFile.getFilePointer(); + if (cache != null && cachelen - seek >= len) { + // read from cache + System.arraycopy(cache, (int) (seek), b, off, len); + RAFile.seek(seek + len); + return; + } + // cannot use the cache + RAFile.readFully(b, off, len); + return; + } + + public synchronized void seek(final long pos) throws IOException { + RAFile.seek(pos); + } + + public synchronized void close() { + if (RAFile != null) try { + try{RAFile.getChannel().close();} catch (IOException e) {} + RAFile.close(); + } catch (IOException e) { + e.printStackTrace(); + } + this.cache = null; + this.RAFile = null; + } + + protected void finalize() throws Throwable { + this.close(); + super.finalize(); + } + +} diff --git a/source/de/anomic/kelondro/io/random/FileWriter.java b/source/de/anomic/kelondro/io/random/FileWriter.java deleted file mode 100644 index 0b837005f..000000000 --- a/source/de/anomic/kelondro/io/random/FileWriter.java +++ /dev/null @@ -1,103 +0,0 @@ -// RandomAccessFileWriter.java -// ----------------------- -// (C) 2004-2008 by Michael Peter Christen; mc@yacy.net -// first published on http://yacy.net -// Frankfurt, Germany, 09.12.2008 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.kelondro.io.random; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.Map; - - - -public final class FileWriter extends AbstractWriter implements Writer { - - private RandomAccessFile RAFile; - - public FileWriter(final File file) throws IOException, FileNotFoundException { - this.name = file.getName(); - this.file = file; - RAFile = new RandomAccessFile(file, "rw"); - } - - public synchronized long length() throws IOException { - return this.RAFile.length(); - } - - public synchronized void setLength(long length) throws IOException { - RAFile.setLength(length); - } - - public synchronized long available() throws IOException { - return this.length() - RAFile.getFilePointer(); - } - - public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException { - RAFile.readFully(b, off, len); - } - - public synchronized void write(final byte[] b, final int off, final int len) throws IOException { - RAFile.write(b, off, len); - } - - public synchronized void seek(final long pos) throws IOException { - RAFile.seek(pos); - } - - public synchronized void close() throws IOException { - if (RAFile != null) RAFile.close(); - this.RAFile = null; - } - - protected void finalize() throws Throwable { - if (RAFile != null) { - this.close(); - } - super.finalize(); - } - - // some static tools - public static void writeMap(final File f, final Map map, final String comment) throws IOException { - final File fp = f.getParentFile(); - if (fp != null) fp.mkdirs(); - Writer kra = null; - try { - kra = new CachedFileWriter(f); - kra.writeMap(map, comment); - kra.close(); - } finally { - if (kra != null) try {kra.close();}catch(final Exception e){} - } - } - - public static Map readMap(final File f) throws IOException { - Writer kra = null; - try { - kra = new CachedFileWriter(f); - final Map map = kra.readMap(); - kra.close(); - return map; - } finally { - if (kra != null) try {kra.close();}catch(final Exception e){} - } - } - -} diff --git a/source/de/anomic/kelondro/table/Records.java b/source/de/anomic/kelondro/table/Records.java index 159fe9a30..b9f8f9760 100644 --- a/source/de/anomic/kelondro/table/Records.java +++ b/source/de/anomic/kelondro/table/Records.java @@ -43,7 +43,7 @@ import de.anomic.kelondro.index.Row; import de.anomic.kelondro.index.Row.EntryIndex; import de.anomic.kelondro.io.chunks.IOChunksInterface; import de.anomic.kelondro.io.chunks.RandomAccessIOChunks; -import de.anomic.kelondro.io.random.FileWriter; +import de.anomic.kelondro.io.random.CachedFileWriter; import de.anomic.kelondro.io.random.Writer; import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.NaturalOrder; @@ -351,7 +351,7 @@ public class Records { public static int staticsize(final File file) { if (!(file.exists())) return 0; try { - final Writer ra = new FileWriter(new File(file.getCanonicalPath())); + final Writer ra = new CachedFileWriter(new File(file.getCanonicalPath())); final IOChunksInterface entryFile = new RandomAccessIOChunks(ra, ra.name()); final int used = entryFile.readInt(POS_USEDC); // works only if consistency with file size is given @@ -385,7 +385,7 @@ public class Records { if (file.exists()) { // opens an existing tree this.filename = file.getCanonicalPath(); - Writer raf = new FileWriter(new File(this.filename)); + Writer raf = new CachedFileWriter(new File(this.filename)); //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); //kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000); //kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000); @@ -393,7 +393,7 @@ public class Records { initExistingFile(raf); } else { this.filename = file.getCanonicalPath(); - final Writer raf = new FileWriter(new File(this.filename)); + final Writer raf = new CachedFileWriter(new File(this.filename)); // kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); // kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); initNewFile(raf, FHandles, txtProps); @@ -417,7 +417,7 @@ public class Records { assert f != null; this.entryFile.close(); FileUtils.deletedelete(f); - ra = new FileWriter(f); + ra = new CachedFileWriter(f); initNewFile(ra, this.HANDLES.length, this.TXTPROPS.length); }