boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fine-tuning of cache usage from SVN 5386 and a bug fix for overflow in available() method

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5387 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 1779c3c507
commit 5b94498643
10 changed files with 269 additions and 178 deletions
Show Stats Download Patch File Download Diff File

10
source/de/anomic/kelondro/kelondroAbstractRA.java
Unescape View File

@ -51,17 +51,19 @@ abstract class kelondroAbstractRA implements kelondroRA {
// pseudo-native methods: // pseudo-native methods:
abstract public void readFully(byte[] b, int off, int len) throws IOException; abstract public void readFully(byte[] b, int off, int len) throws IOException;
abstract public long length() throws IOException; abstract public long length() throws IOException;
abstract public int available() throws IOException; abstract public void setLength(long length) throws IOException;
abstract public long available() throws IOException;
abstract public void write(byte[] b, int off, int len) throws IOException; abstract public void write(byte[] b, int off, int len) throws IOException;
abstract public void seek(long pos) throws IOException; abstract public void seek(long pos) throws IOException;
abstract public void close() throws IOException; abstract public void close() throws IOException;
// derived methods: // derived methods:
public byte[] readFully() throws IOException { public byte[] readFully() throws IOException {
int a = this.available(); long a = this.available();
if (a <= 0) return null; if (a <= 0) return null;
final byte[] buffer = new byte[a]; if (a > Integer.MAX_VALUE) throw new IOException("available too large for a single array");
this.readFully(buffer, 0, a); final byte[] buffer = new byte[(int) a];
this.readFully(buffer, 0, (int) a);
return buffer; return buffer;
} }

6
source/de/anomic/kelondro/kelondroAbstractRecords.java
Unescape View File

@ -970,7 +970,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
try { try {
nn = next00(); nn = next00();
} catch (final IOException e) { } catch (final IOException e) {
serverLog.logSevere("kelondroCachedRecords", filename + " failed with " + e.getMessage(), e); serverLog.logSevere("kelondroAbstractRecords", filename + " failed with " + e.getMessage(), e);
return null; return null;
} }
byte[] key = null; byte[] key = null;
@ -1001,9 +1001,9 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
bulkstart = pos.index; bulkstart = pos.index;
final int maxlength = Math.min(USAGE.allCount() - bulkstart, bulksize); final int maxlength = Math.min(USAGE.allCount() - bulkstart, bulksize);
if (((POS_NODES) + ((long) bulkstart) * ((long) recordsize)) < 0) if (((POS_NODES) + ((long) bulkstart) * ((long) recordsize)) < 0)
serverLog.logSevere("kelondroCachedRecords", "DEBUG: negative offset. POS_NODES = " + POS_NODES + ", bulkstart = " + bulkstart + ", recordsize = " + recordsize); serverLog.logSevere("kelondroAbstractRecords", "DEBUG: negative offset. POS_NODES = " + POS_NODES + ", bulkstart = " + bulkstart + ", recordsize = " + recordsize);
if ((maxlength * recordsize) < 0) if ((maxlength * recordsize) < 0)
serverLog.logSevere("kelondroCachedRecords", "DEBUG: negative length. maxlength = " + maxlength + ", recordsize = " + recordsize); serverLog.logSevere("kelondroAbstractRecords", "DEBUG: negative length. maxlength = " + maxlength + ", recordsize = " + recordsize);
entryFile.readFully((POS_NODES) + ((long) bulkstart) * ((long) recordsize), bulk, 0, maxlength * recordsize); entryFile.readFully((POS_NODES) + ((long) bulkstart) * ((long) recordsize), bulk, 0, maxlength * recordsize);
} }
/* POS_NODES = 302, bulkstart = 3277, recordsize = 655386 /* POS_NODES = 302, bulkstart = 3277, recordsize = 655386

6
source/de/anomic/kelondro/kelondroBLOBHeap.java
Unescape View File

@ -45,7 +45,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
private TreeMap<Long, Integer> free; // list of {size, seek} pairs denoting space and position of free records private TreeMap<Long, Integer> free; // list of {size, seek} pairs denoting space and position of free records
private final File heapFile; // the file of the heap private final File heapFile; // the file of the heap
private final kelondroByteOrder ordering; // the ordering on keys private final kelondroByteOrder ordering; // the ordering on keys
private kelondroFileRA file; // a random access to the file private kelondroCachedFileRA file; // a random access to the file
private HashMap<String, byte[]> buffer; // a write buffer to limit IO to the file; attention: Maps cannot use byte[] as key private HashMap<String, byte[]> buffer; // a write buffer to limit IO to the file; attention: Maps cannot use byte[] as key
private int buffersize; // bytes that are buffered in buffer private int buffersize; // bytes that are buffered in buffer
private int buffermax; // maximum size of the buffer private int buffermax; // maximum size of the buffer
@ -89,7 +89,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
this.free = new TreeMap<Long, Integer>(); this.free = new TreeMap<Long, Integer>();
this.buffer = new HashMap<String, byte[]>(); this.buffer = new HashMap<String, byte[]>();
this.buffersize = 0; this.buffersize = 0;
this.file = new kelondroFileRA(heapFile); this.file = new kelondroCachedFileRA(heapFile);
byte[] key = new byte[keylength]; byte[] key = new byte[keylength];
int reclen; int reclen;
long seek = 0; long seek = 0;
@ -356,7 +356,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
e.printStackTrace(); e.printStackTrace();
} }
this.heapFile.delete(); this.heapFile.delete();
this.file = new kelondroFileRA(heapFile); this.file = new kelondroCachedFileRA(heapFile);
} }
/** /**

8
source/de/anomic/kelondro/kelondroBLOBTree.java
Unescape View File

@ -417,8 +417,12 @@ public class kelondroBLOBTree implements kelondroBLOB {
return p-1; return p-1;
} }
public int available() throws IOException { public void setLength(long length) throws IOException {
return (int) (length() - seekpos); throw new UnsupportedOperationException();
}
public long available() throws IOException {
return length() - seekpos;
} }
public int read() throws IOException { public int read() throws IOException {

6
source/de/anomic/kelondro/kelondroBufferedRA.java
Unescape View File

@ -45,7 +45,7 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
return this.sbb; return this.sbb;
} }
public int available() throws IOException { public long available() throws IOException {
return Integer.MAX_VALUE - sbb.length(); return Integer.MAX_VALUE - sbb.length();
} }
@ -57,6 +57,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
return sbb.length(); return sbb.length();
} }
public void setLength(long length) throws IOException {
sbb.resize((int) length);
}
public int read() throws IOException { public int read() throws IOException {
return 0xff & sbb.byteAt((int) pos++); return 0xff & sbb.byteAt((int) pos++);
} }

134
source/de/anomic/kelondro/kelondroCachedFileRA.java
Unescape View File

@ -0,0 +1,134 @@
// kelondroCachedFileRA.java
// -----------------------
// part of The Kelondro Database
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://yacy.net
// Frankfurt, Germany, 2004-2008
// last major change: 09.12.2008
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
public final class kelondroCachedFileRA extends kelondroAbstractRA implements kelondroRA {
private RandomAccessFile RAFile;
private byte[] cache;
private long cachestart;
private int cachelen;
public kelondroCachedFileRA(final File file) throws IOException, FileNotFoundException {
this.name = file.getName();
this.file = file;
RAFile = new RandomAccessFile(file, "rw");
cache = new byte[8192];
cachestart = 0;
cachelen = 0;
}
public synchronized long length() throws IOException {
return this.RAFile.length();
}
public synchronized void setLength(long length) throws IOException {
cachelen = 0;
RAFile.setLength(length);
}
public synchronized long available() throws IOException {
return this.length() - RAFile.getFilePointer();
}
public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException {
long seek = RAFile.getFilePointer();
if (cache != null && cachestart <= seek && cachelen - seek + cachestart >= len) {
// read from cache
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": CACHE HIT at " + seek);
System.arraycopy(cache, (int) (seek - cachestart), b, off, len);
RAFile.seek(seek + len);
return;
}
if (cache == null || cache.length < len) {
// cannot fill cache here
RAFile.readFully(b, off, len);
return;
}
// we fill the cache here
int available = (int) (this.RAFile.length() - seek);
if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len);
if (cachestart + cachelen == seek && cache.length - cachelen >= len) {
RAFile.readFully(cache, cachelen, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes");
System.arraycopy(cache, cachelen, b, off, len);
cachelen += len;
} else {
// fill the cache as much as possible
int m = Math.min(available, cache.length);
RAFile.readFully(cache, 0, m);
cachestart = seek;
cachelen = m;
if (m != len) RAFile.seek(seek + len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": replace fill " + len + " bytes");
System.arraycopy(cache, 0, b, off, len);
}
}
public synchronized void write(final byte[] b, final int off, final int len) throws IOException {
//assert len > 0;
// write to file
if (this.cache.length > 512) this.cache = new byte[512]; // the large cache is only useful during an initialization phase
long seekpos = this.RAFile.getFilePointer();
if (this.cachelen + len <= this.cache.length && this.cachestart + this.cachelen == seekpos) {
// append to cache
System.arraycopy(b, off, this.cache, this.cachelen, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write append " + len + " bytes");
this.cachelen += len;
} else if (len <= this.cache.length) {
// copy to cache
System.arraycopy(b, off, this.cache, 0, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write copy " + len + " bytes");
this.cachelen = len;
this.cachestart = seekpos;
} else {
// delete cache
this.cachelen = 0;
}
RAFile.write(b, off, len);
}
public synchronized void seek(final long pos) throws IOException {
RAFile.seek(pos);
}
public synchronized void close() throws IOException {
if (RAFile != null) RAFile.close();
this.cache = null;
this.RAFile = null;
}
protected void finalize() throws Throwable {
if (RAFile != null) {
this.close();
}
super.finalize();
}
}

8
source/de/anomic/kelondro/kelondroChannelRA.java
Unescape View File

@ -45,8 +45,12 @@ public final class kelondroChannelRA extends kelondroAbstractRA implements kelon
return channel.size(); return channel.size();
} }
public int available() throws IOException { public void setLength(long length) throws IOException {
return (int) (channel.size() - channel.position()); channel.truncate(length);
}
public long available() throws IOException {
return channel.size() - channel.position();
} }
public final void readFully(final byte[] b, final int off, final int len) throws IOException { public final void readFully(final byte[] b, final int off, final int len) throws IOException {

2
source/de/anomic/kelondro/kelondroEcoFS.java
Unescape View File

@ -92,7 +92,7 @@ public class kelondroEcoFS {
// open an existing table file // open an existing table file
try { try {
raf = new RandomAccessFile(tablefile, "rw"); raf = new RandomAccessFile(tablefile,"rw");
} catch (final FileNotFoundException e) { } catch (final FileNotFoundException e) {
// should never happen // should never happen
e.printStackTrace(); e.printStackTrace();

264
source/de/anomic/kelondro/kelondroFileRA.java
Unescape View File

@ -1,161 +1,103 @@
// kelondroFileRA.java // kelondroFileRA.java
// ----------------------- // -----------------------
// part of The Kelondro Database // part of The Kelondro Database
// (C) by Michael Peter Christen; mc@yacy.net // (C) by Michael Peter Christen; mc@yacy.net
// first published on http://yacy.net // first published on http://yacy.net
// Frankfurt, Germany, 2004-2008 // Frankfurt, Germany, 2004-2008
// last major change: 09.12.2008 // last major change: 09.12.2008
// //
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by // it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or // the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version. // (at your option) any later version.
// //
// This program is distributed in the hope that it will be useful, // This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of // but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details. // GNU General Public License for more details.
// //
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro; package de.anomic.kelondro;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.util.Map; import java.util.Map;
public final class kelondroFileRA extends kelondroAbstractRA implements kelondroRA { public final class kelondroFileRA extends kelondroAbstractRA implements kelondroRA {
private RandomAccessFile RAFile; private RandomAccessFile RAFile;
private byte[] cache;
private long cachestart; public kelondroFileRA(final File file) throws IOException, FileNotFoundException {
private int cachelen; this.name = file.getName();
this.file = file;
public kelondroFileRA(final File file) throws IOException, FileNotFoundException { RAFile = new RandomAccessFile(file, "rw");
this.name = file.getName(); }
this.file = file;
RAFile = new RandomAccessFile(file, "rw"); public synchronized long length() throws IOException {
cache = new byte[8192]; return this.RAFile.length();
cachestart = 0; }
cachelen = 0;
} public synchronized void setLength(long length) throws IOException {
RAFile.setLength(length);
public synchronized long length() throws IOException { }
return this.RAFile.length();
} public synchronized long available() throws IOException {
return this.length() - RAFile.getFilePointer();
public synchronized void setLength(long length) throws IOException { }
cachelen = 0;
RAFile.setLength(length); public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException {
} RAFile.readFully(b, off, len);
}
public synchronized int available() throws IOException {
return (int) (this.length() - RAFile.getFilePointer()); public synchronized void write(final byte[] b, final int off, final int len) throws IOException {
} RAFile.write(b, off, len);
}
public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException {
long seek = RAFile.getFilePointer(); public synchronized void seek(final long pos) throws IOException {
if (cache != null && cachestart <= seek && cachelen - seek + cachestart >= len) { RAFile.seek(pos);
// read from cache }
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": CACHE HIT at " + seek);
System.arraycopy(cache, (int) (seek - cachestart), b, off, len); public synchronized void close() throws IOException {
RAFile.seek(seek + len); if (RAFile != null) RAFile.close();
return; this.RAFile = null;
} }
if (cache == null || cache.length < len) {
// cannot fill cache here protected void finalize() throws Throwable {
RAFile.readFully(b, off, len); if (RAFile != null) {
return; this.close();
} }
// we fill the cache here super.finalize();
int available = (int) (this.RAFile.length() - seek); }
if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len);
if (cachestart + cachelen == seek && cache.length - cachelen >= len) { // some static tools
RAFile.readFully(cache, cachelen, len); public static void writeMap(final File f, final Map<String, String> map, final String comment) throws IOException {
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes"); final File fp = f.getParentFile();
System.arraycopy(cache, cachelen, b, off, len); if (fp != null) fp.mkdirs();
cachelen += len; kelondroRA kra = null;
} else { try {
// fill the cache as much as possible kra = new kelondroCachedFileRA(f);
int m = Math.min(available, cache.length); kra.writeMap(map, comment);
RAFile.readFully(cache, 0, m); kra.close();
cachestart = seek; } finally {
cachelen = m; if (kra != null) try {kra.close();}catch(final Exception e){}
if (m != len) RAFile.seek(seek + len); }
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": replace fill " + len + " bytes"); }
System.arraycopy(cache, 0, b, off, len);
} public static Map<String, String> readMap(final File f) throws IOException {
kelondroRA kra = null;
} try {
kra = new kelondroCachedFileRA(f);
public synchronized void write(final byte[] b, final int off, final int len) throws IOException { final Map<String, String> map = kra.readMap();
//assert len > 0; kra.close();
// write to file return map;
//if (this.cache.length > 2048) this.cache = new byte[2048]; // the large cache is only useful during an initialization phase } finally {
long seekpos = this.RAFile.getFilePointer(); if (kra != null) try {kra.close();}catch(final Exception e){}
if (this.cachelen + len <= this.cache.length && this.cachestart + this.cachelen == seekpos) { }
// append to cache }
System.arraycopy(b, off, this.cache, this.cachelen, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write append " + len + " bytes"); }
this.cachelen += len;
} else if (len <= this.cache.length) {
// copy to cache
System.arraycopy(b, off, this.cache, 0, len);
//System.out.println("*** DEBUG FileRA " + this.file.getName() + ": write copy " + len + " bytes");
this.cachelen = len;
this.cachestart = seekpos;
} else {
// delete cache
this.cachelen = 0;
}
RAFile.write(b, off, len);
}
public synchronized void seek(final long pos) throws IOException {
RAFile.seek(pos);
}
public synchronized void close() throws IOException {
if (RAFile != null) RAFile.close();
this.cache = null;
this.RAFile = null;
}
protected void finalize() throws Throwable {
if (RAFile != null) {
this.close();
}
super.finalize();
}
// some static tools
public static void writeMap(final File f, final Map<String, String> map, final String comment) throws IOException {
final File fp = f.getParentFile();
if (fp != null) fp.mkdirs();
kelondroRA kra = null;
try {
kra = new kelondroFileRA(f);
kra.writeMap(map, comment);
kra.close();
} finally {
if (kra != null) try {kra.close();}catch(final Exception e){}
}
}
public static Map<String, String> readMap(final File f) throws IOException {
kelondroRA kra = null;
try {
kra = new kelondroFileRA(f);
final Map<String, String> map = kra.readMap();
kra.close();
return map;
} finally {
if (kra != null) try {kra.close();}catch(final Exception e){}
}
}
}

3
source/de/anomic/kelondro/kelondroRA.java
Unescape View File

@ -43,7 +43,8 @@ public interface kelondroRA {
// pseudo-native methods: // pseudo-native methods:
public long length() throws IOException; public long length() throws IOException;
public int available() throws IOException; public void setLength(long length) throws IOException;
public long available() throws IOException;
public void readFully(byte[] b, int off, int len) throws IOException; public void readFully(byte[] b, int off, int len) throws IOException;

Write Preview
Loading…
Cancel
Save