diff --git a/source/de/anomic/kelondro/kelondroBufferedIOChunks.java b/source/de/anomic/kelondro/kelondroBufferedIOChunks.java new file mode 100644 index 000000000..4cf587449 --- /dev/null +++ b/source/de/anomic/kelondro/kelondroBufferedIOChunks.java @@ -0,0 +1,156 @@ +// kelondroBufferedIOChunks.java +// ----------------------- +// part of The Kelondro Database +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2005 +// created: 11.12.2004 +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notive above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +// this is a WRITE-buffer! +// the buffer MUST be flushed before closing of the underlying kelondroRA + +package de.anomic.kelondro; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks implements kelondroIOChunks { + + protected kelondroRA ra; + private int bufferkb; + private long commitTimeout; + private HashMap buffer; + private long lastCommit = 0; + + + public kelondroBufferedIOChunks(kelondroRA ra, String name, int bufferkb, long commitTimeout) { + this.name = name; + this.ra = ra; + this.bufferkb = bufferkb; + this.commitTimeout = commitTimeout; + this.buffer = new HashMap(); + this.lastCommit = System.currentTimeMillis(); + } + + public int read(long pos, byte[] b, int off, int len) throws IOException { + assert (b.length >= off + len): "read pos=" + pos + ", b.length=" + b.length + ", off=" + off + ", len=" + len; + + // check commit time + if (this.lastCommit + this.commitTimeout > System.currentTimeMillis()) { + commit(); + this.lastCommit = System.currentTimeMillis(); + } + + // do the read + if ((off == 0) && (b.length == len)) { + synchronized (this.buffer) { + byte[] bb = (byte[]) buffer.get(new Long(pos)); + if (bb == null) { + synchronized (this.ra) { + this.ra.seek(pos); + return ra.read(b, off, len); + } + } else { + if (bb.length >= len) { + System.arraycopy(bb, 0, b, off, len); + return len; + } else { + System.arraycopy(bb, 0, b, off, bb.length); + return bb.length; + } + } + } + } else { + byte[] bb = new byte[len]; + int r = read(pos + off, bb, 0, len); + System.arraycopy(bb, 0, b, off, r); + return r; + } + } + + public void write(long pos, byte[] b, int off, int len) throws IOException { + assert (b.length >= off + len): "write pos=" + pos + ", b.length=" + b.length + ", b='" + new String(b) + "', off=" + off + ", len=" + len; + + // do the write into buffer + byte[] bb = new byte[len]; + System.arraycopy(b, off, bb, 0, len); + synchronized (buffer) { + buffer.put(new Long(pos), bb); + } + + // check commit time + if (this.lastCommit + this.commitTimeout > System.currentTimeMillis()) { + commit(); + this.lastCommit = System.currentTimeMillis(); + } + } + + public void commit() throws IOException { + synchronized (buffer) { + if (buffer.size() == 0) return; + Iterator i = buffer.entrySet().iterator(); + Map.Entry entry; + long pos; + byte[] b; + synchronized (this.ra) { + while (i.hasNext()) { + entry = (Map.Entry) i.next(); + pos = ((Long) entry.getKey()).longValue(); + b = (byte[]) entry.getValue(); + this.ra.seek(pos); + this.ra.write(b); + } + } + buffer.clear(); + } + } + + public void close() throws IOException { + if (this.ra != null) { + commit(); + this.ra.close(); + } + this.ra = null; + } + + protected void finalize() throws Throwable { + if (this.ra != null) this.close(); + super.finalize(); + } + +} diff --git a/source/de/anomic/kelondro/kelondroRAIOChunks.java b/source/de/anomic/kelondro/kelondroRAIOChunks.java index 08933580a..e91218827 100644 --- a/source/de/anomic/kelondro/kelondroRAIOChunks.java +++ b/source/de/anomic/kelondro/kelondroRAIOChunks.java @@ -4,7 +4,7 @@ // (C) by Michael Peter Christen; mc@anomic.de // first published on http://www.anomic.de // Frankfurt, Germany, 2005 -// created: 1.12.2004 +// created: 11.12.2004 // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 3c0b9bfb6..77a513c32 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -219,6 +219,7 @@ public class kelondroRecords { int[] columns, int FHandles, int txtProps, int txtPropWidth) throws IOException { // create new Chunked IO + //this.entryFile = new kelondroBufferedIOChunks(ra, ra.name(), 1024, 5000); this.entryFile = new kelondroRAIOChunks(ra, ra.name()); // store dynamic run-time data @@ -319,6 +320,7 @@ public class kelondroRecords { private void init(kelondroRA ra) throws IOException { // read from Chunked IO + //this.entryFile = new kelondroBufferedIOChunks(ra, ra.name(), 1024, 5000); this.entryFile = new kelondroRAIOChunks(ra, ra.name()); // read dynamic variables that are back-ups of stored values in file; diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index d64cd760d..0a91db140 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -925,6 +925,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser } public boolean coreCrawlJob() { + try { if (urlPool.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) == 0) { //log.logDebug("CoreCrawl: queue is empty"); return false; @@ -993,6 +994,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser processLocalCrawling(urlEntry, profile, stats); return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } } public int limitCrawlTriggerJobSize() {