From 9ae9062bd397edba3d3c06468c152670f72dabd7 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sat, 12 Aug 2006 00:58:43 +0000 Subject: [PATCH] * disabled new kelondroFlex table for NURLs * added new RAM index Class * fixed possible synchronization problem in kelondroRecords git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2388 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../kelondro/kelondroCollectionIndex.java | 3 - .../de/anomic/kelondro/kelondroFlexTable.java | 4 +- .../de/anomic/kelondro/kelondroRAMIndex.java | 78 +++++++++++++++++++ .../de/anomic/kelondro/kelondroRecords.java | 6 +- source/de/anomic/plasma/plasmaCrawlNURL.java | 6 +- 5 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 source/de/anomic/kelondro/kelondroRAMIndex.java diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 9d85428ba..4fddce3af 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -81,9 +81,6 @@ public class kelondroCollectionIndex { return new File(path, filenameStub + "." + lf + "." + cs + ".properties"); // kelondro collection array } - /* - - */ public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroOrder indexOrder, long buffersize, long preloadTime, int loadfactor, kelondroRow rowdef) throws IOException { diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 294346a20..92fdd03b9 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -130,8 +130,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr public synchronized kelondroRow.Entry get(byte[] key) throws IOException { synchronized (index) { int i = index.geti(key); - if (i >= this.size()) System.out.println("error"); if (i < 0) return null; + // i may be greater than this.size(), because this table may have deleted entries + // the deleted entries are subtracted from the 'real' tablesize, so the size may be + // smaller than an index to a row entry return super.get(i); } } diff --git a/source/de/anomic/kelondro/kelondroRAMIndex.java b/source/de/anomic/kelondro/kelondroRAMIndex.java new file mode 100644 index 000000000..0c36da603 --- /dev/null +++ b/source/de/anomic/kelondro/kelondroRAMIndex.java @@ -0,0 +1,78 @@ +// kelondroRAMIndex.java +// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany +// first published 12.08.2006 on http://www.anomic.de +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +import java.util.Iterator; +import java.util.TreeMap; + +import de.anomic.kelondro.kelondroRow.Entry; + +public class kelondroRAMIndex implements kelondroIndex { + + private TreeMap index; + private kelondroOrder order; + private kelondroRow rowdef; + + public kelondroRAMIndex(kelondroOrder defaultOrder, kelondroRow rowdef) { + this.index = new TreeMap(defaultOrder); + this.order = defaultOrder; + this.rowdef = rowdef; + } + + public kelondroOrder order() { + return this.order; + } + + public int size() { + return this.index.size(); + } + + public kelondroRow row() { + return this.rowdef; + } + + public Entry get(byte[] key) { + return (kelondroRow.Entry) index.get(key); + } + + public Entry put(Entry row) { + return (kelondroRow.Entry) index.put(row.getColBytes(0), row); + } + + public Entry remove(byte[] key) { + return (kelondroRow.Entry) index.remove(key); + } + + public Iterator rows(boolean up, boolean rotating, byte[] firstKey) { + return index.values().iterator(); + } + + public void close() { + index = null; + } + +} diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 6e7220e7e..7fcbaba36 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -536,9 +536,11 @@ public class kelondroRecords { synchronized (cacheHeaders) { cacheHeaders.removeb(handle.index); cacheDelete++; + dispose(handle); } + } else { + dispose(handle); } - dispose(handle); } public final class Node { @@ -740,7 +742,7 @@ public class kelondroRecords { public byte[] setValueRow(byte[] row) throws IOException { // if the index is defined, then write values directly to the file, else only to the object - assert row.length == ROW.objectsize(); + if (row.length != ROW.objectsize()) throw new IOException("setValueRow with wrong (" + row.length + ") row length instead correct: " + ROW.objectsize()); byte[] result = getValueRow(); // previous value (this loads the values if not already happened) // set values diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index 0143b84fe..87ef8e982 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -167,14 +167,14 @@ public class plasmaCrawlNURL extends indexURL { private void openHashCache() { File oldCacheFile = new File(cacheStacksPath, "urlNotice1.db"); - File newCacheFile = new File(cacheStacksPath, "urlNotice2.table"); - if (newCacheFile.exists()) try { + //File newCacheFile = new File(cacheStacksPath, "urlNotice2.table"); + /*if (newCacheFile.exists()) try { urlHashCache = new kelondroFlexTable(cacheStacksPath, "urlNotice2.table", kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef, true); } catch (IOException e) { e.printStackTrace(); oldCacheFile.delete(); urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true); - } else if (oldCacheFile.exists()) try { + } else*/ if (oldCacheFile.exists()) try { // open existing cache kelondroTree tree = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent); tree.assignRowdef(rowdef);