From 24856810020a8ac195f610239b12d669f98a9739 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 25 Jan 2008 11:44:27 +0000 Subject: [PATCH] added termination control for RotateIterator git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4399 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/IndexControlURLs_p.java | 2 +- source/de/anomic/kelondro/kelondroCollectionIndex.java | 2 +- source/de/anomic/kelondro/kelondroDyn.java | 2 +- source/de/anomic/kelondro/kelondroEcoFS.java | 4 ++-- source/de/anomic/kelondro/kelondroEcoTable.java | 5 ++++- source/de/anomic/kelondro/kelondroMapTable.java | 2 +- source/de/anomic/kelondro/kelondroObjects.java | 2 +- source/de/anomic/kelondro/kelondroRotateIterator.java | 9 ++++++--- source/de/anomic/plasma/plasmaWordIndex.java | 2 +- source/de/anomic/yacy/yacyDHTAction.java | 2 +- 10 files changed, 19 insertions(+), 13 deletions(-) diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index c28b62f00..f393c0c1b 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -164,7 +164,7 @@ public class IndexControlURLs_p { // generate list if (post.containsKey("urlhashsimilar")) { try { - final Iterator entryIt = new kelondroRotateIterator(sb.wordIndex.loadedURL.entries(true, urlhash), new String(kelondroBase64Order.zero(urlhash.length()))); + final Iterator entryIt = new kelondroRotateIterator(sb.wordIndex.loadedURL.entries(true, urlhash), new String(kelondroBase64Order.zero(urlhash.length())), sb.wordIndex.size()); StringBuffer result = new StringBuffer("Sequential List of URL-Hashes:
"); indexURLEntry entry; int i = 0; diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 63c928ddd..1752e5481 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -1049,7 +1049,7 @@ public class kelondroCollectionIndex { public keycollectionIterator(byte[] startKey, byte[] secondKey, boolean rot) throws IOException { // iterator of {byte[], kelondroRowSet} Objects kelondroCloneableIterator i = index.rows(true, startKey); - indexRowIterator = (rot) ? new kelondroRotateIterator(i, secondKey) : i; + indexRowIterator = (rot) ? new kelondroRotateIterator(i, secondKey, index.size()) : i; } public boolean hasNext() { diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index 0b34dc009..d48dce0b0 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -219,7 +219,7 @@ public class kelondroDyn { // iterates only the keys of the Nodes // enumerated objects are of type String dynKeyIterator i = new dynKeyIterator(index.rows(up, null)); - if (rotating) return new kelondroRotateIterator(i, null); else return i; + if (rotating) return new kelondroRotateIterator(i, null, index.size()); else return i; } public synchronized dynKeyIterator dynKeys(boolean up, byte[] firstKey) throws IOException { diff --git a/source/de/anomic/kelondro/kelondroEcoFS.java b/source/de/anomic/kelondro/kelondroEcoFS.java index f34279d62..a391dbacf 100644 --- a/source/de/anomic/kelondro/kelondroEcoFS.java +++ b/source/de/anomic/kelondro/kelondroEcoFS.java @@ -39,14 +39,14 @@ public class kelondroEcoFS { * The access rules are in such a way that a minimum of IO operations are necessary * Two caches provide a mirror to content in the file: a read cache and a write buffer * The read cache contains a number of entries from the file; a mirror that moves - * whenever information outsite the mirror is requested. + * whenever information outside the mirror is requested. * The write buffer always exists only at the end of the file. It contains only records * that have never been written to the file before. When the write buffer is flushed, * the file grows * The record file may also shrink when the last entry of the file is removed. * Removal of Entries inside the file is not possible, but such entries can be erased * by overwriting the data with zero bytes - * All access to the file is made with byte[] that are generated outsite of this class + * All access to the file is made with byte[] that are generated outside of this class * This class only references byte[] that are handed over to methods of this class. */ diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index 478f9ff35..939b0a50f 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -119,7 +119,7 @@ public class kelondroEcoTable implements kelondroIndex { byte[] record = new byte[rowdef.objectsize]; byte[] key = new byte[rowdef.primaryKeyLength]; int fs = (int) file.size(); - System.out.print("*** initializing RAM index for EcoTable " + tablefile + ":"); + System.out.print("*** initializing RAM index for EcoTable " + tablefile.getName() + ":"); for (int i = 0; i < fs; i++) { // read entry file.get(i, record, 0); @@ -325,6 +325,9 @@ public class kelondroEcoTable implements kelondroIndex { public synchronized Entry put(Entry row) throws IOException { assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert ((table == null) || (table.size() == index.size())); + assert row != null; + assert row.bytes() != null; + if ((row == null) || (row.bytes() == null)) return null; int i = index.geti(row.getPrimaryKeyBytes()); if (i == -1) { addUnique(row); diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index 9dec9d428..7d0bbe00f 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -147,7 +147,7 @@ public class kelondroMapTable { kelondroIndex tree = (kelondroIndex) tTables.get(tablename); if (tree == null) throw new RuntimeException("kelondroTables.bytes: tree table '" + tablename + "' does not exist."); kelondroCloneableIterator i = tree.rows(up, firstKey); - if (rotating) return new kelondroRotateIterator(i, secondKey); else return i; + if (rotating) return new kelondroRotateIterator(i, secondKey, tree.size()); else return i; } // if you need the long-values from a row-iteration, please use kelondroRecords.bytes2long to convert from byte[] to long diff --git a/source/de/anomic/kelondro/kelondroObjects.java b/source/de/anomic/kelondro/kelondroObjects.java index 9491cf7f0..4b50ab558 100644 --- a/source/de/anomic/kelondro/kelondroObjects.java +++ b/source/de/anomic/kelondro/kelondroObjects.java @@ -140,7 +140,7 @@ public class kelondroObjects { public synchronized kelondroCloneableIterator keys(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException { // simple enumeration of key names without special ordering kelondroCloneableIterator i = dyn.dynKeys(up, firstKey); - if (rotating) return new kelondroRotateIterator(i, secondKey); else return i; + if (rotating) return new kelondroRotateIterator(i, secondKey, dyn.sizeDyn()); else return i; } diff --git a/source/de/anomic/kelondro/kelondroRotateIterator.java b/source/de/anomic/kelondro/kelondroRotateIterator.java index d5d6c115e..7dc88f1f5 100644 --- a/source/de/anomic/kelondro/kelondroRotateIterator.java +++ b/source/de/anomic/kelondro/kelondroRotateIterator.java @@ -31,21 +31,23 @@ public class kelondroRotateIterator implements kelondroCloneableIterator { kelondroCloneableIterator a, clone; Object modifier; boolean nempty; + int terminationCount; - public kelondroRotateIterator(kelondroCloneableIterator a, Object modifier) { + public kelondroRotateIterator(kelondroCloneableIterator a, Object modifier, int terminationCount) { // this works currently only for String-type key iterations this.a = a; this.modifier = modifier; + this.terminationCount = terminationCount; this.clone = (kelondroCloneableIterator) a.clone(modifier); this.nempty = this.clone.hasNext(); } public kelondroRotateIterator clone(Object modifier) { - return new kelondroRotateIterator(a, modifier); + return new kelondroRotateIterator(a, modifier, terminationCount - 1); } public boolean hasNext() { - return this.nempty; + return (terminationCount > 0) && (this.nempty); } public E next() { @@ -56,6 +58,7 @@ public class kelondroRotateIterator implements kelondroCloneableIterator { a = (kelondroCloneableIterator) clone.clone(modifier); assert a.hasNext(); } + terminationCount--; return a.next(); } diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 82019cf44..ce2410611 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -552,7 +552,7 @@ public final class plasmaWordIndex implements indexRI { public synchronized kelondroCloneableIterator wordContainers(String startHash, boolean ram, boolean rot) { kelondroCloneableIterator i = wordContainers(startHash, ram); if (rot) { - return new kelondroRotateIterator(i, new String(kelondroBase64Order.zero(startHash.length()))); + return new kelondroRotateIterator(i, new String(kelondroBase64Order.zero(startHash.length())), dhtOutCache.size() + ((ram) ? 0 : collections.size())); } else { return i; } diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index cfc3acf6d..8b400ca6e 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -278,7 +278,7 @@ public class yacyDHTAction implements yacyPeerAction { public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeMap clusterhashes) { // clusterhashes is a String(hash)/String(IP) - mapping - kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableMapIterator(clusterhashes, urlHash), null); + kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableMapIterator(clusterhashes, urlHash), null, clusterhashes.size()); String hash; int count = clusterhashes.size(); // counter to ensure termination while ((i.hasNext()) && (count-- > 0)) {