From 99130490090b5bdeeb465a2f1943da7f6f0f17dd Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 15 Nov 2005 01:20:05 +0000 Subject: [PATCH] fixed outOfMemory bug caused by loops in kelondroTree during enumeration git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1079 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../de/anomic/kelondro/kelondroRecords.java | 14 ++++++++ source/de/anomic/kelondro/kelondroTree.java | 33 ++++++++++++------- .../anomic/plasma/plasmaWordIndexCache.java | 1 + source/de/anomic/tools/enumerateFiles.java | 2 +- source/yacy.java | 1 + 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index e21ebcd3c..2c87d6eb3 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -75,6 +75,7 @@ import java.util.NoSuchElementException; import java.util.StringTokenizer; import java.util.Map; import java.util.Iterator; +import java.util.logging.Logger; public class kelondroRecords { @@ -145,6 +146,8 @@ public class kelondroRecords { protected long XcacheStartup; // startup time; for cache aging protected kelondroMScoreCluster cacheScore; // controls cache aging + // optional logger + protected Logger theLogger = null; public kelondroRecords(File file, long buffersize /* bytes */, short ohbytec, short ohhandlec, @@ -236,6 +239,17 @@ public class kelondroRecords { // thats it! } + public void setLogger(Logger newLogger) { + this.theLogger = newLogger; + } + + public void logWarning(String message) { + if (this.theLogger == null) + System.err.println("KELONDRO WARNING for file " + this.filename + ": " + message); + else + this.theLogger.warning("KELONDRO WARNING for file " + this.filename + ": " + message); + } + public void clear() throws IOException { // Removes all mappings from this map //throw new UnsupportedOperationException("clear not supported"); diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 39ed031b3..eedff1fc8 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -208,17 +208,17 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr parentnode = thenode; thenode = getNode(thisHandle, thenode, (child == -1) ? leftchild : rightchild); } catch (IllegalArgumentException e) { - System.out.println("WARNING: kelondroTree.Search.searchproc: fixed a broken handle"); + logWarning("kelondroTree.Search.process: fixed a broken handle"); found = false; return; } if (thenode == null) { - throw new kelondroException(filename, "kelondroTree.searchproc: thenode==null"); + throw new kelondroException(filename, "kelondroTree.Search.process: thenode==null"); } try { otherkey = new String(thenode.getKey()); } catch (NullPointerException e) { - throw new kelondroException(filename, "kelondroTree.searchproc: nullpointer" + e.getMessage()); + throw new kelondroException(filename, "kelondroTree.Search.process: nullpointer" + e.getMessage()); } if (visitedNodeKeys.containsKey(otherkey)) { // we have loops in the database. @@ -228,6 +228,8 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr thenode.setOHHandle(parent, null); thenode.setOHHandle(leftchild, null); thenode.setOHHandle(rightchild, null); + thenode.commit(CP_NONE); + /* Iterator fix = visitedNodeKeys.entrySet().iterator(); Map.Entry entry; while (fix.hasNext()) { @@ -239,9 +241,10 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr thenode.setOHHandle(leftchild, null); thenode.setOHHandle(rightchild, null); } - thenode.commit(CP_NONE); - //printCache(); - throw new kelondroException(filename, "database contains loops; the loop-nodes have been auto-fixed"); + */ + logWarning("kelondroTree.Search.process: database contains loops; the loop-nodes have been auto-fixed"); + found = false; + return; } //System.out.println("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug c = compare(key, thenode.getKey()); @@ -804,7 +807,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr int c = compare(firstKey, nextNode.getKey()); if ((c > 0) && (up)) { // firstKey > nextNode.getKey() - System.out.println("CORRECTING ITERATOR: firstKey=" + new String(firstKey) + ", nextNode=" + new String(nextNode.getKey())); + logWarning("CORRECTING ITERATOR: firstKey=" + new String(firstKey) + ", nextNode=" + new String(nextNode.getKey())); nextNode = (ii.hasNext()) ? (Node) ii.next() : null; } if ((c < 0) && (!(up))) { @@ -871,9 +874,9 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr // go to next node searchHandle = searchNode.getOHHandle(ct); - if (searchHandle == null) throw new kelondroException(filename, "start node does not exist (handle null)"); + if (searchHandle == null) throw new kelondroException(filename, "nodeIterator.init: start node does not exist (handle null)"); searchNode = getNode(searchHandle, searchNode, ct); - if (searchNode == null) throw new kelondroException(filename, "start node does not exist (node null)"); + if (searchNode == null) throw new kelondroException(filename, "nodeIterator.init: start node does not exist (node null)"); } // now every parent node to the start node is on the stack } @@ -894,8 +897,8 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr } catch (IOException e) { throw new kelondroException(filename, "io-error while rot"); } - if (nextNode == null) throw new kelondroException(filename, "no more entries available"); - if ((count > size()) && (!(rot))) throw new kelondroException(filename, "internal loopback; database corrupted"); + if (nextNode == null) throw new kelondroException(filename, "nodeIterator.next: no more entries available"); + if ((count > size()) && (!(rot))) throw new kelondroException(filename, "nodeIterator.next: internal loopback; database corrupted"); Object ret = nextNode; // middle-case @@ -906,10 +909,18 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr if (childHandle != null) { //System.out.println("go to other leg, stack size=" + nodeStack.size()); // we have walked one leg of the tree; now go to the other one: step down to next child + HashSet visitedNodeHandles = new HashSet(); // to detect loops nodeStack.addLast(new Object[]{nextNode, new Integer(childtype)}); nextNode = getNode(childHandle, nextNode, childtype); childtype = (up) ? leftchild : rightchild; while ((childHandle = nextNode.getOHHandle(childtype)) != null) { + if (visitedNodeHandles.contains(childHandle)) { + // try to repair the nextNode + nextNode.setOHHandle(childtype, null); + logWarning("nodeIterator.next: internal loopback; fixed loop and try to go on"); + break; + } + visitedNodeHandles.add(childHandle); try { nodeStack.addLast(new Object[]{nextNode, new Integer(childtype)}); nextNode = getNode(childHandle, nextNode, childtype); diff --git a/source/de/anomic/plasma/plasmaWordIndexCache.java b/source/de/anomic/plasma/plasmaWordIndexCache.java index 268afc3f9..f3c7b3968 100644 --- a/source/de/anomic/plasma/plasmaWordIndexCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexCache.java @@ -579,6 +579,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { plasmaWordIndexEntry entry; while (entries.hasNext()) { entry = (plasmaWordIndexEntry) entries.next(); + System.out.println("ENTRY = " + entry.getUrlHash()); container.add(new plasmaWordIndexEntry[]{entry}, System.currentTimeMillis()); } // we have read all elements, now delete the entity diff --git a/source/de/anomic/tools/enumerateFiles.java b/source/de/anomic/tools/enumerateFiles.java index ef6474b65..45e8e23f3 100644 --- a/source/de/anomic/tools/enumerateFiles.java +++ b/source/de/anomic/tools/enumerateFiles.java @@ -68,7 +68,7 @@ public class enumerateFiles implements Enumeration { TreeSet t = new TreeSet(); String[] l = root.list(); // System.out.println("D " + l.toString()); - for (int i = 0; i < l.length; i++) t.add(new File(root, l[i])); + if (l != null) for (int i = 0; i < l.length; i++) t.add(new File(root, l[i])); this.hierarchy.add(t); // start with search by filling the buffer this.buffer = nextElement0(); diff --git a/source/yacy.java b/source/yacy.java index 84f343bc5..9062e8eec 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -626,6 +626,7 @@ public final class yacy { while (words.hasMoreElements()) try { wordfile = (File) words.nextElement(); wordhash = wordfile.getName().substring(0, 12); + System.out.println("NOW: " + wordhash); migration = wordIndexCache.migrateWords2Assortment(wordhash); if (migration == 0) log.logInfo("SKIPPED " + wordhash + ": " + ((wordfile.exists()) ? "too big" : "database corrupted; deleted"));