boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed outOfMemory bug caused by loops in kelondroTree during enumeration

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1079 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent f8f9d509d5
commit 9913049009
5 changed files with 39 additions and 12 deletions
Show Stats Download Patch File Download Diff File

14
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -75,6 +75,7 @@ import java.util.NoSuchElementException;
import java.util.StringTokenizer;
import java.util.Map;
import java.util.Iterator;
import java.util.logging.Logger;
public class kelondroRecords {
@ -145,6 +146,8 @@ public class kelondroRecords {
protected long XcacheStartup; // startup time; for cache aging
protected kelondroMScoreCluster cacheScore; // controls cache aging
// optional logger
protected Logger theLogger = null;
public kelondroRecords(File file, long buffersize /* bytes */,
short ohbytec, short ohhandlec,
@ -236,6 +239,17 @@ public class kelondroRecords {
// thats it!
}
public void setLogger(Logger newLogger) {
this.theLogger = newLogger;
}
public void logWarning(String message) {
if (this.theLogger == null)
System.err.println("KELONDRO WARNING for file " + this.filename + ": " + message);
else
this.theLogger.warning("KELONDRO WARNING for file " + this.filename + ": " + message);
}
public void clear() throws IOException {
// Removes all mappings from this map
//throw new UnsupportedOperationException("clear not supported");

33
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -208,17 +208,17 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
parentnode = thenode;
thenode = getNode(thisHandle, thenode, (child == -1) ? leftchild : rightchild);
} catch (IllegalArgumentException e) {
System.out.println("WARNING: kelondroTree.Search.searchproc: fixed a broken handle");
logWarning("kelondroTree.Search.process: fixed a broken handle");
found = false;
return;
}
if (thenode == null) {
throw new kelondroException(filename, "kelondroTree.searchproc: thenode==null");
throw new kelondroException(filename, "kelondroTree.Search.process: thenode==null");
}
try {
otherkey = new String(thenode.getKey());
} catch (NullPointerException e) {
throw new kelondroException(filename, "kelondroTree.searchproc: nullpointer" + e.getMessage());
throw new kelondroException(filename, "kelondroTree.Search.process: nullpointer" + e.getMessage());
}
if (visitedNodeKeys.containsKey(otherkey)) {
// we have loops in the database.
@ -228,6 +228,8 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
thenode.setOHHandle(parent, null);
thenode.setOHHandle(leftchild, null);
thenode.setOHHandle(rightchild, null);
thenode.commit(CP_NONE);
/*
Iterator fix = visitedNodeKeys.entrySet().iterator();
Map.Entry entry;
while (fix.hasNext()) {
@ -239,9 +241,10 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
thenode.setOHHandle(leftchild, null);
thenode.setOHHandle(rightchild, null);
}
thenode.commit(CP_NONE);
//printCache();
throw new kelondroException(filename, "database contains loops; the loop-nodes have been auto-fixed");
*/
logWarning("kelondroTree.Search.process: database contains loops; the loop-nodes have been auto-fixed");
found = false;
return;
}
//System.out.println("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug
c = compare(key, thenode.getKey());
@ -804,7 +807,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
int c = compare(firstKey, nextNode.getKey());
if ((c > 0) && (up)) {
// firstKey > nextNode.getKey()
System.out.println("CORRECTING ITERATOR: firstKey=" + new String(firstKey) + ", nextNode=" + new String(nextNode.getKey()));
logWarning("CORRECTING ITERATOR: firstKey=" + new String(firstKey) + ", nextNode=" + new String(nextNode.getKey()));
nextNode = (ii.hasNext()) ? (Node) ii.next() : null;
}
if ((c < 0) && (!(up))) {
@ -871,9 +874,9 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
// go to next node
searchHandle = searchNode.getOHHandle(ct);
if (searchHandle == null) throw new kelondroException(filename, "start node does not exist (handle null)");
if (searchHandle == null) throw new kelondroException(filename, "nodeIterator.init: start node does not exist (handle null)");
searchNode = getNode(searchHandle, searchNode, ct);
if (searchNode == null) throw new kelondroException(filename, "start node does not exist (node null)");
if (searchNode == null) throw new kelondroException(filename, "nodeIterator.init: start node does not exist (node null)");
}
// now every parent node to the start node is on the stack
}
@ -894,8 +897,8 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
} catch (IOException e) {
throw new kelondroException(filename, "io-error while rot");
}
if (nextNode == null) throw new kelondroException(filename, "no more entries available");
if ((count > size()) && (!(rot))) throw new kelondroException(filename, "internal loopback; database corrupted");
if (nextNode == null) throw new kelondroException(filename, "nodeIterator.next: no more entries available");
if ((count > size()) && (!(rot))) throw new kelondroException(filename, "nodeIterator.next: internal loopback; database corrupted");
Object ret = nextNode;
// middle-case
@ -906,10 +909,18 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
if (childHandle != null) {
//System.out.println("go to other leg, stack size=" + nodeStack.size());
// we have walked one leg of the tree; now go to the other one: step down to next child
HashSet visitedNodeHandles = new HashSet(); // to detect loops
nodeStack.addLast(new Object[]{nextNode, new Integer(childtype)});
nextNode = getNode(childHandle, nextNode, childtype);
childtype = (up) ? leftchild : rightchild;
while ((childHandle = nextNode.getOHHandle(childtype)) != null) {
if (visitedNodeHandles.contains(childHandle)) {
// try to repair the nextNode
nextNode.setOHHandle(childtype, null);
logWarning("nodeIterator.next: internal loopback; fixed loop and try to go on");
break;
}
visitedNodeHandles.add(childHandle);
try {
nodeStack.addLast(new Object[]{nextNode, new Integer(childtype)});
nextNode = getNode(childHandle, nextNode, childtype);

1
source/de/anomic/plasma/plasmaWordIndexCache.java
Unescape View File

@ -579,6 +579,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
plasmaWordIndexEntry entry;
while (entries.hasNext()) {
entry = (plasmaWordIndexEntry) entries.next();
System.out.println("ENTRY = " + entry.getUrlHash());
container.add(new plasmaWordIndexEntry[]{entry}, System.currentTimeMillis());
}
// we have read all elements, now delete the entity

2
source/de/anomic/tools/enumerateFiles.java
Unescape View File

@ -68,7 +68,7 @@ public class enumerateFiles implements Enumeration {
TreeSet t = new TreeSet();
String[] l = root.list();
// System.out.println("D " + l.toString());
for (int i = 0; i < l.length; i++) t.add(new File(root, l[i]));
if (l != null) for (int i = 0; i < l.length; i++) t.add(new File(root, l[i]));
this.hierarchy.add(t);
// start with search by filling the buffer
this.buffer = nextElement0();

1
source/yacy.java
Unescape View File

@ -626,6 +626,7 @@ public final class yacy {
while (words.hasMoreElements()) try {
wordfile = (File) words.nextElement();
wordhash = wordfile.getName().substring(0, 12);
System.out.println("NOW: " + wordhash);
migration = wordIndexCache.migrateWords2Assortment(wordhash);
if (migration == 0)
log.logInfo("SKIPPED " + wordhash + ": " + ((wordfile.exists()) ? "too big" : "database corrupted; deleted"));

Write Preview
Loading…
Cancel
Save