From d99ff745aafcb617d78cbdc647bacd51bd2500eb Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Wed, 18 Mar 2009 10:29:13 +0000
Subject: [PATCH] fix for
 http://forum.yacy-websuche.de/viewtopic.php?p=13378#p13378

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5726 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 source/de/anomic/crawler/Balancer.java        | 25 ++++++-------------
 .../kelondro/index/IntegerHandleIndex.java    |  4 ++-
 .../kelondro/index/ObjectIndexCache.java      |  4 ++-
 3 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/source/de/anomic/crawler/Balancer.java b/source/de/anomic/crawler/Balancer.java
index 142705137..800b4977f 100644
--- a/source/de/anomic/crawler/Balancer.java
+++ b/source/de/anomic/crawler/Balancer.java
@@ -58,7 +58,6 @@ public class Balancer {
     private final File               cacheStacksPath;
     private final String             stackname;
     private boolean                  top;             // to alternate between top and bottom of the file stack
-    private final boolean            fullram;
     private long                     minimumLocalDelta;
     private long                     minimumGlobalDelta;
     
@@ -71,13 +70,13 @@ public class Balancer {
         this.domainStacks   = new ConcurrentHashMap<String, LinkedList<String>>();
         this.urlRAMStack    = new ArrayList<String>();
         this.top            = true;
-        this.fullram        = fullram;
         this.minimumLocalDelta = minimumLocalDelta;
         this.minimumGlobalDelta = minimumGlobalDelta;
         
         // create a stack for newly entered entries
         if (!(cachePath.exists())) cachePath.mkdir(); // make the path
-        openFileIndex();
+        cacheStacksPath.mkdirs();
+        urlFileIndex = new EcoTable(new File(cacheStacksPath, stackname + indexSuffix), CrawlEntry.rowdef, (fullram) ? EcoTable.tailCacheUsageAuto : EcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
         if (urlFileStack.size() != urlFileIndex.size() || (urlFileIndex.size() < 10000 && urlFileIndex.size() > 0)) {
             // fix the file stack
             Log.logInfo("Balancer", "re-creating the " + stackname + " balancer stack, size = " + urlFileIndex.size() + ((urlFileStack.size() == urlFileIndex.size()) ? "" : " (the old stack size was wrong)" ));
@@ -130,24 +129,14 @@ public class Balancer {
     }
     
     public synchronized void clear() {
+        try {
+            urlFileIndex.clear();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
         urlFileStack.clear();
         domainStacks.clear();
         urlRAMStack.clear();
-        resetFileIndex();
-    }
-    
-    private void openFileIndex() {
-        cacheStacksPath.mkdirs();
-        urlFileIndex = new EcoTable(new File(cacheStacksPath, stackname + indexSuffix), CrawlEntry.rowdef, (fullram) ? EcoTable.tailCacheUsageAuto : EcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
-    }
-    
-    private void resetFileIndex() {
-        if (urlFileIndex != null) {
-            urlFileIndex.close();
-            urlFileIndex = null;
-            new File(cacheStacksPath, stackname + indexSuffix).delete();
-        }
-        openFileIndex();
     }
     
     public synchronized CrawlEntry get(final String urlhash) throws IOException {
diff --git a/source/de/anomic/kelondro/index/IntegerHandleIndex.java b/source/de/anomic/kelondro/index/IntegerHandleIndex.java
index 0414f8def..1e3c3a828 100644
--- a/source/de/anomic/kelondro/index/IntegerHandleIndex.java
+++ b/source/de/anomic/kelondro/index/IntegerHandleIndex.java
@@ -73,10 +73,12 @@ public class IntegerHandleIndex {
         InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
         byte[] a = new byte[keylength + 4];
         int c;
+        Row.Entry entry;
         while (true) {
             c = is.read(a);
             if (c <= 0) break;
-            this.index.addUnique(this.rowdef.newEntry(a));
+            entry = this.rowdef.newEntry(a); // may be null if a is not well-formed
+            if (entry != null) this.index.addUnique(entry);
         }
         is.close();
         assert this.index.size() == file.length() / (keylength + 4);
diff --git a/source/de/anomic/kelondro/index/ObjectIndexCache.java b/source/de/anomic/kelondro/index/ObjectIndexCache.java
index 130f0d10e..0f5686b38 100644
--- a/source/de/anomic/kelondro/index/ObjectIndexCache.java
+++ b/source/de/anomic/kelondro/index/ObjectIndexCache.java
@@ -105,6 +105,7 @@ public class ObjectIndexCache implements ObjectIndex {
    
 	public synchronized void put(final Row.Entry entry) {
         assert (entry != null);
+        if (entry == null) return;
         finishInitialization();
         // if the new entry is within the initialization part, just overwrite it
         assert index0.isSorted();
@@ -122,8 +123,9 @@ public class ObjectIndexCache implements ObjectIndex {
 		while (i.hasNext()) put(i.next());
 	}
 
-	public synchronized void addUnique(final Row.Entry entry) {    	
+	public synchronized void addUnique(final Row.Entry entry) {
     	assert (entry != null);
+    	if (entry == null) return;
         if (index1 == null) {
             // we are in the initialization phase
         	index0.addUnique(entry);