From c67d22116eec97a0a381f48e836d1564b3e7a0d4 Mon Sep 17 00:00:00 2001
From: "(no author)" <(no author)@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Mon, 8 Jan 2007 13:13:30 +0000
Subject: [PATCH] added exists-check based only on RAM index lookup: - faster
 double-check during crawling - less IO

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3179 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 lib/svnRevNr.jar                              | Bin 3173 -> 3174 bytes
 source/dbtest.java                            |   4 ++++
 source/de/anomic/index/indexCachedRI.java     |   7 +++++++
 source/de/anomic/index/indexCollectionRI.java |  10 +++++++++-
 source/de/anomic/index/indexRAMRI.java        |   4 ++++
 source/de/anomic/index/indexRI.java           |   1 +
 source/de/anomic/kelondro/kelondroCache.java  |   4 ++++
 .../kelondro/kelondroCollectionIndex.java     |   4 ++++
 .../kelondro/kelondroFlexSplitTable.java      |  10 ++++++++++
 source/de/anomic/kelondro/kelondroIndex.java  |   1 +
 source/de/anomic/kelondro/kelondroRowSet.java |   5 +++++
 .../anomic/kelondro/kelondroSplittedTree.java |   4 ++++
 source/de/anomic/kelondro/kelondroTree.java   |   4 ++++
 source/de/anomic/plasma/plasmaCrawlEURL.java  |   2 +-
 source/de/anomic/plasma/plasmaCrawlLURL.java  |   2 +-
 source/de/anomic/plasma/plasmaWordIndex.java  |   7 +++++++
 .../plasma/plasmaWordIndexFileCluster.java    |   4 ++++
 17 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/lib/svnRevNr.jar b/lib/svnRevNr.jar
index 173e7aeee46964ff844eb09c062a6e8310e55736..849022679d2efc0badd08d0f659470768187d337 100644
GIT binary patch
delta 322
zcmaDV@l1j@z?+$civa}qrmIin)!^ivu5K1mmVLKsqMdyGXWuiYeY|z`F7kToYMncC
zeshq)72^j_i_SO;73z5Sobw1}=weho`^Ben=bY{!Veg>elDS%ruYAAqJmNi5a@I%0
zWQq(oS5c7YoX^GFUfx07B_cMTCw-k_Qn|B6*!XkF++v~b?zJLar$03Xo&M9sB-Gs%
z++DG=VkZOGT^vAXaWDW~wDEB-E2GxtLbhNQMy<(5x%`=duAMB-ZGmv^<Ops{d646K
z6CbUPV`X4q26{^b>@x_lr1AR1kK#ZNGct*Q<bWn?O)g{;=7y<R(wIK^6StDQ7K^f3
dYFy1jai9bUOTrWY=_QT3CmZwVuyKNQ0RTllVEO<6

delta 324
zcmaDR@l=90z?+$civa{=c+@8HYH&*PsF|4w9(q(c(N4bJ_tZwNLk0p34{axu?+9#f
z5n1W6{nklEC9ZWv$0oCViF%i`FhBop%>MuVCw_WH-I?n0`2cT-UBu;2^S3B3o0qUr
zHLo_;SG}cGF3<k|zgz9mH+ELvIPmIIVu$6)1)IgHv}G)QZZ#D&-u<q174M59>9hKZ
zoC1Zi*ZgFHIE$TufrH_8g7U^Ey{wEkHy5!5voPMAe2mMV8R*!_65JLD*G`V)wv-1d
zHS0}$v^tKJfq@z5D-p2AAjFc!Ym+x}E6d+xQ8r^#?GRUHVPKfd%fKLvq9$$fXKp31
bn$)<OhvGmv5SBzyvum;mj}99LH%Jfw$NFj-

diff --git a/source/dbtest.java b/source/dbtest.java
index 5c2b690f3..7fa86ca52 100644
--- a/source/dbtest.java
+++ b/source/dbtest.java
@@ -492,6 +492,10 @@ final class dbTable implements kelondroIndex {
         return this.rowdef;
     }
     
+    public boolean has(byte[] key) throws IOException {
+        return (get(key) != null);
+    }
+    
     public kelondroRow.Entry get(byte[] key) throws IOException {
         try {
             String sqlQuery = new String
diff --git a/source/de/anomic/index/indexCachedRI.java b/source/de/anomic/index/indexCachedRI.java
index 21763a432..c9b002164 100644
--- a/source/de/anomic/index/indexCachedRI.java
+++ b/source/de/anomic/index/indexCachedRI.java
@@ -143,6 +143,13 @@ public class indexCachedRI implements indexRI {
         busyCacheFlush = false;
     }
     
+    public boolean hasContainer(String wordHash) {
+        if (riExtern.hasContainer(wordHash)) return true;
+        if (riIntern.hasContainer(wordHash)) return true;
+        if (backend.hasContainer(wordHash)) return true;
+        return false;
+    }
+    
     public indexContainer getContainer(String wordHash, Set urlselection, long maxTime) {
         // get from cache
         indexContainer container = riExtern.getContainer(wordHash, urlselection, maxTime);
diff --git a/source/de/anomic/index/indexCollectionRI.java b/source/de/anomic/index/indexCollectionRI.java
index af302501c..50052a169 100644
--- a/source/de/anomic/index/indexCollectionRI.java
+++ b/source/de/anomic/index/indexCollectionRI.java
@@ -118,7 +118,15 @@ public class indexCollectionRI implements indexRI {
         }
 
     }
-     
+
+    public synchronized boolean hasContainer(String wordHash) {
+        try {
+            return collectionIndex.has(wordHash.getBytes());
+        } catch (IOException e) {
+            return false;
+        }
+    }
+    
     public synchronized indexContainer getContainer(String wordHash, Set urlselection, long maxtime) {
         try {
             kelondroRowSet collection = collectionIndex.get(wordHash.getBytes());
diff --git a/source/de/anomic/index/indexRAMRI.java b/source/de/anomic/index/indexRAMRI.java
index 710719882..a44ea4763 100644
--- a/source/de/anomic/index/indexRAMRI.java
+++ b/source/de/anomic/index/indexRAMRI.java
@@ -332,6 +332,10 @@ public final class indexRAMRI implements indexRI {
         return (((long) intTime) * (long) 1000) + initTime;
     }
     
+    public synchronized boolean hasContainer(String wordHash) {
+        return cache.containsKey(wordHash);
+    }
+    
     public synchronized indexContainer getContainer(String wordHash, Set urlselection, long maxtime_dummy) {
 
         // retrieve container
diff --git a/source/de/anomic/index/indexRI.java b/source/de/anomic/index/indexRI.java
index 16e9097bc..68ad3ff80 100644
--- a/source/de/anomic/index/indexRI.java
+++ b/source/de/anomic/index/indexRI.java
@@ -40,6 +40,7 @@ public interface indexRI {
         
     public long getUpdateTime(String wordHash);
     public int indexSize(String wordHash);
+    public boolean hasContainer(String wordHash); // should only be used if in case that true is returned the getContainer is NOT called
     public indexContainer getContainer(String wordHash, Set urlselection, long maxtime);
     public indexContainer deleteContainer(String wordHash);
     
diff --git a/source/de/anomic/kelondro/kelondroCache.java b/source/de/anomic/kelondro/kelondroCache.java
index 1e7aeaf39..084ec3fb9 100644
--- a/source/de/anomic/kelondro/kelondroCache.java
+++ b/source/de/anomic/kelondro/kelondroCache.java
@@ -278,6 +278,10 @@ public class kelondroCache implements kelondroIndex {
         index.close();
     }
 
+    public boolean has(byte[] key) throws IOException {
+        return (get(key) != null);
+    }
+    
     public synchronized Entry get(byte[] key) throws IOException {
         // first look into the miss cache
         if (readMissCache != null) {
diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java
index 26400c6b6..b8e1f1072 100644
--- a/source/de/anomic/kelondro/kelondroCollectionIndex.java
+++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java
@@ -420,6 +420,10 @@ public class kelondroCollectionIndex {
         return (int) indexrow.getColLong(idx_col_chunkcount);
     }
     
+    public synchronized boolean has(byte[] key) throws IOException {
+        return index.has(key);
+    }
+    
     public synchronized kelondroRowSet get(byte[] key) throws IOException {
         // find an entry, if one exists
         kelondroRow.Entry indexrow = index.get(key);
diff --git a/source/de/anomic/kelondro/kelondroFlexSplitTable.java b/source/de/anomic/kelondro/kelondroFlexSplitTable.java
index 5291013f3..070ed45cb 100644
--- a/source/de/anomic/kelondro/kelondroFlexSplitTable.java
+++ b/source/de/anomic/kelondro/kelondroFlexSplitTable.java
@@ -169,6 +169,16 @@ public class kelondroFlexSplitTable implements kelondroIndex {
         return this.rowdef;
     }
     
+    public boolean has(byte[] key) throws IOException {
+        Iterator i = tables.values().iterator();
+        kelondroIndex table;
+        while (i.hasNext()) {
+            table = (kelondroIndex) i.next();
+            if (table.has(key)) return true;
+        }
+        return false;
+    }
+    
     public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
         Object[] keeper = keeperOf(key);
         if (keeper == null) return null;
diff --git a/source/de/anomic/kelondro/kelondroIndex.java b/source/de/anomic/kelondro/kelondroIndex.java
index 2a93665ad..dea140236 100644
--- a/source/de/anomic/kelondro/kelondroIndex.java
+++ b/source/de/anomic/kelondro/kelondroIndex.java
@@ -59,6 +59,7 @@ public interface kelondroIndex {
     public int size() throws IOException;
     public kelondroProfile profile();
     public kelondroRow row() throws IOException;
+    public boolean has(byte[] key) throws IOException; // use this only if there is no get in case that has returns true
     public kelondroRow.Entry get(byte[] key) throws IOException;
     public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException;
     public kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException;
diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java
index 1d826694b..61b781db4 100644
--- a/source/de/anomic/kelondro/kelondroRowSet.java
+++ b/source/de/anomic/kelondro/kelondroRowSet.java
@@ -24,6 +24,7 @@
 
 package de.anomic.kelondro;
 
+import java.io.IOException;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.Random;
@@ -63,6 +64,10 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
         this.profile = new kelondroProfile();
     }
     
+    public boolean has(byte[] key) throws IOException {
+        return (get(key) != null);
+    }
+    
     public kelondroRow.Entry get(byte[] key) {
         return get(key, 0, key.length);
     }
diff --git a/source/de/anomic/kelondro/kelondroSplittedTree.java b/source/de/anomic/kelondro/kelondroSplittedTree.java
index 36c8f8cdc..22d681b36 100644
--- a/source/de/anomic/kelondro/kelondroSplittedTree.java
+++ b/source/de/anomic/kelondro/kelondroSplittedTree.java
@@ -105,6 +105,10 @@ public class kelondroSplittedTree implements kelondroIndex {
         return (int) order.partition(key, ff);
     }
     
+    public boolean has(byte[] key) throws IOException {
+        throw new UnsupportedOperationException("has should not be used with kelondroSplittedTree.");
+    }
+    
     public kelondroRow.Entry get(byte[] key) throws IOException {
         return ktfs[partition(key)].get(key);
     }
diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java
index 15d97e1c4..3d859c599 100644
--- a/source/de/anomic/kelondro/kelondroTree.java
+++ b/source/de/anomic/kelondro/kelondroTree.java
@@ -168,6 +168,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
         else n.commit(CP_HIGH);
     }
 
+    public boolean has(byte[] key) throws IOException {
+        throw new UnsupportedOperationException("has should not be used with kelondroTree.");
+    }
+    
     // Returns the value to which this map maps the specified key.
     public kelondroRow.Entry get(byte[] key) throws IOException {
         kelondroRow.Entry result;
diff --git a/source/de/anomic/plasma/plasmaCrawlEURL.java b/source/de/anomic/plasma/plasmaCrawlEURL.java
index f5bdb38a0..ad6416463 100644
--- a/source/de/anomic/plasma/plasmaCrawlEURL.java
+++ b/source/de/anomic/plasma/plasmaCrawlEURL.java
@@ -250,7 +250,7 @@ public class plasmaCrawlEURL {
 
     public boolean exists(String urlHash) {
         try {
-            return (urlIndexFile.get(urlHash.getBytes()) != null);
+            return urlIndexFile.has(urlHash.getBytes());
         } catch (IOException e) {
             return false;
         }
diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java
index b102a835c..9a6d9289f 100644
--- a/source/de/anomic/plasma/plasmaCrawlLURL.java
+++ b/source/de/anomic/plasma/plasmaCrawlLURL.java
@@ -360,7 +360,7 @@ public final class plasmaCrawlLURL {
 
     public synchronized boolean exists(String urlHash) {
         try {
-            return (urlIndexFile.get(urlHash.getBytes()) != null);
+            return urlIndexFile.has(urlHash.getBytes());
         } catch (IOException e) {
             return false;
         }
diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java
index 94a0437c5..bfe14c2ce 100644
--- a/source/de/anomic/plasma/plasmaWordIndex.java
+++ b/source/de/anomic/plasma/plasmaWordIndex.java
@@ -293,6 +293,13 @@ public final class plasmaWordIndex implements indexRI {
         return wordCount;
     }
 
+    public boolean hasContainer(String wordHash) {
+        if (dhtOutCache.hasContainer(wordHash)) return true;
+        if (dhtInCache.hasContainer(wordHash)) return true;
+        if (collections.hasContainer(wordHash)) return true;
+        return false;
+    }
+    
     public indexContainer getContainer(String wordHash, Set urlselection, long maxTime) {
 
         // get from cache
diff --git a/source/de/anomic/plasma/plasmaWordIndexFileCluster.java b/source/de/anomic/plasma/plasmaWordIndexFileCluster.java
index 7d229fe17..75f0a376e 100644
--- a/source/de/anomic/plasma/plasmaWordIndexFileCluster.java
+++ b/source/de/anomic/plasma/plasmaWordIndexFileCluster.java
@@ -224,6 +224,10 @@ public class plasmaWordIndexFileCluster implements indexRI {
         return plasmaWordIndexFile.wordHash2path(databaseRoot, wordHash).exists();
     }
     
+    public synchronized boolean hasContainer(String wordHash) {
+        return getContainer(wordHash, new TreeSet(), -1) != null;
+    }
+    
     public synchronized indexContainer getContainer(String wordHash, Set urlselection, long maxTime) {
         long start = System.currentTimeMillis();
         if ((maxTime < 0) || (maxTime > 60000)) maxTime=60000; // maximum is one minute