enhanced remove() operation: in many cases it is not necessary to return the removed object to the called.

for such cases the delete() operation was introduced which is sometimes much cheaper in operation since it does not need to create objects to hold the removed content and it does not need to read those objects.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6824 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/crawler/ZURL.java

@@ -113,7 +113,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
         if (hash == null) return false;
         //System.out.println("*** DEBUG ZURL " + this.urlIndex.filename() + " remove " + hash);
         try {
-            urlIndex.remove(hash);
+            urlIndex.delete(hash);
             return true;
         } catch (final IOException e) {
             return false;

source/de/anomic/search/MetadataRepository.java

@@ -680,7 +680,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
         // then delete the urls using this list
         int cnt = 0;
         for (String h: l) {
-            if (urlIndexFile.remove(h.getBytes()) != null) cnt++;
+            if (urlIndexFile.delete(h.getBytes())) cnt++;
         }
         
         // finally remove the line with statistics

source/de/anomic/yacy/yacyNewsDB.java

@@ -110,7 +110,7 @@ public class yacyNewsDB {
     }
 
     public void remove(final String id) throws IOException {
-        news.remove(id.getBytes());
+        news.delete(id.getBytes());
     }
 
     public synchronized yacyNewsRecord put(final yacyNewsRecord record) throws IOException, RowSpaceExceededException {

source/net/yacy/dbtest.java

@@ -151,7 +151,7 @@ public class dbtest {
             System.out.println("remove: " + NaturalOrder.arrayList(entry.getKey(), 0, entry.getKey().length));
             try {
                 getTable_test().remove(entry.getKey());
-                if (getTable_reference() != null) getTable_reference().remove(entry.getKey());
+                if (getTable_reference() != null) getTable_reference().delete(entry.getKey());
             } catch (final IOException e) {
                 System.err.println(e);
                 Log.logException(e);
@@ -377,7 +377,7 @@ public class dbtest {
                     start = System.currentTimeMillis();
                     for (int i = 0; i < count; i++) {
                         key = randomHash(random);
-                        table_test.remove(key);
+                        table_test.delete(key);
                     }
                     remove = System.currentTimeMillis() - start;
                     

source/net/yacy/kelondro/index/BufferedObjectIndex.java

@@ -157,6 +157,14 @@ public class BufferedObjectIndex implements ObjectIndex, Iterable<Row.Entry> {
         }
     }
 
+    public boolean delete(byte[] key) throws IOException {
+        synchronized (this.backend) {
+            boolean b = this.buffer.delete(key);
+            if (b) return true;
+            return this.backend.delete(key);
+        }
+    }
+
     public ArrayList<RowCollection> removeDoubles() throws IOException, RowSpaceExceededException {
         synchronized (this.backend) {
             flushBuffer();

source/net/yacy/kelondro/index/Cache.java

@@ -298,7 +298,7 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
         
         // remove entry from miss- and hit-cache
         if (readMissCache != null) {
-            if (readMissCache.remove(key) != null) {
+            if (readMissCache.delete(key)) {
                 this.hasnotHit++;
             }
         }
@@ -329,7 +329,7 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
         
         // remove entry from miss- and hit-cache
         if (readMissCache != null) {
-            if (readMissCache.remove(key) != null) {
+            if (readMissCache.delete(key)) {
                 this.hasnotHit++;
                 // the entry does not exist before
                 try {
@@ -378,7 +378,7 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
         
         // remove entry from miss- and hit-cache
         if (readMissCache != null) {
-            this.readMissCache.remove(key);
+            this.readMissCache.delete(key);
             this.hasnotDelete++;
             // the entry does not exist before
         }
@@ -413,7 +413,7 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
         
         // remove entry from miss- and hit-cache
         if (readMissCache != null) {
-            this.readMissCache.remove(key);
+            this.readMissCache.delete(key);
             this.hasnotDelete++;
         }
 
@@ -453,6 +453,37 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
         // todo: remove reported entries from the cache!!!
     }
     
+    public final synchronized boolean delete(final byte[] key) throws IOException {
+        checkMissSpace();
+        
+        // add entry to miss-cache
+        if (checkMissSpace()) try {
+            // set the miss cache; if there was already an entry we know that the return value must be null
+            final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key));
+            if (dummy == null) {
+                this.hasnotUnique++;
+            } else {
+                this.hasnotHit++;
+                this.hasnotDouble++;
+            }
+        } catch (RowSpaceExceededException e) {
+            clearCache();
+        }
+        
+        // remove entry from hit-cache
+        if (readHitCache != null) {
+            final Row.Entry entry = readHitCache.remove(key);
+            if (entry == null) {
+                this.readMiss++;
+            } else {
+                this.readHit++;
+                this.cacheDelete++;
+            }
+        }
+        
+        return index.delete(key);
+    }
+
     public final synchronized Row.Entry remove(final byte[] key) throws IOException {
         checkMissSpace();
         
@@ -498,8 +529,7 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
             clearCache();
         }
         if (readHitCache != null) {
-            final Row.Entry dummy = readHitCache.remove(key);
-            if (dummy != null) this.cacheDelete++;
+            if (readHitCache.delete(key)) this.cacheDelete++;
         }
         return entry;
     }

source/net/yacy/kelondro/index/ObjectIndex.java

@@ -52,6 +52,7 @@ public interface ObjectIndex extends Iterable<Row.Entry> {
     public void put(Row.Entry row) throws IOException, RowSpaceExceededException;
     public void addUnique(Row.Entry row) throws RowSpaceExceededException, IOException; // no double-check
     public ArrayList<RowCollection> removeDoubles() throws IOException, RowSpaceExceededException; // removes all elements that are double (to be used after all addUnique)
+    public boolean delete(byte[] key) throws IOException;
     public Row.Entry remove(byte[] key) throws IOException;
     public Row.Entry removeOne() throws IOException;
     public CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key

source/net/yacy/kelondro/index/ObjectIndexCache.java

@@ -199,6 +199,20 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
         return d0;
 	}
 	
+    public final synchronized boolean delete(final byte[] key) {
+        finishInitialization();
+        // if the new entry is within the initialization part, just delete it
+        boolean b = index0.delete(key);
+        if (b) {
+            assert index0.get(key) == null; // check if remove worked
+            return true;
+        }
+        // else remove it from the index1
+        b = index1.delete(key);
+        assert index1.get(key) == null : "removed " + ((b) ? " true" : " false") + ", and index entry still exists"; // check if remove worked
+        return b;
+    }
+
     public final synchronized Row.Entry remove(final byte[] key) {
         finishInitialization();
         // if the new entry is within the initialization part, just delete it

source/net/yacy/kelondro/index/RowSet.java

@@ -166,33 +166,37 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
         }
     }
 
-    private final Row.Entry remove(final byte[] a, final int start, final int length) {
-        final int index = find(a, start, length);
-        if (index < 0) {
-            return null;
-        }
-        final Row.Entry entry = super.get(index, true);
-        super.removeRow(index, true); // keep order of collection!
-        //int findagainindex = 0;
-        //assert (findagainindex = find(a, start, length)) < 0 : "remove: chunk found again at index position (after  remove) " + findagainindex + ", index(before) = " + index + ", inset=" + NaturalOrder.arrayList(super.chunkcache, super.rowdef.objectsize * findagainindex, length) + ", searchkey=" + NaturalOrder.arrayList(a, start, length); // check if the remove worked
-        return entry;
-    }
-
     /**
      * remove a byte[] from the set.
      * if the entry was found, return the entry, but delete the entry from the set
      * if the entry was not found, return null.
      */
+    public final synchronized boolean delete(final byte[] a) {
+        boolean exists = false;
+        int index;
+        while (true) {
+            index = find(a, 0, a.length);
+            if (index < 0) {
+                return exists;
+            } else {
+                exists = true;
+                super.removeRow(index, true); // keep order of collection!
+            }
+        }
+    }
+
     public final synchronized Row.Entry remove(final byte[] a) {
         Row.Entry entry = null;
-        Row.Entry tmp;
-        do {
-            tmp = remove(a, 0, a.length);
-            if (tmp != null) {
-                entry = tmp;
+        int index;
+        while (true) {
+            index = find(a, 0, a.length);
+            if (index < 0) {
+                return entry;
+            } else {
+                entry = super.get(index, true);
+                super.removeRow(index, true); // keep order of collection!
             }
-        } while (tmp != null);
-        return entry;
+        }
     }
 
     private final int find(final byte[] a, final int astart, final int alength) {
@@ -492,7 +496,7 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
                 e.printStackTrace();
             }
         d.sort();
-        d.remove("fuenf".getBytes());
+        d.delete("fuenf".getBytes());
         final Iterator<Row.Entry> ii = d.iterator();
         String s;
         System.out.print("INPUT-ITERATOR: ");
@@ -594,11 +598,11 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
                 e.printStackTrace();
             }
             if (i % 1000 == 0) {
-                for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
+                for (int j = 0; j < delkeys.length; j++) c.delete(delkeys[j]);
                 c.sort();
             }
         }
-        for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
+        for (int j = 0; j < delkeys.length; j++) c.delete(delkeys[j]);
         c.sort();
         random = new Random(0);
         for (int i = 0; i < testsize; i++) {

source/net/yacy/kelondro/index/RowSetArray.java

@@ -167,6 +167,12 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
         accessArray(i).put(row);
     }
 
+    public final boolean delete(final byte[] key) {
+        final int i = indexFor(key);
+        if (i < 0) return false;
+        return accessArray(i).delete(key);
+    }
+
     public final Entry remove(final byte[] key) {
         final int i = indexFor(key);
         if (i < 0) return null;

source/net/yacy/kelondro/rwi/ReferenceContainer.java

@@ -184,7 +184,7 @@ public class ReferenceContainer<ReferenceType extends Reference> extends RowSet
     public int removeEntries(final HandleSet urlHashes) {
         int count = 0;
         final Iterator<byte[]> i = urlHashes.iterator();
-        while (i.hasNext()) count += (remove(i.next()) == null) ? 0 : 1;
+        while (i.hasNext()) count += (delete(i.next())) ? 1 : 0;
         return count;
     }
 
@@ -460,7 +460,7 @@ public class ReferenceContainer<ReferenceType extends Reference> extends RowSet
                 if ((ie0 != null) && (ie1 != null)) {
                     assert (ie0.metadataHash().length == keylength) : "ie0.urlHash() = " + new String(ie0.metadataHash());
                     assert (ie1.metadataHash().length == keylength) : "ie1.urlHash() = " + new String(ie1.metadataHash());
-                    if (iterate_pivot) se.remove(); pivot.remove(ie0.metadataHash());
+                    if (iterate_pivot) se.remove(); pivot.delete(ie0.metadataHash());
                 }
             }
         return pivot;

source/net/yacy/kelondro/rwi/ReferenceContainerCache.java

@@ -334,7 +334,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
         ByteArray tha = new ByteArray(termHash);
         synchronized (cache) {
 	        final ReferenceContainer<ReferenceType> c = cache.get(tha);
-	        if ((c != null) && (c.remove(urlHashBytes) != null)) {
+	        if (c != null && c.delete(urlHashBytes)) {
 	            // removal successful
 	            if (c.isEmpty()) {
 	                delete(termHash);

source/net/yacy/kelondro/table/SQLTable.java

@@ -273,6 +273,10 @@ public class SQLTable implements ObjectIndex, Iterable<Row.Entry> {
         }
     }
     
+    public boolean delete(final byte[] key) throws IOException {
+        return remove(key) != null;
+    }
+    
     public Row.Entry removeOne() {
         return null;
     }

source/net/yacy/kelondro/table/SplitTable.java

@@ -387,6 +387,12 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
         return report;
     }
     
+    public boolean delete(final byte[] key) throws IOException {
+        final ObjectIndex table = keeperOf(key);
+        if (table == null) return false;
+        return table.delete(key);
+    }
+    
     public Row.Entry remove(final byte[] key) throws IOException {
         final ObjectIndex table = keeperOf(key);
         if (table == null) return null;

source/net/yacy/kelondro/table/Table.java

@@ -592,6 +592,10 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
         }
     }
     
+    public boolean delete(final byte[] key) throws IOException {
+        return remove(key) != null;
+    }
+    
     public synchronized Entry remove(final byte[] key) throws IOException {
         assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
         assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size();