diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 0f37fe494..2aa7174c2 100755 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -113,7 +113,7 @@ public class ZURL implements Iterable { if (hash == null) return false; //System.out.println("*** DEBUG ZURL " + this.urlIndex.filename() + " remove " + hash); try { - urlIndex.remove(hash); + urlIndex.delete(hash); return true; } catch (final IOException e) { return false; diff --git a/source/de/anomic/search/MetadataRepository.java b/source/de/anomic/search/MetadataRepository.java index 2e7954d12..df1cbfcff 100644 --- a/source/de/anomic/search/MetadataRepository.java +++ b/source/de/anomic/search/MetadataRepository.java @@ -680,7 +680,7 @@ public final class MetadataRepository implements Iterable { // then delete the urls using this list int cnt = 0; for (String h: l) { - if (urlIndexFile.remove(h.getBytes()) != null) cnt++; + if (urlIndexFile.delete(h.getBytes())) cnt++; } // finally remove the line with statistics diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 1aa026a51..d685a2c88 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -110,7 +110,7 @@ public class yacyNewsDB { } public void remove(final String id) throws IOException { - news.remove(id.getBytes()); + news.delete(id.getBytes()); } public synchronized yacyNewsRecord put(final yacyNewsRecord record) throws IOException, RowSpaceExceededException { diff --git a/source/net/yacy/dbtest.java b/source/net/yacy/dbtest.java index d22e41835..2f503c50f 100644 --- a/source/net/yacy/dbtest.java +++ b/source/net/yacy/dbtest.java @@ -151,7 +151,7 @@ public class dbtest { System.out.println("remove: " + NaturalOrder.arrayList(entry.getKey(), 0, entry.getKey().length)); try { getTable_test().remove(entry.getKey()); - if (getTable_reference() != null) getTable_reference().remove(entry.getKey()); + if (getTable_reference() != null) getTable_reference().delete(entry.getKey()); } catch (final IOException e) { System.err.println(e); Log.logException(e); @@ -377,7 +377,7 @@ public class dbtest { start = System.currentTimeMillis(); for (int i = 0; i < count; i++) { key = randomHash(random); - table_test.remove(key); + table_test.delete(key); } remove = System.currentTimeMillis() - start; diff --git a/source/net/yacy/kelondro/index/BufferedObjectIndex.java b/source/net/yacy/kelondro/index/BufferedObjectIndex.java index 5833aad6d..0f12c1278 100644 --- a/source/net/yacy/kelondro/index/BufferedObjectIndex.java +++ b/source/net/yacy/kelondro/index/BufferedObjectIndex.java @@ -157,6 +157,14 @@ public class BufferedObjectIndex implements ObjectIndex, Iterable { } } + public boolean delete(byte[] key) throws IOException { + synchronized (this.backend) { + boolean b = this.buffer.delete(key); + if (b) return true; + return this.backend.delete(key); + } + } + public ArrayList removeDoubles() throws IOException, RowSpaceExceededException { synchronized (this.backend) { flushBuffer(); diff --git a/source/net/yacy/kelondro/index/Cache.java b/source/net/yacy/kelondro/index/Cache.java index cb179f901..3d2aabe76 100644 --- a/source/net/yacy/kelondro/index/Cache.java +++ b/source/net/yacy/kelondro/index/Cache.java @@ -298,7 +298,7 @@ public final class Cache implements ObjectIndex, Iterable { // remove entry from miss- and hit-cache if (readMissCache != null) { - if (readMissCache.remove(key) != null) { + if (readMissCache.delete(key)) { this.hasnotHit++; } } @@ -329,7 +329,7 @@ public final class Cache implements ObjectIndex, Iterable { // remove entry from miss- and hit-cache if (readMissCache != null) { - if (readMissCache.remove(key) != null) { + if (readMissCache.delete(key)) { this.hasnotHit++; // the entry does not exist before try { @@ -378,7 +378,7 @@ public final class Cache implements ObjectIndex, Iterable { // remove entry from miss- and hit-cache if (readMissCache != null) { - this.readMissCache.remove(key); + this.readMissCache.delete(key); this.hasnotDelete++; // the entry does not exist before } @@ -413,7 +413,7 @@ public final class Cache implements ObjectIndex, Iterable { // remove entry from miss- and hit-cache if (readMissCache != null) { - this.readMissCache.remove(key); + this.readMissCache.delete(key); this.hasnotDelete++; } @@ -453,6 +453,37 @@ public final class Cache implements ObjectIndex, Iterable { // todo: remove reported entries from the cache!!! } + public final synchronized boolean delete(final byte[] key) throws IOException { + checkMissSpace(); + + // add entry to miss-cache + if (checkMissSpace()) try { + // set the miss cache; if there was already an entry we know that the return value must be null + final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key)); + if (dummy == null) { + this.hasnotUnique++; + } else { + this.hasnotHit++; + this.hasnotDouble++; + } + } catch (RowSpaceExceededException e) { + clearCache(); + } + + // remove entry from hit-cache + if (readHitCache != null) { + final Row.Entry entry = readHitCache.remove(key); + if (entry == null) { + this.readMiss++; + } else { + this.readHit++; + this.cacheDelete++; + } + } + + return index.delete(key); + } + public final synchronized Row.Entry remove(final byte[] key) throws IOException { checkMissSpace(); @@ -498,8 +529,7 @@ public final class Cache implements ObjectIndex, Iterable { clearCache(); } if (readHitCache != null) { - final Row.Entry dummy = readHitCache.remove(key); - if (dummy != null) this.cacheDelete++; + if (readHitCache.delete(key)) this.cacheDelete++; } return entry; } diff --git a/source/net/yacy/kelondro/index/ObjectIndex.java b/source/net/yacy/kelondro/index/ObjectIndex.java index 7daa42431..6c96fafcd 100644 --- a/source/net/yacy/kelondro/index/ObjectIndex.java +++ b/source/net/yacy/kelondro/index/ObjectIndex.java @@ -52,6 +52,7 @@ public interface ObjectIndex extends Iterable { public void put(Row.Entry row) throws IOException, RowSpaceExceededException; public void addUnique(Row.Entry row) throws RowSpaceExceededException, IOException; // no double-check public ArrayList removeDoubles() throws IOException, RowSpaceExceededException; // removes all elements that are double (to be used after all addUnique) + public boolean delete(byte[] key) throws IOException; public Row.Entry remove(byte[] key) throws IOException; public Row.Entry removeOne() throws IOException; public CloneableIterator keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key diff --git a/source/net/yacy/kelondro/index/ObjectIndexCache.java b/source/net/yacy/kelondro/index/ObjectIndexCache.java index 0be430c28..98735ef4b 100644 --- a/source/net/yacy/kelondro/index/ObjectIndexCache.java +++ b/source/net/yacy/kelondro/index/ObjectIndexCache.java @@ -199,6 +199,20 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable return d0; } + public final synchronized boolean delete(final byte[] key) { + finishInitialization(); + // if the new entry is within the initialization part, just delete it + boolean b = index0.delete(key); + if (b) { + assert index0.get(key) == null; // check if remove worked + return true; + } + // else remove it from the index1 + b = index1.delete(key); + assert index1.get(key) == null : "removed " + ((b) ? " true" : " false") + ", and index entry still exists"; // check if remove worked + return b; + } + public final synchronized Row.Entry remove(final byte[] key) { finishInitialization(); // if the new entry is within the initialization part, just delete it diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index 2af40a400..8f7661cba 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -166,33 +166,37 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable ii = d.iterator(); String s; System.out.print("INPUT-ITERATOR: "); @@ -594,11 +598,11 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable, Clon accessArray(i).put(row); } + public final boolean delete(final byte[] key) { + final int i = indexFor(key); + if (i < 0) return false; + return accessArray(i).delete(key); + } + public final Entry remove(final byte[] key) { final int i = indexFor(key); if (i < 0) return null; diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainer.java b/source/net/yacy/kelondro/rwi/ReferenceContainer.java index 456600053..3dca96c56 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainer.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainer.java @@ -184,7 +184,7 @@ public class ReferenceContainer extends RowSet public int removeEntries(final HandleSet urlHashes) { int count = 0; final Iterator i = urlHashes.iterator(); - while (i.hasNext()) count += (remove(i.next()) == null) ? 0 : 1; + while (i.hasNext()) count += (delete(i.next())) ? 1 : 0; return count; } @@ -460,7 +460,7 @@ public class ReferenceContainer extends RowSet if ((ie0 != null) && (ie1 != null)) { assert (ie0.metadataHash().length == keylength) : "ie0.urlHash() = " + new String(ie0.metadataHash()); assert (ie1.metadataHash().length == keylength) : "ie1.urlHash() = " + new String(ie1.metadataHash()); - if (iterate_pivot) se.remove(); pivot.remove(ie0.metadataHash()); + if (iterate_pivot) se.remove(); pivot.delete(ie0.metadataHash()); } } return pivot; diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index e2de222f5..99bb1b968 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -334,7 +334,7 @@ public final class ReferenceContainerCache exte ByteArray tha = new ByteArray(termHash); synchronized (cache) { final ReferenceContainer c = cache.get(tha); - if ((c != null) && (c.remove(urlHashBytes) != null)) { + if (c != null && c.delete(urlHashBytes)) { // removal successful if (c.isEmpty()) { delete(termHash); diff --git a/source/net/yacy/kelondro/table/SQLTable.java b/source/net/yacy/kelondro/table/SQLTable.java index 6474bb786..77fb88aa2 100644 --- a/source/net/yacy/kelondro/table/SQLTable.java +++ b/source/net/yacy/kelondro/table/SQLTable.java @@ -273,6 +273,10 @@ public class SQLTable implements ObjectIndex, Iterable { } } + public boolean delete(final byte[] key) throws IOException { + return remove(key) != null; + } + public Row.Entry removeOne() { return null; } diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index aee24d286..4fdbc4967 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -387,6 +387,12 @@ public class SplitTable implements ObjectIndex, Iterable { return report; } + public boolean delete(final byte[] key) throws IOException { + final ObjectIndex table = keeperOf(key); + if (table == null) return false; + return table.delete(key); + } + public Row.Entry remove(final byte[] key) throws IOException { final ObjectIndex table = keeperOf(key); if (table == null) return null; diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 56ce8d58d..3b91cc789 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -592,6 +592,10 @@ public class Table implements ObjectIndex, Iterable { } } + public boolean delete(final byte[] key) throws IOException { + return remove(key) != null; + } + public synchronized Entry remove(final byte[] key) throws IOException { assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size();