diff --git a/source/de/anomic/yacy/graphics/NetworkGraph.java b/source/de/anomic/yacy/graphics/NetworkGraph.java index 0cde05e51..f8ccb7233 100644 --- a/source/de/anomic/yacy/graphics/NetworkGraph.java +++ b/source/de/anomic/yacy/graphics/NetworkGraph.java @@ -36,6 +36,7 @@ import java.util.Date; import java.util.Iterator; import net.yacy.cora.document.Hit; +import net.yacy.kelondro.logging.Log; import net.yacy.visualization.PrintTool; import net.yacy.visualization.RasterPlotter; @@ -196,10 +197,13 @@ public class NetworkGraph { Iterator e = seedDB.seedsConnected(true, false, null, (float) 0.0); while (e.hasNext() && count < maxCount) { seed = e.next(); - if (seed != null) { - drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, coronaangle); - count++; + if (seed == null) { + Log.logWarning("NetworkGraph", "connected seed == null"); + continue; } + //Log.logInfo("NetworkGraph", "drawing peer " + seed.getName()); + drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, coronaangle); + count++; } totalCount += count; @@ -208,12 +212,16 @@ public class NetworkGraph { e = seedDB.seedsSortedDisconnected(false, yacySeed.LASTSEEN); while (e.hasNext() && count < maxCount) { seed = e.next(); - if (seed != null) { - lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); - if (lastseen > passiveLimit) break; // we have enough, this list is sorted so we don't miss anything - drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, coronaangle); - count++; + if (seed == null) { + Log.logWarning("NetworkGraph", "disconnected seed == null"); + continue; } + lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); + if (lastseen > passiveLimit) { + break; // we have enough, this list is sorted so we don't miss anything + } + drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, coronaangle); + count++; } totalCount += count; @@ -222,12 +230,16 @@ public class NetworkGraph { e = seedDB.seedsSortedPotential(false, yacySeed.LASTSEEN); while (e.hasNext() && count < maxCount) { seed = e.next(); - if (seed != null) { - lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); - if (lastseen > potentialLimit) break; // we have enough, this list is sorted so we don't miss anything - drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, coronaangle); - count++; + if (seed == null) { + Log.logWarning("NetworkGraph", "potential seed == null"); + continue; + } + lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); + if (lastseen > potentialLimit) { + break; // we have enough, this list is sorted so we don't miss anything } + drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, coronaangle); + count++; } totalCount += count; diff --git a/source/de/anomic/yacy/graphics/WebStructureGraph.java b/source/de/anomic/yacy/graphics/WebStructureGraph.java index 65a14e847..4bee73cb6 100644 --- a/source/de/anomic/yacy/graphics/WebStructureGraph.java +++ b/source/de/anomic/yacy/graphics/WebStructureGraph.java @@ -46,6 +46,7 @@ import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.MicroDate; import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; +import net.yacy.kelondro.util.LookAheadIterator; public class WebStructureGraph { @@ -463,21 +464,15 @@ public class WebStructureGraph { return new structureIterator(latest); } - public class structureIterator implements Iterator { + public class structureIterator extends LookAheadIterator implements Iterator { private final Iterator> i; - private structureEntry nextentry; public structureIterator(final boolean latest) { i = ((latest) ? structure_new : structure_old).entrySet().iterator(); - next0(); } - public boolean hasNext() { - return nextentry != null; - } - - private void next0() { + public structureEntry next0() { Map.Entry entry = null; String dom = null, ref = ""; while (i.hasNext()) { @@ -488,24 +483,10 @@ public class WebStructureGraph { if (dom.length() >= 8) break; dom = null; } - if ((entry == null) || (dom == null)) { - nextentry = null; - return; - } + if (entry == null || dom == null) return null; assert (ref.length() - 8) % 10 == 0 : "refs = " + ref + ", length = " + ref.length(); - nextentry = new structureEntry(dom.substring(0, 6), dom.substring(7), ref.substring(0, 8), refstr2map(ref)); - } - - public structureEntry next() { - final structureEntry r = nextentry; - next0(); - return r; - } - - public void remove() { - throw new UnsupportedOperationException("not implemented"); + return new structureEntry(dom.substring(0, 6), dom.substring(7), ref.substring(0, 8), refstr2map(ref)); } - } public static class structureEntry { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 0cee8af95..e5a18feb4 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -1018,7 +1018,7 @@ public final class yacySeedDB implements AlternativeDomainNames { } private yacySeed internalNext() { - if ((it == null) || (!(it.hasNext()))) return null; + if (it == null || !(it.hasNext())) return null; try { ConcurrentHashMap dna; while (it.hasNext()) { diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index cea13b62b..84922e2ff 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -45,6 +45,7 @@ import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.RotateIterator; import net.yacy.kelondro.util.FileUtils; +import net.yacy.kelondro.util.LookAheadIterator; import net.yacy.kelondro.util.MemoryControl; @@ -571,7 +572,7 @@ public class HeapReader { * static iterator of entries in BLOBHeap files: * this is used to import heap dumps into a write-enabled index heap */ - public static class entries implements + public static class entries extends LookAheadIterator> implements CloneableIterator>, Iterator>, Iterable> { @@ -579,14 +580,12 @@ public class HeapReader { DataInputStream is; int keylen; private final File blobFile; - Map.Entry nextEntry; public entries(final File blobFile, final int keylen) throws IOException { if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist"); this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 8*1024*1024)); this.keylen = keylen; this.blobFile = blobFile; - this.nextEntry = next0(); } public CloneableIterator> clone(Object modifier) { @@ -601,14 +600,7 @@ public class HeapReader { } } - public boolean hasNext() { - if (is == null) return false; - if (this.nextEntry != null) return true; - close(); - return false; - } - - private Map.Entry next0() { + public Map.Entry next0() { try { byte b; int len; @@ -637,10 +629,6 @@ public class HeapReader { // so far we have read this.keylen - 1 + 1 = this.keylen bytes. // there must be a remaining number of len - this.keylen bytes left for the BLOB if (len < this.keylen) return null; // a strange case that can only happen in case of corrupted data -// if (is.available() < (len - this.keylen)) { // this really indicates corrupted data but doesn't work for >2GB Blobs -// Log.logWarning("HeapReader", "corrupted data by entry of " + len + " bytes at available of: " + is.available() + " in " + this.blobFile.getName()); -// return null; -// } payload = new byte[len - this.keylen]; // the remaining record entries if (is.read(payload) < payload.length) return null; return new entry(key, payload); @@ -650,20 +638,6 @@ public class HeapReader { } } - public Map.Entry next() { - final Map.Entry n = this.nextEntry; - this.nextEntry = next0(); - return n; - } - - public void remove() { - throw new UnsupportedOperationException("blobs cannot be altered during read-only iteration"); - } - - public Iterator> iterator() { - return this; - } - public void close() { if (is != null) try { is.close(); } catch (final IOException e) {} is = null; diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index 71c95c71a..877a8c981 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -38,6 +38,7 @@ import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; +import net.yacy.kelondro.util.LookAheadIterator; import net.yacy.kelondro.util.ScoreCluster; @@ -363,29 +364,17 @@ public class MapDataMining extends MapHeap { super.close(); } - public class mapIterator implements Iterator> { + public class mapIterator extends LookAheadIterator> implements Iterator> { // enumerates Map-Type elements // the key is also included in every map that is returned; it's key is 'key' Iterator keyIterator; - Map n; public mapIterator(final Iterator keyIterator) { this.keyIterator = keyIterator; - this.n = next0(); - } - - public boolean hasNext() { - return this.n != null; - } - - public Map next() { - final Map n1 = n; - n = next0(); - return n1; } - private Map next0() { + public Map next0() { if (keyIterator == null) return null; byte[] nextKey; Map map; @@ -394,7 +383,8 @@ public class MapDataMining extends MapHeap { try { map = get(nextKey); } catch (final IOException e) { - break; + Log.logWarning("MapDataMining", e.getMessage()); + continue; } catch (RowSpaceExceededException e) { Log.logException(e); continue; @@ -405,9 +395,5 @@ public class MapDataMining extends MapHeap { } return null; } - - public void remove() { - throw new UnsupportedOperationException(); - } } // class mapIterator } diff --git a/source/net/yacy/kelondro/data/word/Word.java b/source/net/yacy/kelondro/data/word/Word.java index ec1c867b3..302cc3a3d 100644 --- a/source/net/yacy/kelondro/data/word/Word.java +++ b/source/net/yacy/kelondro/data/word/Word.java @@ -55,7 +55,7 @@ public class Word { public static final int commonHashLength = 12; private static final int hashCacheSize = Math.max(10000, Math.min(100000, (int) (MemoryControl.available() / 20000L))); - private static final ARC hashCache = new ConcurrentARC(hashCacheSize, Runtime.getRuntime().availableProcessors()); + private static final ARC hashCache = new ConcurrentARC(hashCacheSize, Runtime.getRuntime().availableProcessors() + 1); // object carries statistics for words and sentences public int count; // number of occurrences diff --git a/source/net/yacy/kelondro/index/BufferedObjectIndex.java b/source/net/yacy/kelondro/index/BufferedObjectIndex.java index 503f1d6e6..8da9012b6 100644 --- a/source/net/yacy/kelondro/index/BufferedObjectIndex.java +++ b/source/net/yacy/kelondro/index/BufferedObjectIndex.java @@ -149,6 +149,13 @@ public class BufferedObjectIndex implements Index, Iterable { } } + /** + * Adds the row to the index. The row is identified by the primary key of the row. + * @param row a index row + * @return true if this set did _not_ already contain the given row. + * @throws IOException + * @throws RowSpaceExceededException + */ public boolean put(Entry row) throws IOException, RowSpaceExceededException { synchronized (this.backend) { checkBuffer(); diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java index 3236ce5e8..080358cf1 100644 --- a/source/net/yacy/kelondro/index/HandleMap.java +++ b/source/net/yacy/kelondro/index/HandleMap.java @@ -194,6 +194,14 @@ public final class HandleMap implements Iterable { return indexentry.getColLong(1); } + /** + * Adds the key-value pair to the index. + * @param key the index key + * @param l the value + * @return the previous entry of the index + * @throws IOException + * @throws RowSpaceExceededException + */ public final synchronized long put(final byte[] key, final long l) throws RowSpaceExceededException { assert l >= 0 : "l = " + l; assert (key != null); diff --git a/source/net/yacy/kelondro/index/HandleSet.java b/source/net/yacy/kelondro/index/HandleSet.java index 94a7eb74e..11ea9cc1b 100644 --- a/source/net/yacy/kelondro/index/HandleSet.java +++ b/source/net/yacy/kelondro/index/HandleSet.java @@ -143,6 +143,13 @@ public final class HandleSet implements Iterable, Cloneable { for (byte[] b: aset) put(b); } + /** + * Adds the key to the set + * @param key + * @return true if this set did _not_ already contain the given key. + * @throws IOException + * @throws RowSpaceExceededException + */ public final synchronized boolean put(final byte[] key) throws RowSpaceExceededException { assert (key != null); final Row.Entry newentry = index.row().newEntry(); diff --git a/source/net/yacy/kelondro/index/Index.java b/source/net/yacy/kelondro/index/Index.java index b850e6820..7483f2e5a 100644 --- a/source/net/yacy/kelondro/index/Index.java +++ b/source/net/yacy/kelondro/index/Index.java @@ -44,6 +44,14 @@ public interface Index extends Iterable { public boolean has(byte[] key); // use this only if there is no get in case that has returns true public Row.Entry get(byte[] key) throws IOException; public Row.Entry replace(Row.Entry row) throws RowSpaceExceededException, IOException; + + /** + * Adds the row to the index. The row is identified by the primary key of the row. + * @param row a index row + * @return true if this set did _not_ already contain the given row. + * @throws IOException + * @throws RowSpaceExceededException + */ public boolean put(Row.Entry row) throws IOException, RowSpaceExceededException; public void addUnique(Row.Entry row) throws RowSpaceExceededException, IOException; // no double-check public ArrayList removeDoubles() throws IOException, RowSpaceExceededException; // removes all elements that are double (to be used after all addUnique) diff --git a/source/net/yacy/kelondro/index/RAMIndex.java b/source/net/yacy/kelondro/index/RAMIndex.java index 3b93b3491..f24bd01d0 100644 --- a/source/net/yacy/kelondro/index/RAMIndex.java +++ b/source/net/yacy/kelondro/index/RAMIndex.java @@ -157,33 +157,44 @@ public final class RAMIndex implements Index, Iterable { // else place it in the index1 return index1.replace(entry); } - - public final synchronized boolean put(final Row.Entry entry) throws RowSpaceExceededException { + + /** + * Adds the row to the index. The row is identified by the primary key of the row. + * @param row a index row + * @return true if this set did _not_ already contain the given row. + * @throws IOException + * @throws RowSpaceExceededException + */ + public final boolean put(final Row.Entry entry) throws RowSpaceExceededException { assert (entry != null); - if (entry == null) return false; - finishInitialization(); - // if the new entry is within the initialization part, just overwrite it - assert index0.isSorted(); - final byte[] key = entry.getPrimaryKeyBytes(); - if (index0.has(key)) { - // replace the entry - index0.put(entry); - return true; + if (entry == null) return true; + synchronized (this) { + finishInitialization(); + // if the new entry is within the initialization part, just overwrite it + assert index0.isSorted(); + final byte[] key = entry.getPrimaryKeyBytes(); + if (index0.has(key)) { + // replace the entry + index0.put(entry); + return false; + } + // else place it in the index1 + return index1.put(entry); } - // else place it in the index1 - return index1.put(entry); } - public final synchronized void addUnique(final Row.Entry entry) throws RowSpaceExceededException { + public final void addUnique(final Row.Entry entry) throws RowSpaceExceededException { assert (entry != null); if (entry == null) return; - if (index1 == null) { - // we are in the initialization phase - index0.addUnique(entry); - return; - } - // initialization is over, add to secondary index - index1.addUnique(entry); + synchronized (this) { + if (index1 == null) { + // we are in the initialization phase + index0.addUnique(entry); + return; + } + // initialization is over, add to secondary index + index1.addUnique(entry); + } } public final void addUnique(final List rows) throws RowSpaceExceededException { diff --git a/source/net/yacy/kelondro/index/RAMIndexCluster.java b/source/net/yacy/kelondro/index/RAMIndexCluster.java index 10dd313c0..e6fcfc7c4 100644 --- a/source/net/yacy/kelondro/index/RAMIndexCluster.java +++ b/source/net/yacy/kelondro/index/RAMIndexCluster.java @@ -114,6 +114,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea public final void addUnique(final Entry row) throws RowSpaceExceededException { final int i = indexFor(row); + assert i >= 0 : "i = " + i; if (i < 0) return; accessArray(i).addUnique(row); } @@ -171,10 +172,18 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return MergeIterator.cascade(col, this.rowdef.objectOrder, MergeIterator.simpleMerge, up); } } - + + /** + * Adds the row to the index. The row is identified by the primary key of the row. + * @param row a index row + * @return true if this set did _not_ already contain the given row. + * @throws IOException + * @throws RowSpaceExceededException + */ public final boolean put(final Entry row) throws RowSpaceExceededException { final int i = indexFor(row); - if (i < 0) return false; + assert i >= 0 : "i = " + i; + if (i < 0) return true; return accessArray(i).put(row); } @@ -236,6 +245,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea public final Entry replace(final Entry row) throws RowSpaceExceededException { final int i = indexFor(row); + assert i >= 0 : "i = " + i; if (i < 0) return null; return accessArray(i).replace(row); } diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index 3bbf1988c..2afbebc3f 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -24,6 +24,7 @@ package net.yacy.kelondro.index; +import java.io.IOException; import java.util.Iterator; import java.util.Random; @@ -113,6 +114,13 @@ public class RowSet extends RowCollection implements Index, Iterable return get(index, true); } + /** + * Adds the row to the index. The row is identified by the primary key of the row. + * @param row a index row + * @return true if this set did _not_ already contain the given row. + * @throws IOException + * @throws RowSpaceExceededException + */ public final synchronized boolean put(final Row.Entry entry) throws RowSpaceExceededException { assert (entry != null); assert (entry.getPrimaryKeyBytes() != null); @@ -124,12 +132,12 @@ public class RowSet extends RowCollection implements Index, Iterable final int index = find(entry.bytes(), 0); if (index < 0) { super.addUnique(entry); - return false; + return true; } else { final int sb = this.sortBound; // save the sortBound, because it is not altered (we replace at the same place) set(index, entry); // this may alter the sortBound, which we will revert in the next step this.sortBound = sb; // revert a sortBound altering - return true; + return false; } } diff --git a/source/net/yacy/kelondro/table/ChunkIterator.java b/source/net/yacy/kelondro/table/ChunkIterator.java index d08558c94..edc1d6a91 100644 --- a/source/net/yacy/kelondro/table/ChunkIterator.java +++ b/source/net/yacy/kelondro/table/ChunkIterator.java @@ -34,8 +34,9 @@ import java.io.IOException; import java.util.Iterator; import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.LookAheadIterator; -public class ChunkIterator implements Iterator { +public class ChunkIterator extends LookAheadIterator implements Iterator { private final int chunksize; @@ -52,7 +53,6 @@ public class ChunkIterator implements Iterator { private final DataInputStream stream; - private byte[] nextBytes; private final int recordsize; public ChunkIterator(final File file, final int recordsize, final int chunksize) throws FileNotFoundException { @@ -61,13 +61,8 @@ public class ChunkIterator implements Iterator { this.recordsize = recordsize; this.chunksize = chunksize; this.stream = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 64 * 1024)); - this.nextBytes = next0(); } - public boolean hasNext() { - return nextBytes != null; - } - public byte[] next0() { final byte[] chunk = new byte[chunksize]; int r, s; @@ -91,14 +86,4 @@ public class ChunkIterator implements Iterator { return null; } } - - public byte[] next() { - final byte[] n = this.nextBytes; - this.nextBytes = next0(); - return n; - } - - public void remove() { - throw new UnsupportedOperationException("no remove in ChunkIterator possible"); - } } diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index c58169185..6eac4e067 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -354,6 +354,13 @@ public class SplitTable implements Index, Iterable { return null; } + /** + * Adds the row to the index. The row is identified by the primary key of the row. + * @param row a index row + * @return true if this set did _not_ already contain the given row. + * @throws IOException + * @throws RowSpaceExceededException + */ public boolean put(final Row.Entry row) throws IOException, RowSpaceExceededException { assert row.objectsize() <= this.rowdef.objectsize; Index keeper = keeperOf(row.getColBytes(0, true)); @@ -362,7 +369,9 @@ public class SplitTable implements Index, Iterable { assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; keeper = (this.current == null) ? newTable() : checkTable(this.tables.get(this.current)); } - return keeper.put(row); + boolean b = keeper.put(row); + assert b; + return b; } diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 4e657557d..8fcce0efd 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -543,12 +543,19 @@ public class Table implements Index, Iterable { return rowdef.newEntry(b); } + /** + * Adds the row to the index. The row is identified by the primary key of the row. + * @param row a index row + * @return true if this set did _not_ already contain the given row. + * @throws IOException + * @throws RowSpaceExceededException + */ public synchronized boolean put(final Entry row) throws IOException, RowSpaceExceededException { assert file == null || file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size() + ", file = " + this.filename(); assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size() + ", file = " + this.filename(); assert row != null; assert row.bytes() != null; - if (file == null || row == null || row.bytes() == null) return false; + if (file == null || row == null || row.bytes() == null) return true; final int i = (int) index.get(row.getPrimaryKeyBytes()); if (i == -1) { try { @@ -558,7 +565,7 @@ public class Table implements Index, Iterable { this.table = null; addUnique(row); } - return false; + return true; } if (table == null) { @@ -576,7 +583,7 @@ public class Table implements Index, Iterable { } assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size(); - return true; + return false; } public Entry put(final Entry row, final Date entryDate) throws IOException, RowSpaceExceededException {