diff --git a/source/de/anomic/crawler/CrawlSwitchboard.java b/source/de/anomic/crawler/CrawlSwitchboard.java index 00f49e66d..4eeba12c8 100644 --- a/source/de/anomic/crawler/CrawlSwitchboard.java +++ b/source/de/anomic/crawler/CrawlSwitchboard.java @@ -87,12 +87,12 @@ public final class CrawlSwitchboard { final File profilesActiveFile = new File(queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES); try { - this.profilesActiveCrawls = new MapHeap(profilesActiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_'); + this.profilesActiveCrawls = new MapHeap(profilesActiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' '); } catch (IOException e) { Log.logException(e);Log.logException(e); FileUtils.deletedelete(profilesActiveFile); try { - this.profilesActiveCrawls = new MapHeap(profilesActiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_'); + this.profilesActiveCrawls = new MapHeap(profilesActiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' '); } catch (IOException e1) { Log.logException(e1); this.profilesActiveCrawls = null; @@ -107,12 +107,12 @@ public final class CrawlSwitchboard { final File profilesPassiveFile = new File(queuesRoot, DBFILE_PASSIVE_CRAWL_PROFILES); try { - this.profilesPassiveCrawls = new MapHeap(profilesPassiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_'); + this.profilesPassiveCrawls = new MapHeap(profilesPassiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' '); } catch (IOException e) { Log.logException(e);Log.logException(e); FileUtils.deletedelete(profilesActiveFile); try { - this.profilesPassiveCrawls = new MapHeap(profilesPassiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_'); + this.profilesPassiveCrawls = new MapHeap(profilesPassiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' '); } catch (IOException e1) { Log.logException(e1); this.profilesPassiveCrawls = null; @@ -259,7 +259,7 @@ public final class CrawlSwitchboard { final File pdb = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES); if (pdb.exists()) FileUtils.deletedelete(pdb); try { - this.profilesActiveCrawls = new MapHeap(pdb, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_'); + this.profilesActiveCrawls = new MapHeap(pdb, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' '); } catch (IOException e1) { Log.logException(e1); this.profilesActiveCrawls = null; diff --git a/source/de/anomic/crawler/Latency.java b/source/de/anomic/crawler/Latency.java index 8c54303eb..ad7f58426 100644 --- a/source/de/anomic/crawler/Latency.java +++ b/source/de/anomic/crawler/Latency.java @@ -246,7 +246,7 @@ public class Latency { robotsEntry = null; } robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis(); - if (robotsEntry != null && robotsDelay == 0) return "no waiting for exclusive granted peer"; // no limits if granted exclusively for this peer + if (robotsEntry != null && robotsDelay == 0 && robotsEntry.getAgentName() != null) return "no waiting for exclusive granted peer"; // no limits if granted exclusively for this peer } s.append(", robots.delay = ").append(robotsDelay); diff --git a/source/de/anomic/data/BookmarksDB.java b/source/de/anomic/data/BookmarksDB.java index a37f2996a..3cce5e037 100644 --- a/source/de/anomic/data/BookmarksDB.java +++ b/source/de/anomic/data/BookmarksDB.java @@ -71,7 +71,7 @@ public class BookmarksDB { bookmarksFile.getParentFile().mkdirs(); //this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false)); //this.bookmarksTable = new MapView(BLOBTree.toHeap(bookmarksFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, bookmarksFileNew), 1000, '_'); - this.bookmarks = new MapHeap(bookmarksFile, 12, NaturalOrder.naturalOrder, 1024 * 64, 1000, '_'); + this.bookmarks = new MapHeap(bookmarksFile, 12, NaturalOrder.naturalOrder, 1024 * 64, 1000, ' '); // tags tags = new ConcurrentHashMap(); diff --git a/source/de/anomic/data/UserDB.java b/source/de/anomic/data/UserDB.java index 0c88f9a07..edb753fc9 100644 --- a/source/de/anomic/data/UserDB.java +++ b/source/de/anomic/data/UserDB.java @@ -72,7 +72,7 @@ public final class UserDB { FileUtils.deletedelete(userTableFile); userTableFile.getParentFile().mkdirs(); try { - userTable = new MapHeap(userTableFile, 256, NaturalOrder.naturalOrder, 1024 * 64, 10, '_'); + userTable = new MapHeap(userTableFile, 128, NaturalOrder.naturalOrder, 1024 * 64, 10, '_'); } catch (IOException e) { Log.logException(e); } @@ -390,9 +390,10 @@ public final class UserDB { public long getTimeUsed() { long ret = 0L; - if (this.mem.containsKey(TIME_USED)) { + String s = this.mem.get(TIME_USED); + if (s != null && s.length() > 0) { try{ - ret = Long.parseLong(this.mem.get(TIME_USED)); + ret = Long.parseLong(s); } catch (final NumberFormatException e){ Log.logException(e); } @@ -408,15 +409,16 @@ public final class UserDB { public long getTimeLimit() { long ret = 0L; - if (this.mem.containsKey(TIME_LIMIT)) { + String s = this.mem.get(TIME_LIMIT); + if (s != null && s.length() > 0) { try { - ret = Long.parseLong(this.mem.get(TIME_LIMIT)); + ret = Long.parseLong(s); } catch (final NumberFormatException e){ Log.logException(e); } } else { try { - this.setProperty(TIME_LIMIT,"0"); + this.setProperty(TIME_LIMIT, "0"); } catch (final Exception e) { Log.logException(e); } @@ -426,15 +428,16 @@ public final class UserDB { public long getTrafficSize() { long ret = 0L; - if (this.mem.containsKey(TRAFFIC_SIZE)) { + String s = this.mem.get(TRAFFIC_SIZE); + if (s != null && s.length() > 0) { try { - ret = Long.parseLong(this.mem.get(TRAFFIC_SIZE)); + ret = Long.parseLong(s); } catch (final NumberFormatException e) { Log.logException(e); } } else { try { - this.setProperty(TRAFFIC_SIZE,"0"); + this.setProperty(TRAFFIC_SIZE, "0"); } catch (final Exception e) { Log.logException(e); } diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index ddcf81db8..739c314a8 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -79,7 +79,7 @@ public final class Cache { // open the response header database final File dbfile = new File(cachePath, RESPONSE_HEADER_DB_NAME); try { - responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 1024, 100, '_'); + responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 1024, 100, ' '); } catch (final IOException e) { Log.logException(e); } diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 359726d41..4949950a5 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -119,6 +119,7 @@ public final class yacySeedDB implements AlternativeDomainNames { this.myBotIDs = new HashSet(); this.myBotIDs.add("yacy"); this.myBotIDs.add("yacybot"); + this.myBotIDs.add("yacyproxy"); this.netRedundancy = redundancy; this.scheme = new VerticalWordPartitionScheme(partitionExponent); diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index d0f25ccc2..d60f6e130 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -590,13 +590,11 @@ public class HeapReader { } catch (IOException e) { Log.logException(e); } - } else { - // this is small.. just free resources, do not write index - if (free != null) free.clear(); - free = null; - if (index != null) index.close(); - index = null; } + if (free != null) free.clear(); + free = null; + if (index != null) index.close(); + index = null; } } diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index f5b4188c5..1e261b677 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -65,7 +65,7 @@ public class MapDataMining extends MapHeap { final String[] longaccfields, final String[] floataccfields, final Object externalHandler) throws IOException { - super(heapFile, keylength, ordering, buffermax, cachesize, '_'); + super(heapFile, keylength, ordering, buffermax, cachesize, ' '); // create fast ordering clusters and acc fields this.sortfields = sortfields; diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index 515bbc415..45c288676 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -267,6 +267,17 @@ public class MapHeap implements Map> { } return key; } + + private byte[] removeFillchar(byte[] key) { + if (key == null) return key; + int p = key.length - 1; + while (p >= 0 && key[p] == fillchar) p--; + if (p == key.length - 1) return key; + // copy part of key into new byte[] + byte[] k = new byte[p + 1]; + System.arraycopy(key, 0, k, 0, k.length); + return k; + } protected Map get(byte[] key, final boolean storeCache) throws IOException, RowSpaceExceededException { // load map from cache @@ -324,27 +335,62 @@ public class MapHeap implements Map> { */ public synchronized CloneableIterator keys(final boolean up, final boolean rotating) throws IOException { // simple enumeration of key names without special ordering - return blob.keys(up, rotating); + return new KeyIterator(up, rotating, null, null); } /** - * iterate over all keys + * return an iteration of the keys in the map + * the keys in the map are de-normalized which means that the fill-character is removed * @param up + * @param rotating * @param firstKey + * @param secondKey * @return * @throws IOException */ - public CloneableIterator keys(final boolean up, final byte[] firstKey) throws IOException { - return keys(up, false, firstKey, null); - } - public synchronized CloneableIterator keys(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException { - // simple enumeration of key names without special ordering - final CloneableIterator i = blob.keys(up, firstKey); - if (rotating) return new RotateIterator(i, secondKey, blob.size()); - return i; + return new KeyIterator(up, rotating, firstKey, secondKey); } + public class KeyIterator implements CloneableIterator, Iterator { + + final boolean up, rotating; + final byte[] firstKey, secondKey; + Iterator iterator; + + public KeyIterator(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException { + this.up = up; + this.rotating = rotating; + this.firstKey = firstKey; + this.secondKey = secondKey; + final CloneableIterator i = blob.keys(up, firstKey); + iterator = (rotating) ? new RotateIterator(i, secondKey, blob.size()) : i; + } + + public byte[] next() { + return removeFillchar(iterator.next()); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public void remove() { + iterator.remove(); + } + + @Override + public CloneableIterator clone(Object modifier) { + try { + return new KeyIterator(this.up, this.rotating, this.firstKey, this.secondKey); + } catch (IOException e) { + return null; + } + } + + } public synchronized MapIterator entries(final boolean up, final boolean rotating) throws IOException { return new MapIterator(keys(up, rotating)); @@ -399,11 +445,12 @@ public class MapHeap implements Map> { } public Map next() { - final byte[] nextKey = keyIterator.next(); + byte[] nextKey = keyIterator.next(); if (nextKey == null) { finish = true; return null; } + nextKey = normalizeKey(nextKey); // the key must be normalized because the keyIterator may iterate over not-normalized keys try { final Map obj = get(nextKey, false); if (obj == null) throw new kelondroException("no more elements available");