diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java
index 94c4ee48e..0d8e28374 100644
--- a/source/net/yacy/cora/protocol/Domains.java
+++ b/source/net/yacy/cora/protocol/Domains.java
@@ -453,6 +453,7 @@ public class Domains {
globalHosts = null;
} else try {
globalHosts = new KeyList(globalHostsnameCache);
+ Log.logInfo("Domains", "loaded globalHosts cache of hostnames, size = " + globalHosts.size());
} catch (final IOException e) {
globalHosts = null;
}
diff --git a/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java b/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java
index 879e6e5ed..a240c3238 100644
--- a/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java
+++ b/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java
@@ -12,12 +12,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
- *
+ *
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
- *
+ *
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see .
@@ -41,7 +41,7 @@ import java.util.concurrent.TimeUnit;
*/
public class WeakPriorityBlockingQueue {
-
+
private final TreeSet> queue; // object within the stack, ordered using a TreeSet
private final Semaphore enqueued; // semaphore for elements in the stack
private final ArrayList> drained; // objects that had been on the stack but had been removed
@@ -70,7 +70,7 @@ public class WeakPriorityBlockingQueue {
this.queue.clear();
this.enqueued.drainPermits();
}
-
+
/**
* test if the queue is empty
* @return true if the queue is empty, false if not
@@ -78,7 +78,7 @@ public class WeakPriorityBlockingQueue {
public boolean isEmpty() {
return this.queue.isEmpty() & this.drained.isEmpty();
}
-
+
/**
* get the number of elements in the queue, waiting to be removed with take() or poll()
* @return
@@ -89,7 +89,7 @@ public class WeakPriorityBlockingQueue {
/**
- * get the number of elements that had been drained so far and are wainting
+ * get the number of elements that had been drained so far and are waiting
* in a list to get enumerated with element()
* @return
*/
@@ -103,9 +103,9 @@ public class WeakPriorityBlockingQueue {
* @return
*/
public synchronized int sizeAvailable() {
- return this.queue.size() + this.drained.size();
+ return Math.min(this.maxsize, this.queue.size() + this.drained.size());
}
-
+
/**
* put a element on the stack using a order of the weight
* elements that had been on the stack cannot be put in again,
@@ -126,7 +126,7 @@ public class WeakPriorityBlockingQueue {
}
assert this.queue.size() >= this.enqueued.availablePermits() : "(put) queue.size() = " + this.queue.size() + ", enqueued.availablePermits() = " + this.enqueued.availablePermits();
}
-
+
/**
* return the element with the smallest weight and remove it from the stack
* @return null if no element is on the queue or the head of the queue
@@ -138,7 +138,7 @@ public class WeakPriorityBlockingQueue {
return takeUnsafe();
}
}
-
+
/**
* Retrieves and removes the head of this queue, waiting if necessary
* up to the specified wait time if no elements are present on this queue.
@@ -153,7 +153,7 @@ public class WeakPriorityBlockingQueue {
return takeUnsafe();
}
}
-
+
/**
* Retrieves and removes the head of this queue, waiting if no elements are present on this queue.
* @return the head element from the queue
@@ -165,17 +165,17 @@ public class WeakPriorityBlockingQueue {
return takeUnsafe();
}
}
-
+
private Element takeUnsafe() {
final Element element = this.queue.first();
assert element != null;
this.queue.remove(element);
- this.drained.add(element);
+ if (this.drained.size() < this.maxsize) this.drained.add(element);
assert this.queue.size() >= this.enqueued.availablePermits() : "(take) queue.size() = " + this.queue.size() + ", enqueued.availablePermits() = " + this.enqueued.availablePermits();
return element;
}
-
+
/**
* return the element with the smallest weight, but do not remove it
* @return null if no element is on the queue or the head of the queue
@@ -184,7 +184,7 @@ public class WeakPriorityBlockingQueue {
if (this.queue.isEmpty()) return null;
return this.queue.first();
}
-
+
/**
* all objects that have been returned by poll or take are stored in a back-up list
* where they can be retrieved afterward. The elements from that list are stored in
@@ -214,7 +214,7 @@ public class WeakPriorityBlockingQueue {
return this.drained.get(position);
}
}
-
+
/**
* retrieve an element from the drained queue but wait until a timeout
* until returning null when no element will be available within the time
@@ -237,7 +237,7 @@ public class WeakPriorityBlockingQueue {
if (position >= this.drained.size()) return null; // we still don't have that element
return this.drained.get(position);
}
-
+
/**
* return the specific amount of entries as they would be retrievable with element()
* if count is < 0 then all elements are taken
@@ -249,11 +249,11 @@ public class WeakPriorityBlockingQueue {
if (count < 0) {
return list();
}
- if (count > sizeAvailable()) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + sizeAvailable() + ")");
+ if (count > sizeAvailable()) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + sizeAvailable() + ")");
while (count > this.drained.size()) this.poll();
return this.drained;
}
-
+
/**
* return all entries as they would be retrievable with element()
* @return a list of all elements in the stack
@@ -263,7 +263,7 @@ public class WeakPriorityBlockingQueue {
while (!this.queue.isEmpty()) this.poll();
return this.drained;
}
-
+
/**
* iterate over all elements available. All elements that are still in the queue are drained to recorded positions
* @return an iterator over all drained positions.
@@ -283,20 +283,23 @@ public class WeakPriorityBlockingQueue {
@Override
public String toString();
}
-
+
protected abstract static class AbstractElement implements Element {
public long weight;
public E element;
-
+
+ @Override
public long getWeight() {
return this.weight;
}
-
+
+ @Override
public E getElement() {
return this.element;
}
-
+
+ @Override
public boolean equals(Element o) {
return this.element.equals(o.getElement());
}
@@ -305,13 +308,13 @@ public class WeakPriorityBlockingQueue {
public int hashCode() {
return this.element.hashCode();
}
-
+
@Override
public String toString() {
- return element.toString() + "/" + weight;
+ return this.element.toString() + "/" + this.weight;
}
}
-
+
/**
* natural ordering elements, can be used as container of objects in the priority queue
* the elements with smallest ordering weights are first in the queue when elements are taken
@@ -323,10 +326,12 @@ public class WeakPriorityBlockingQueue {
this.weight = weight;
}
+ @Override
public int compare(NaturalElement o1, NaturalElement o2) {
return o1.compareTo(o2);
}
-
+
+ @Override
public int compareTo(NaturalElement o) {
if (this.element == o.getElement()) return 0;
if (this.element.equals(o.getElement())) return 0;
@@ -338,9 +343,9 @@ public class WeakPriorityBlockingQueue {
if (o1h < o2h) return -1;
return 0;
}
-
+
}
-
+
/**
* reverse ordering elements, can be used as container of objects in the priority queue
* the elements with highest ordering weights are first in the queue when elements are taken
@@ -352,10 +357,12 @@ public class WeakPriorityBlockingQueue {
this.weight = weight;
}
+ @Override
public int compare(ReverseElement o1, ReverseElement o2) {
return o1.compareTo(o2);
}
-
+
+ @Override
public int compareTo(ReverseElement o) {
if (this.element == o.getElement()) return 0;
if (this.element.equals(o.getElement())) return 0;
@@ -368,7 +375,7 @@ public class WeakPriorityBlockingQueue {
return 0;
}
}
-
+
public static void main(String[] args) {
final WeakPriorityBlockingQueue a = new WeakPriorityBlockingQueue(3);
//final Element REVERSE_POISON = new ReverseElement("", Long.MIN_VALUE);
diff --git a/source/net/yacy/cora/storage/HashARC.java b/source/net/yacy/cora/storage/HashARC.java
index b67a95d33..3a9f54dd6 100644
--- a/source/net/yacy/cora/storage/HashARC.java
+++ b/source/net/yacy/cora/storage/HashARC.java
@@ -37,13 +37,13 @@ public final class HashARC extends SimpleARC implements Map, I
public HashARC(final int cacheSize) {
this.cacheSize = cacheSize / 2;
- super.levelA = Collections.synchronizedMap(new LinkedHashMap(cacheSize, 0.1f, accessOrder) {
+ super.levelA = Collections.synchronizedMap(new LinkedHashMap(1, 0.1f, accessOrder) {
private static final long serialVersionUID = 1L;
@Override protected boolean removeEldestEntry(final Map.Entry eldest) {
return size() > HashARC.this.cacheSize;
}
});
- this.levelB = Collections.synchronizedMap(new LinkedHashMap(cacheSize, 0.1f, accessOrder) {
+ this.levelB = Collections.synchronizedMap(new LinkedHashMap(1, 0.1f, accessOrder) {
private static final long serialVersionUID = 1L;
@Override protected boolean removeEldestEntry(final Map.Entry eldest) {
return size() > HashARC.this.cacheSize;
diff --git a/source/net/yacy/cora/storage/KeyList.java b/source/net/yacy/cora/storage/KeyList.java
index 9fd0b6588..3697403ad 100644
--- a/source/net/yacy/cora/storage/KeyList.java
+++ b/source/net/yacy/cora/storage/KeyList.java
@@ -76,6 +76,10 @@ public class KeyList implements Iterable {
}
+ public int size() {
+ return this.keys.size();
+ }
+
public boolean contains(final String key) {
return this.keys.containsKey(key.trim().toLowerCase());
}
diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java
index 42590aaee..0747e415c 100644
--- a/source/net/yacy/kelondro/index/HandleMap.java
+++ b/source/net/yacy/kelondro/index/HandleMap.java
@@ -98,6 +98,7 @@ public final class HandleMap implements Iterable {
is.close();
is = null;
assert this.index.size() == file.length() / (keylength + idxbytes);
+ trim();
}
public void trim() {
@@ -415,6 +416,7 @@ public final class HandleMap implements Iterable {
return this.result.get();
}
+ @Override
public final HandleMap call() throws IOException {
try {
finishloop: while (true) {
@@ -439,7 +441,8 @@ public final class HandleMap implements Iterable {
}
}
- public Iterator iterator() {
+ @Override
+ public Iterator iterator() {
return rows(true, null);
}
}
diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
index 829db45b1..bda9669f3 100644
--- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
+++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
@@ -473,6 +473,7 @@ public final class ReferenceContainerArray {
}
}
}
+ references.trim();
System.out.println("CELL REFERENCE COLLECTION finished");
return references;
}
diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java
index 3e0e94999..b2483be11 100644
--- a/source/net/yacy/kelondro/table/Table.java
+++ b/source/net/yacy/kelondro/table/Table.java
@@ -187,6 +187,7 @@ public class Table implements Index, Iterable {
}
}
}
+ this.index.trim();
// open the file
this.file = new BufferedRecords(new Records(tablefile, rowdef.objectsize), this.buffersize);
@@ -594,6 +595,7 @@ public class Table implements Index, Iterable {
* @throws IOException
* @throws RowSpaceExceededException
*/
+ @Override
public boolean put(final Entry row) throws IOException, RowSpaceExceededException {
assert row != null;
if (this.file == null || row == null) return true;
@@ -702,6 +704,7 @@ public class Table implements Index, Iterable {
}
}
+ @Override
public boolean delete(final byte[] key) throws IOException {
return remove(key) != null;
}
diff --git a/source/net/yacy/search/query/SnippetProcess.java b/source/net/yacy/search/query/SnippetProcess.java
index 6a6188210..ed8151973 100644
--- a/source/net/yacy/search/query/SnippetProcess.java
+++ b/source/net/yacy/search/query/SnippetProcess.java
@@ -109,8 +109,8 @@ public class SnippetProcess {
this.urlRetrievalAllTime = 0;
this.snippetComputationAllTime = 0;
- this.result = new WeakPriorityBlockingQueue(-1); // this is the result, enriched with snippets, ranked and ordered by ranking
- this.images = new WeakPriorityBlockingQueue(-1);
+ this.result = new WeakPriorityBlockingQueue(Math.max(1000, 10 * query.itemsPerPage())); // this is the result, enriched with snippets, ranked and ordered by ranking
+ this.images = new WeakPriorityBlockingQueue(Math.max(1000, 10 * query.itemsPerPage()));
// snippets do not need to match with the complete query hashes,
// only with the query minus the stopwords which had not been used for the search
diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java
index 60133bed2..60da24613 100644
--- a/source/net/yacy/search/snippet/TextSnippet.java
+++ b/source/net/yacy/search/snippet/TextSnippet.java
@@ -206,10 +206,10 @@ public class TextSnippet implements Comparable, Comparator