diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 94c4ee48e..0d8e28374 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -453,6 +453,7 @@ public class Domains { globalHosts = null; } else try { globalHosts = new KeyList(globalHostsnameCache); + Log.logInfo("Domains", "loaded globalHosts cache of hostnames, size = " + globalHosts.size()); } catch (final IOException e) { globalHosts = null; } diff --git a/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java b/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java index 879e6e5ed..a240c3238 100644 --- a/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java +++ b/source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java @@ -12,12 +12,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -41,7 +41,7 @@ import java.util.concurrent.TimeUnit; */ public class WeakPriorityBlockingQueue { - + private final TreeSet> queue; // object within the stack, ordered using a TreeSet private final Semaphore enqueued; // semaphore for elements in the stack private final ArrayList> drained; // objects that had been on the stack but had been removed @@ -70,7 +70,7 @@ public class WeakPriorityBlockingQueue { this.queue.clear(); this.enqueued.drainPermits(); } - + /** * test if the queue is empty * @return true if the queue is empty, false if not @@ -78,7 +78,7 @@ public class WeakPriorityBlockingQueue { public boolean isEmpty() { return this.queue.isEmpty() & this.drained.isEmpty(); } - + /** * get the number of elements in the queue, waiting to be removed with take() or poll() * @return @@ -89,7 +89,7 @@ public class WeakPriorityBlockingQueue { /** - * get the number of elements that had been drained so far and are wainting + * get the number of elements that had been drained so far and are waiting * in a list to get enumerated with element() * @return */ @@ -103,9 +103,9 @@ public class WeakPriorityBlockingQueue { * @return */ public synchronized int sizeAvailable() { - return this.queue.size() + this.drained.size(); + return Math.min(this.maxsize, this.queue.size() + this.drained.size()); } - + /** * put a element on the stack using a order of the weight * elements that had been on the stack cannot be put in again, @@ -126,7 +126,7 @@ public class WeakPriorityBlockingQueue { } assert this.queue.size() >= this.enqueued.availablePermits() : "(put) queue.size() = " + this.queue.size() + ", enqueued.availablePermits() = " + this.enqueued.availablePermits(); } - + /** * return the element with the smallest weight and remove it from the stack * @return null if no element is on the queue or the head of the queue @@ -138,7 +138,7 @@ public class WeakPriorityBlockingQueue { return takeUnsafe(); } } - + /** * Retrieves and removes the head of this queue, waiting if necessary * up to the specified wait time if no elements are present on this queue. @@ -153,7 +153,7 @@ public class WeakPriorityBlockingQueue { return takeUnsafe(); } } - + /** * Retrieves and removes the head of this queue, waiting if no elements are present on this queue. * @return the head element from the queue @@ -165,17 +165,17 @@ public class WeakPriorityBlockingQueue { return takeUnsafe(); } } - + private Element takeUnsafe() { final Element element = this.queue.first(); assert element != null; this.queue.remove(element); - this.drained.add(element); + if (this.drained.size() < this.maxsize) this.drained.add(element); assert this.queue.size() >= this.enqueued.availablePermits() : "(take) queue.size() = " + this.queue.size() + ", enqueued.availablePermits() = " + this.enqueued.availablePermits(); return element; } - + /** * return the element with the smallest weight, but do not remove it * @return null if no element is on the queue or the head of the queue @@ -184,7 +184,7 @@ public class WeakPriorityBlockingQueue { if (this.queue.isEmpty()) return null; return this.queue.first(); } - + /** * all objects that have been returned by poll or take are stored in a back-up list * where they can be retrieved afterward. The elements from that list are stored in @@ -214,7 +214,7 @@ public class WeakPriorityBlockingQueue { return this.drained.get(position); } } - + /** * retrieve an element from the drained queue but wait until a timeout * until returning null when no element will be available within the time @@ -237,7 +237,7 @@ public class WeakPriorityBlockingQueue { if (position >= this.drained.size()) return null; // we still don't have that element return this.drained.get(position); } - + /** * return the specific amount of entries as they would be retrievable with element() * if count is < 0 then all elements are taken @@ -249,11 +249,11 @@ public class WeakPriorityBlockingQueue { if (count < 0) { return list(); } - if (count > sizeAvailable()) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + sizeAvailable() + ")"); + if (count > sizeAvailable()) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + sizeAvailable() + ")"); while (count > this.drained.size()) this.poll(); return this.drained; } - + /** * return all entries as they would be retrievable with element() * @return a list of all elements in the stack @@ -263,7 +263,7 @@ public class WeakPriorityBlockingQueue { while (!this.queue.isEmpty()) this.poll(); return this.drained; } - + /** * iterate over all elements available. All elements that are still in the queue are drained to recorded positions * @return an iterator over all drained positions. @@ -283,20 +283,23 @@ public class WeakPriorityBlockingQueue { @Override public String toString(); } - + protected abstract static class AbstractElement implements Element { public long weight; public E element; - + + @Override public long getWeight() { return this.weight; } - + + @Override public E getElement() { return this.element; } - + + @Override public boolean equals(Element o) { return this.element.equals(o.getElement()); } @@ -305,13 +308,13 @@ public class WeakPriorityBlockingQueue { public int hashCode() { return this.element.hashCode(); } - + @Override public String toString() { - return element.toString() + "/" + weight; + return this.element.toString() + "/" + this.weight; } } - + /** * natural ordering elements, can be used as container of objects in the priority queue * the elements with smallest ordering weights are first in the queue when elements are taken @@ -323,10 +326,12 @@ public class WeakPriorityBlockingQueue { this.weight = weight; } + @Override public int compare(NaturalElement o1, NaturalElement o2) { return o1.compareTo(o2); } - + + @Override public int compareTo(NaturalElement o) { if (this.element == o.getElement()) return 0; if (this.element.equals(o.getElement())) return 0; @@ -338,9 +343,9 @@ public class WeakPriorityBlockingQueue { if (o1h < o2h) return -1; return 0; } - + } - + /** * reverse ordering elements, can be used as container of objects in the priority queue * the elements with highest ordering weights are first in the queue when elements are taken @@ -352,10 +357,12 @@ public class WeakPriorityBlockingQueue { this.weight = weight; } + @Override public int compare(ReverseElement o1, ReverseElement o2) { return o1.compareTo(o2); } - + + @Override public int compareTo(ReverseElement o) { if (this.element == o.getElement()) return 0; if (this.element.equals(o.getElement())) return 0; @@ -368,7 +375,7 @@ public class WeakPriorityBlockingQueue { return 0; } } - + public static void main(String[] args) { final WeakPriorityBlockingQueue a = new WeakPriorityBlockingQueue(3); //final Element REVERSE_POISON = new ReverseElement("", Long.MIN_VALUE); diff --git a/source/net/yacy/cora/storage/HashARC.java b/source/net/yacy/cora/storage/HashARC.java index b67a95d33..3a9f54dd6 100644 --- a/source/net/yacy/cora/storage/HashARC.java +++ b/source/net/yacy/cora/storage/HashARC.java @@ -37,13 +37,13 @@ public final class HashARC extends SimpleARC implements Map, I public HashARC(final int cacheSize) { this.cacheSize = cacheSize / 2; - super.levelA = Collections.synchronizedMap(new LinkedHashMap(cacheSize, 0.1f, accessOrder) { + super.levelA = Collections.synchronizedMap(new LinkedHashMap(1, 0.1f, accessOrder) { private static final long serialVersionUID = 1L; @Override protected boolean removeEldestEntry(final Map.Entry eldest) { return size() > HashARC.this.cacheSize; } }); - this.levelB = Collections.synchronizedMap(new LinkedHashMap(cacheSize, 0.1f, accessOrder) { + this.levelB = Collections.synchronizedMap(new LinkedHashMap(1, 0.1f, accessOrder) { private static final long serialVersionUID = 1L; @Override protected boolean removeEldestEntry(final Map.Entry eldest) { return size() > HashARC.this.cacheSize; diff --git a/source/net/yacy/cora/storage/KeyList.java b/source/net/yacy/cora/storage/KeyList.java index 9fd0b6588..3697403ad 100644 --- a/source/net/yacy/cora/storage/KeyList.java +++ b/source/net/yacy/cora/storage/KeyList.java @@ -76,6 +76,10 @@ public class KeyList implements Iterable { } + public int size() { + return this.keys.size(); + } + public boolean contains(final String key) { return this.keys.containsKey(key.trim().toLowerCase()); } diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java index 42590aaee..0747e415c 100644 --- a/source/net/yacy/kelondro/index/HandleMap.java +++ b/source/net/yacy/kelondro/index/HandleMap.java @@ -98,6 +98,7 @@ public final class HandleMap implements Iterable { is.close(); is = null; assert this.index.size() == file.length() / (keylength + idxbytes); + trim(); } public void trim() { @@ -415,6 +416,7 @@ public final class HandleMap implements Iterable { return this.result.get(); } + @Override public final HandleMap call() throws IOException { try { finishloop: while (true) { @@ -439,7 +441,8 @@ public final class HandleMap implements Iterable { } } - public Iterator iterator() { + @Override + public Iterator iterator() { return rows(true, null); } } diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 829db45b1..bda9669f3 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -473,6 +473,7 @@ public final class ReferenceContainerArray { } } } + references.trim(); System.out.println("CELL REFERENCE COLLECTION finished"); return references; } diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 3e0e94999..b2483be11 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -187,6 +187,7 @@ public class Table implements Index, Iterable { } } } + this.index.trim(); // open the file this.file = new BufferedRecords(new Records(tablefile, rowdef.objectsize), this.buffersize); @@ -594,6 +595,7 @@ public class Table implements Index, Iterable { * @throws IOException * @throws RowSpaceExceededException */ + @Override public boolean put(final Entry row) throws IOException, RowSpaceExceededException { assert row != null; if (this.file == null || row == null) return true; @@ -702,6 +704,7 @@ public class Table implements Index, Iterable { } } + @Override public boolean delete(final byte[] key) throws IOException { return remove(key) != null; } diff --git a/source/net/yacy/search/query/SnippetProcess.java b/source/net/yacy/search/query/SnippetProcess.java index 6a6188210..ed8151973 100644 --- a/source/net/yacy/search/query/SnippetProcess.java +++ b/source/net/yacy/search/query/SnippetProcess.java @@ -109,8 +109,8 @@ public class SnippetProcess { this.urlRetrievalAllTime = 0; this.snippetComputationAllTime = 0; - this.result = new WeakPriorityBlockingQueue(-1); // this is the result, enriched with snippets, ranked and ordered by ranking - this.images = new WeakPriorityBlockingQueue(-1); + this.result = new WeakPriorityBlockingQueue(Math.max(1000, 10 * query.itemsPerPage())); // this is the result, enriched with snippets, ranked and ordered by ranking + this.images = new WeakPriorityBlockingQueue(Math.max(1000, 10 * query.itemsPerPage())); // snippets do not need to match with the complete query hashes, // only with the query minus the stopwords which had not been used for the search diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java index 60133bed2..60da24613 100644 --- a/source/net/yacy/search/snippet/TextSnippet.java +++ b/source/net/yacy/search/snippet/TextSnippet.java @@ -206,10 +206,10 @@ public class TextSnippet implements Comparable, Comparator