diff --git a/source/de/anomic/search/MediaSnippet.java b/source/de/anomic/search/MediaSnippet.java index 6f4cda49e..82c1ab00e 100644 --- a/source/de/anomic/search/MediaSnippet.java +++ b/source/de/anomic/search/MediaSnippet.java @@ -25,6 +25,7 @@ package de.anomic.search; import java.util.ArrayList; +import java.util.Comparator; import java.util.Iterator; import java.util.Map; import java.util.TreeSet; @@ -37,7 +38,7 @@ import net.yacy.kelondro.order.Base64Order; import net.yacy.repository.LoaderDispatcher; -public class MediaSnippet implements Comparable { +public class MediaSnippet implements Comparable, Comparator { public ContentDomain type; public DigestURI href, source; public String name, attr; @@ -67,6 +68,10 @@ public class MediaSnippet implements Comparable { return Base64Order.enhancedCoder.compare(this.href.hash().getBytes(), o.href.hash().getBytes()); } + public int compare(MediaSnippet o1, MediaSnippet o2) { + return o1.compareTo(o2); + } + public static ArrayList retrieveMediaSnippets(final DigestURI url, final TreeSet queryhashes, final ContentDomain mediatype, final boolean fetchOnline, final int timeout, final boolean reindexing) { if (queryhashes.size() == 0) { Log.logFine("snippet fetch", "no query hashes given for url " + url); diff --git a/source/de/anomic/search/RankingProcess.java b/source/de/anomic/search/RankingProcess.java index 221f96108..b373fea41 100644 --- a/source/de/anomic/search/RankingProcess.java +++ b/source/de/anomic/search/RankingProcess.java @@ -331,7 +331,7 @@ public final class RankingProcess extends Thread { return rwi; } // second appearances of dom - m.push(rwi); + m.push(rwi.element, rwi.weight); } // no more entries in sorted RWI entries. Now take Elements from the doubleDomCache // find best entry from all caches diff --git a/source/de/anomic/search/ResultEntry.java b/source/de/anomic/search/ResultEntry.java index a8b51e409..c32259216 100644 --- a/source/de/anomic/search/ResultEntry.java +++ b/source/de/anomic/search/ResultEntry.java @@ -28,6 +28,7 @@ package de.anomic.search; import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import net.yacy.document.Condenser; @@ -43,7 +44,7 @@ import net.yacy.kelondro.rwi.Reference; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeedDB; -public class ResultEntry implements Comparable { +public class ResultEntry implements Comparable, Comparator { // payload objects private final URIMetadataRow urlentry; @@ -166,6 +167,9 @@ public class ResultEntry implements Comparable { return urlentry.toString(textSnippet.getLineRaw()); } public int compareTo(ResultEntry o) { - return Base64Order.enhancedCoder.compare(this.hash().getBytes(), o.hash().getBytes()); + return Base64Order.enhancedCoder.compare(this.urlentry.hash().getBytes(), o.urlentry.hash().getBytes()); + } + public int compare(ResultEntry o1, ResultEntry o2) { + return Base64Order.enhancedCoder.compare(o1.urlentry.hash().getBytes(), o2.urlentry.hash().getBytes()); } } diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index 3bf07bad7..93f867790 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -179,7 +179,7 @@ public class ResultFetcher { // apply post-ranking long ranking = Long.valueOf(rankedCache.getOrder().cardinal(resultEntry.word())); ranking += postRanking(resultEntry, rankedCache.getTopics()); - + //System.out.println("*** resultEntry.hash = " + resultEntry.hash()); result.push(resultEntry, ranking); if (nav_topics) rankedCache.addTopics(resultEntry); //System.out.println("DEBUG SNIPPET_LOADING: thread " + id + " got " + resultEntry.url()); @@ -339,7 +339,9 @@ public class ResultFetcher { if (imagemedia != null) { for (int j = 0; j < imagemedia.size(); j++) { ms = imagemedia.get(j); + int b = images.size(); images.push(ms, Long.valueOf(ms.ranking)); + System.out.println("*** image " + ms.href.hash() + " images.size = " + b + "/" + images.size()); } } } diff --git a/source/net/yacy/document/geolocalization/Coordinates.java b/source/net/yacy/document/geolocalization/Coordinates.java index 5171dfa3d..d6fce283b 100644 --- a/source/net/yacy/document/geolocalization/Coordinates.java +++ b/source/net/yacy/document/geolocalization/Coordinates.java @@ -26,7 +26,9 @@ package net.yacy.document.geolocalization; -public class Coordinates implements Comparable { +import java.util.Comparator; + +public class Coordinates implements Comparable, Comparator { private static final double tenmeter = 90.0 / 1.0e6; @@ -76,6 +78,10 @@ public class Coordinates implements Comparable { return 0; } + public int compare(Coordinates o1, Coordinates o2) { + return o1.compareTo(o2); + } + /** * equality test that is needed to use the class inside HashMap/HashSet */ diff --git a/source/net/yacy/document/parser/html/ImageEntry.java b/source/net/yacy/document/parser/html/ImageEntry.java index 8db0cf421..623b44dd6 100644 --- a/source/net/yacy/document/parser/html/ImageEntry.java +++ b/source/net/yacy/document/parser/html/ImageEntry.java @@ -24,9 +24,11 @@ package net.yacy.document.parser.html; +import java.util.Comparator; + import net.yacy.kelondro.data.meta.DigestURI; -public class ImageEntry implements Comparable { +public class ImageEntry implements Comparable, Comparator { private final DigestURI url; private final String alt; @@ -84,6 +86,10 @@ public class ImageEntry implements Comparable { if (thc > ohc) return 1; return this.url.toString().compareTo((h).url.toString()); } + + public int compare(ImageEntry o1, ImageEntry o2) { + return o1.compareTo(o2); + } @Override public boolean equals(final Object o) { diff --git a/source/net/yacy/kelondro/index/Row.java b/source/net/yacy/kelondro/index/Row.java index 16f155d0a..0deadb196 100644 --- a/source/net/yacy/kelondro/index/Row.java +++ b/source/net/yacy/kelondro/index/Row.java @@ -226,7 +226,7 @@ public final class Row { } - public class Entry implements Comparable { + public class Entry implements Comparable, Comparator { private byte[] rowinstance; private int offset; // the offset where the row starts within rowinstance @@ -332,6 +332,10 @@ public final class Row { return objectOrder.compare(this.bytes(), 0, this.getPrimaryKeyLength(), o.bytes(), 0, o.getPrimaryKeyLength()); } + public int compare(Entry o1, Entry o2) { + return o1.compareTo(o2); + } + public final boolean equals(final Entry otherEntry) { // compares the content of the complete entry final byte[] t = this.bytes(); diff --git a/source/net/yacy/kelondro/table/Records.java b/source/net/yacy/kelondro/table/Records.java index 5b9612aab..3052dca34 100644 --- a/source/net/yacy/kelondro/table/Records.java +++ b/source/net/yacy/kelondro/table/Records.java @@ -30,6 +30,7 @@ package net.yacy.kelondro.table; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.Set; @@ -1219,7 +1220,7 @@ public class Records { } - public class Handle implements Comparable { + public class Handle implements Comparable, Comparator { public final static int NUL = Integer.MIN_VALUE; // the meta value for the kelondroTray' NUL abstraction diff --git a/source/net/yacy/kelondro/util/SortStack.java b/source/net/yacy/kelondro/util/SortStack.java index 9d993d19b..cb4af740f 100644 --- a/source/net/yacy/kelondro/util/SortStack.java +++ b/source/net/yacy/kelondro/util/SortStack.java @@ -40,7 +40,7 @@ public class SortStack> { // for a double-check. private TreeMap> onstack; // object within the stack - private HashSet instack; // keeps track which element has been on the stack or is now in the offstack + private HashSet instack; // keeps track which element has been on the stack protected int maxsize; public SortStack(final int maxsize) { @@ -52,11 +52,7 @@ public class SortStack> { } public int size() { - return this.onstack.size(); - } - - public void push(final stackElement se) { - push(se.element, se.weight); + return this.instack.size(); } /** @@ -113,6 +109,7 @@ public class SortStack> { final List l = this.onstack.get(w); final E element = l.remove(0); if (l.size() == 0) this.onstack.remove(w); + this.instack.remove(element); return new stackElement(element, w); } diff --git a/source/net/yacy/kelondro/util/SortStore.java b/source/net/yacy/kelondro/util/SortStore.java index 9b7e4c483..ccb74d953 100644 --- a/source/net/yacy/kelondro/util/SortStore.java +++ b/source/net/yacy/kelondro/util/SortStore.java @@ -27,6 +27,7 @@ package net.yacy.kelondro.util; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; /** @@ -38,10 +39,18 @@ import java.util.Iterator; public class SortStore> extends SortStack { private final ArrayList offstack; // objects that had been on the stack but had been removed + private HashSet offset; // keeps track which element has been on the stack or is now in the offstack + private long largest; + + public SortStore() { + this(-1); + } public SortStore(final int maxsize) { super(maxsize); + this.largest = Long.MIN_VALUE; this.offstack = new ArrayList(); + this.offset = new HashSet(); } public int size() { @@ -53,10 +62,12 @@ public class SortStore> extends SortStack { } public synchronized void push(final E element, final Long weight) { + if (this.offset.contains(element)) return; super.push(element, weight); + this.largest = Math.max(this.largest, weight.longValue()); if (this.maxsize <= 0) return; - while ((super.size() > 0) && (super.size() + this.offstack.size() > this.maxsize)) { - super.pop(); + while ((super.size() > 0) && (this.size() > this.maxsize)) { + this.pop(); } } @@ -69,9 +80,19 @@ public class SortStore> extends SortStack { final stackElement se = super.pop(); if (se == null) return null; this.offstack.add(se); + this.offset.add(se.element); return se; } + public synchronized stackElement top() { + return super.top(); + } + + public synchronized boolean exists(final E element) { + if (super.exists(element)) return true; + return this.offset.contains(element); + } + /** * return an element from a specific position. It is either taken from the offstack, * or removed from the onstack. @@ -84,7 +105,7 @@ public class SortStore> extends SortStack { return this.offstack.get(position); } if (position >= size()) return null; // we don't have that element - while (position >= this.offstack.size()) this.offstack.add(super.pop()); + while (position >= this.offstack.size()) this.pop(); return this.offstack.get(position); } @@ -95,18 +116,18 @@ public class SortStore> extends SortStack { * @param count * @return */ - public ArrayList list(final int count) { + public synchronized ArrayList list(final int count) { if (count < 0) { // shift all elements - while (super.size() > 0) this.offstack.add(super.pop()); + while (super.size() > 0) this.offstack.add(this.pop()); return this.offstack; } if (size() < count) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + size() + ")"); - while (this.offstack.size() < count) this.offstack.add(super.pop()); + while (this.offstack.size() < count) this.offstack.add(this.pop()); return this.offstack; } - public void remove(final E element) { + public synchronized void remove(final E element) { super.remove(element); Iterator i = this.offstack.iterator(); while (i.hasNext()) { @@ -116,4 +137,26 @@ public class SortStore> extends SortStack { } } } + + public synchronized boolean bottom(final long weight) { + if (super.bottom(weight)) return true; + return weight >= this.largest; + } + + public static void main(String[] args) { + SortStore a = new SortStore(); + a.push("abc", 1L); + a.pop(); + a.push("abc", 2L); + a.push("6s_7dfZk4xvc", 1L); + a.push("6s_7dfZk4xvc", 1L); + a.push("6s_7dfZk4xvc", 1L); + a.push("6s_7dfZk4xvc", 1L); + a.push("6s_7dfZk4xvc", 1L); + a.push("6s_7dfZk4xvc", 1L); + a.push("6s_7dfZk4xvc", 1L); + a.push("6s_7dfZk4xvc", 1L); + a.pop(); + System.out.println("size = " + a.size()); + } }