diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.java index 1ef99630e..f944fd2e7 100644 --- a/htroot/AccessTracker_p.java +++ b/htroot/AccessTracker_p.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -33,7 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.TreeSet; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.LinkedBlockingQueue; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -48,20 +48,20 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; public class AccessTracker_p { - + private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US); private static Collection listclone (final Collection m) { - final Collection accessClone = new ConcurrentLinkedQueue(); + final Collection accessClone = new LinkedBlockingQueue(); try { accessClone.addAll(m); } catch (final ConcurrentModificationException e) {} return accessClone; } - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; - + // return variable that accumulates replacements final serverObjects prop = new serverObjects(); prop.setLocalized(!(header.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml")); @@ -70,7 +70,7 @@ public class AccessTracker_p { page = post.getInt("page", 0); } prop.put("page", page); - + final int maxCount = 1000; boolean dark = true; if (page == 0) { @@ -90,7 +90,7 @@ public class AccessTracker_p { } catch (final ConcurrentModificationException e) {} // we don't want to synchronize this prop.put("page_list", entCount); prop.put("page_num", entCount); - + entCount = 0; try { for (final Map.Entry bfe: serverCore.bfHost.entrySet()) { @@ -152,7 +152,7 @@ public class AccessTracker_p { long stimeSum1 = 0; long rtimeSum1 = 0; int m = 0; - + while (ai.hasNext()) { try { query = ai.next(); @@ -198,7 +198,7 @@ public class AccessTracker_p { prop.put("page_list", m); prop.put("page_num", m); prop.put("page_resultcount", rcount); - + // Put -1 instead of NaN as result for empty search list and return the safe HTML blank char for table output if (m == 0) { m = -1; @@ -249,7 +249,7 @@ public class AccessTracker_p { entry = i.next(); host = entry.getKey(); handles = entry.getValue(); - + int dateCount = 0; final Iterator ii = handles.iterator(); while (ii.hasNext()) { @@ -262,7 +262,7 @@ public class AccessTracker_p { final int qph = handles.tailSet(Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size(); qphSum += qph; prop.put("page_list_" + m + "_qph", qph); - + prop.put("page_list_" + m + "_dark", ((dark) ? 1 : 0) ); dark =! dark; prop.putHTML("page_list_" + m + "_host", host); if (page == 5) { @@ -276,7 +276,7 @@ public class AccessTracker_p { } } catch (final ConcurrentModificationException e) {} // we dont want to synchronize this // return empty values to not break the table view if no results can be listed - if (m==0) { + if (m==0) { prop.put("page_list", 1); prop.put("page_list_0_dates_0_date", ""); prop.put("page_list_0_dates", 1); @@ -295,5 +295,5 @@ public class AccessTracker_p { // return rewrite properties return prop; } - + } diff --git a/source/de/anomic/crawler/ResultImages.java b/source/de/anomic/crawler/ResultImages.java index b095744d4..2e1c681dd 100644 --- a/source/de/anomic/crawler/ResultImages.java +++ b/source/de/anomic/crawler/ResultImages.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -27,9 +27,10 @@ package de.anomic.crawler; import java.util.Map; +import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.document.Document; @@ -43,30 +44,30 @@ public class ResultImages { // we maintain two different queues for private and public crawls and divide both into two halves: // such images that appear to be good quality for a image monitor bacause their size is known, and other images // that are not declared with sizes. - private static final ConcurrentLinkedQueue privateImageQueueHigh = new ConcurrentLinkedQueue(); - private static final ConcurrentLinkedQueue privateImageQueueLow = new ConcurrentLinkedQueue(); - private static final ConcurrentLinkedQueue publicImageQueueHigh = new ConcurrentLinkedQueue(); - private static final ConcurrentLinkedQueue publicImageQueueLow = new ConcurrentLinkedQueue(); + private static final Queue privateImageQueueHigh = new LinkedBlockingQueue(); + private static final Queue privateImageQueueLow = new LinkedBlockingQueue(); + private static final Queue publicImageQueueHigh = new LinkedBlockingQueue(); + private static final Queue publicImageQueueLow = new LinkedBlockingQueue(); // we also check all links for a double-check so we don't get the same image more than once in any queue // image links may appear double here even if the pages where the image links are embedded already are checked for double-occurrence: // the same images may be linked from different pages private static final ConcurrentMap doubleCheck = new ConcurrentHashMap(); // (url, time) when the url appeared first - + public static void registerImages(final DigestURI source, final Document document, final boolean privateEntry) { if (document == null) return; if (source == null) return; if (MemoryControl.shortStatus()) clearQueues(); limitQueues(1000); - + final Map images = document.getImages(); for (final ImageEntry image: images.values()) { // do a double-check; attention: this can be time-consuming since this possibly needs a DNS-lookup if (image == null || image.url() == null) continue; if (doubleCheck.containsKey(image.url())) continue; doubleCheck.put(image.url(), System.currentTimeMillis()); - + final String name = image.url().getFile(); boolean good = false; if (image.width() > 120 && @@ -76,7 +77,7 @@ public class ResultImages { name.lastIndexOf(".gif") == -1) { // && ((urlString.lastIndexOf(".jpg") != -1)) || // ((urlString.lastIndexOf(".png") != -1)){ - + good = true; float ratio; if (image.width() > image.height()) { @@ -101,7 +102,7 @@ public class ResultImages { } } } - + public static OriginEntry next(final boolean privateEntryOnly) { OriginEntry e = null; if (privateEntryOnly) { @@ -115,7 +116,7 @@ public class ResultImages { } return e; } - + public static int queueSize(final boolean privateEntryOnly) { int publicSize = 0; if (!privateEntryOnly) { @@ -123,19 +124,19 @@ public class ResultImages { } return privateImageQueueHigh.size() + privateImageQueueLow.size() + publicSize; } - + public static int privateQueueHighSize() { return privateImageQueueHigh.size(); } - + public static int privateQueueLowSize() { return privateImageQueueLow.size(); } - + public static int publicQueueHighSize() { return publicImageQueueHigh.size(); } - + public static int publicQueueLowSize() { return publicImageQueueLow.size(); } @@ -147,14 +148,14 @@ public class ResultImages { publicImageQueueLow.clear(); doubleCheck.clear(); } - + public static void limitQueues(int limit) { while (privateImageQueueHigh.size() > limit) privateImageQueueHigh.poll(); while (privateImageQueueLow.size() > limit) privateImageQueueLow.poll(); while (publicImageQueueHigh.size() > limit) publicImageQueueHigh.poll(); while (publicImageQueueLow.size() > limit) publicImageQueueLow.poll(); } - + public static class OriginEntry { public ImageEntry imageEntry; public MultiProtocolURI baseURL; @@ -163,5 +164,5 @@ public class ResultImages { this.baseURL = baseURL; } } - + } diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 0e6bd96b2..9dba71447 100644 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -31,12 +31,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; import net.yacy.cora.document.ASCII; import net.yacy.cora.document.UTF8; -import net.yacy.cora.services.federated.solr.SolrShardingConnector; import net.yacy.cora.services.federated.solr.SolrConnector; +import net.yacy.cora.services.federated.solr.SolrShardingConnector; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.Index; @@ -76,7 +77,7 @@ public class ZURL implements Iterable { // the class object private Index urlIndex; - private final ConcurrentLinkedQueue stack; + private final Queue stack; private final SolrConnector solrConnector; public ZURL( @@ -105,14 +106,14 @@ public class ZURL implements Iterable { } } //urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true); - this.stack = new ConcurrentLinkedQueue(); + this.stack = new LinkedBlockingQueue(); } public ZURL(final SolrShardingConnector solrConnector) { this.solrConnector = solrConnector; // creates a new ZUR in RAM this.urlIndex = new RowSet(rowdef); - this.stack = new ConcurrentLinkedQueue(); + this.stack = new LinkedBlockingQueue(); } public void clear() throws IOException { @@ -163,6 +164,7 @@ public class ZURL implements Iterable { while (this.stack.size() > maxStackSize) this.stack.poll(); } + @Override public Iterator iterator() { return new EntryIterator(); } @@ -185,14 +187,17 @@ public class ZURL implements Iterable { public EntryIterator() { this.hi = ZURL.this.stack.iterator(); } + @Override public boolean hasNext() { return this.hi.hasNext(); } + @Override public ZURL.Entry next() { return get(this.hi.next()); } + @Override public void remove() { this.hi.remove(); } @@ -326,11 +331,13 @@ public class ZURL implements Iterable { this.error = false; } + @Override public boolean hasNext() { if (this.error) return false; return this.i.hasNext(); } + @Override public Entry next() throws RuntimeException { final Row.Entry e = this.i.next(); if (e == null) return null; @@ -341,6 +348,7 @@ public class ZURL implements Iterable { } } + @Override public void remove() { this.i.remove(); } diff --git a/source/de/anomic/data/URLLicense.java b/source/de/anomic/data/URLLicense.java index 0e329bbd1..b9baf6faf 100644 --- a/source/de/anomic/data/URLLicense.java +++ b/source/de/anomic/data/URLLicense.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -26,9 +26,10 @@ package de.anomic.data; +import java.util.Queue; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.LinkedBlockingQueue; import net.yacy.kelondro.data.meta.DigestURI; @@ -39,47 +40,47 @@ public class URLLicense { // it is used in case of snippet- and preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet private static final int maxQueue = 500; private static final long minCheck = 5000; - + private final Random random; private final ConcurrentHashMap permissions; - private final ConcurrentLinkedQueue aging; + private final Queue aging; private long lastCheck; private final int keylen; - + public URLLicense(final int keylen) { this.permissions = new ConcurrentHashMap(); - this.aging = new ConcurrentLinkedQueue(); + this.aging = new LinkedBlockingQueue(); this.lastCheck = System.currentTimeMillis(); this.random = new Random(System.currentTimeMillis()); this.keylen = keylen; } - + public String aquireLicense(final DigestURI url) { // generate license key - StringBuilder stringBuilder = new StringBuilder(keylen * 2); + StringBuilder stringBuilder = new StringBuilder(this.keylen * 2); if (url == null) return stringBuilder.toString(); - while (stringBuilder.length() < keylen) stringBuilder.append(Integer.toHexString(random.nextInt())); - String license = stringBuilder.substring(0, keylen); + while (stringBuilder.length() < this.keylen) stringBuilder.append(Integer.toHexString(this.random.nextInt())); + String license = stringBuilder.substring(0, this.keylen); // store reference to url with license key - permissions.put(license, url); - aging.add(license); + this.permissions.put(license, url); + this.aging.add(license); if (System.currentTimeMillis() - this.lastCheck > minCheck) { // check aging this.lastCheck = System.currentTimeMillis(); String s; - while (aging.size() > maxQueue) { - s = aging.poll(); - if (s != null) permissions.remove(s); + while (this.aging.size() > maxQueue) { + s = this.aging.poll(); + if (s != null) this.permissions.remove(s); } } // return the license key return license; } - + public DigestURI releaseLicense(final String license) { DigestURI url = null; - url = permissions.remove(license); + url = this.permissions.remove(license); return url; } - + } diff --git a/source/de/anomic/server/serverAccessTracker.java b/source/de/anomic/server/serverAccessTracker.java index 2d0701d95..2ea8ede05 100644 --- a/source/de/anomic/server/serverAccessTracker.java +++ b/source/de/anomic/server/serverAccessTracker.java @@ -24,8 +24,9 @@ import java.util.Collection; import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.LinkedBlockingQueue; public class serverAccessTracker { @@ -34,7 +35,7 @@ public class serverAccessTracker { private final long maxTrackingTime; private final int maxTrackingCount; private final int maxHostCount; - private final ConcurrentHashMap> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries + private final ConcurrentHashMap> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries private long lastCleanup; public static class Track { @@ -56,7 +57,7 @@ public class serverAccessTracker { this.maxTrackingTime = maxTrackingTime; this.maxTrackingCount = maxTrackingCount; this.maxHostCount = maxTrackingHostCount; - this.accessTracker = new ConcurrentHashMap>(); + this.accessTracker = new ConcurrentHashMap>(); } /* @@ -70,8 +71,8 @@ public class serverAccessTracker { } // clear entries which had no entry for the maxTrackingTime time - final Iterator>> i = this.accessTracker.entrySet().iterator(); - ConcurrentLinkedQueue track; + final Iterator>> i = this.accessTracker.entrySet().iterator(); + Queue track; while (i.hasNext()) { track = i.next().getValue(); clearTooOldAccess(track); @@ -111,7 +112,7 @@ public class serverAccessTracker { return c; } - private void clearTooOldAccess(final ConcurrentLinkedQueue access) { + private void clearTooOldAccess(final Queue access) { final long time = System.currentTimeMillis() - this.maxTrackingTime; final Iterator e = access.iterator(); Track l; @@ -130,9 +131,9 @@ public class serverAccessTracker { // learn that a specific host has accessed a specific path if (accessPath == null) accessPath="NULL"; - ConcurrentLinkedQueue track = this.accessTracker.get(host); + Queue track = this.accessTracker.get(host); if (track == null) { - track = new ConcurrentLinkedQueue(); + track = new LinkedBlockingQueue(); track.add(new Track(System.currentTimeMillis(), accessPath)); // add to tracker this.accessTracker.put(host, track); @@ -145,7 +146,7 @@ public class serverAccessTracker { public Collection accessTrack(final String host) { // returns mapping from Long(accesstime) to path - final ConcurrentLinkedQueue access = this.accessTracker.get(host); + final Queue access = this.accessTracker.get(host); if (access == null) return null; // clear too old entries clearTooOldAccess(access); @@ -157,7 +158,7 @@ public class serverAccessTracker { public Iterator accessHosts() { // returns an iterator of hosts in tracker (String) - final Map> accessTrackerClone = new ConcurrentHashMap>(); + final Map> accessTrackerClone = new ConcurrentHashMap>(); accessTrackerClone.putAll(this.accessTracker); return accessTrackerClone.keySet().iterator(); } diff --git a/source/net/yacy/kelondro/data/image/ImageReferenceVars.java b/source/net/yacy/kelondro/data/image/ImageReferenceVars.java index 7683c7bcc..5a75be5a8 100644 --- a/source/net/yacy/kelondro/data/image/ImageReferenceVars.java +++ b/source/net/yacy/kelondro/data/image/ImageReferenceVars.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -27,7 +27,8 @@ package net.yacy.kelondro.data.image; import java.util.Collection; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.order.Bitfield; @@ -43,8 +44,8 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere * object for termination of concurrent blocking queue processing */ public static final ImageReferenceVars poison = new ImageReferenceVars(); - - + + public Bitfield flags; public long lastModified; public byte[] urlHash; @@ -54,9 +55,9 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere posinphrase, posofphrase, urlcomps, urllength, virtualAge, wordsintext, wordsintitle; - private final ConcurrentLinkedQueue positions; + private final Queue positions; public double termFrequency; - + public ImageReferenceVars( final byte[] urlHash, final int urlLength, // byte-length of complete URL @@ -65,7 +66,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere final int hitcount, // how often appears this word in the text final int wordcount, // total number of words final int phrasecount, // total number of phrases - final ConcurrentLinkedQueue ps, // positions of words that are joined into the reference + final Queue ps, // positions of words that are joined into the reference final int posinphrase, // position of word in its phrase final int posofphrase, // number of the phrase where word appears final long lastmodified, // last-modified time of the document where word appears @@ -90,7 +91,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere this.llocal = outlinksSame; this.lother = outlinksOther; this.phrasesintext = phrasecount; - this.positions = new ConcurrentLinkedQueue(); + this.positions = new LinkedBlockingQueue(); for (Integer i: ps) this.positions.add(i); this.posinphrase = posinphrase; this.posofphrase = posofphrase; @@ -101,7 +102,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere this.wordsintitle = titleLength; this.termFrequency = termfrequency; } - + public ImageReferenceVars(final ImageReference e) { this.flags = e.flags(); //this.freshUntil = e.freshUntil(); @@ -113,7 +114,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere this.llocal = e.llocal(); this.lother = e.lother(); this.phrasesintext = e.phrasesintext(); - this.positions = new ConcurrentLinkedQueue(); + this.positions = new LinkedBlockingQueue(); for (Integer i: e.positions()) this.positions.add(i); this.posinphrase = e.posinphrase(); this.posofphrase = e.posofphrase(); @@ -124,7 +125,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere this.wordsintitle = e.wordsintitle(); this.termFrequency = e.termFrequency(); } - + /** * initializer for special poison object */ @@ -148,7 +149,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere this.wordsintitle = 0; this.termFrequency = 0.0; } - + @Override public ImageReferenceVars clone() { final ImageReferenceVars c = new ImageReferenceVars( @@ -172,7 +173,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere this.termFrequency); return c; } - + public void join(final ImageReferenceVars v) { // combine the distance this.positions.addAll(v.positions); @@ -184,118 +185,139 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere this.termFrequency = this.termFrequency + v.termFrequency; } + @Override public Bitfield flags() { - return flags; + return this.flags; } /* public long freshUntil() { return freshUntil; } */ + @Override public String getLanguage() { - return language; + return this.language; } + @Override public char getType() { - return type; + return this.type; } + @Override public int hitcount() { - return hitcount; + return this.hitcount; } + @Override public boolean isOlder(final Reference other) { assert false; // should not be used return false; } + @Override public long lastModified() { - return lastModified; + return this.lastModified; } + @Override public int llocal() { - return llocal; + return this.llocal; } + @Override public int lother() { - return lother; + return this.lother; } + @Override public int phrasesintext() { - return phrasesintext; + return this.phrasesintext; } + @Override public int posinphrase() { - return posinphrase; + return this.posinphrase; } + @Override public Collection positions() { return this.positions; } + @Override public int posofphrase() { - return posofphrase; + return this.posofphrase; } - + public ImageReferenceRow toRowEntry() { return new ImageReferenceRow( - urlHash, - urllength, // byte-length of complete URL - urlcomps, // number of path components - wordsintitle, // length of description/length (longer are better?) - hitcount, // how often appears this word in the text - wordsintext, // total number of words - phrasesintext, // total number of phrases - positions.iterator().next(), // position of word in all words - posinphrase, // position of word in its phrase - posofphrase, // number of the phrase where word appears - lastModified, // last-modified time of the document where word appears + this.urlHash, + this.urllength, // byte-length of complete URL + this.urlcomps, // number of path components + this.wordsintitle, // length of description/length (longer are better?) + this.hitcount, // how often appears this word in the text + this.wordsintext, // total number of words + this.phrasesintext, // total number of phrases + this.positions.iterator().next(), // position of word in all words + this.posinphrase, // position of word in its phrase + this.posofphrase, // number of the phrase where word appears + this.lastModified, // last-modified time of the document where word appears System.currentTimeMillis(), // update time; - language, // (guessed) language of document - type, // type of document - llocal, // outlinks to same domain - lother, // outlinks to other domain - flags // attributes to the url and to the word according the url + this.language, // (guessed) language of document + this.type, // type of document + this.llocal, // outlinks to same domain + this.lother, // outlinks to other domain + this.flags // attributes to the url and to the word according the url ); } - + + @Override public Entry toKelondroEntry() { return toRowEntry().toKelondroEntry(); } + @Override public String toPropertyForm() { return toRowEntry().toPropertyForm(); } + @Override public byte[] urlhash() { - return urlHash; + return this.urlHash; } + @Override public int urlcomps() { - return urlcomps; + return this.urlcomps; } + @Override public int urllength() { - return urllength; + return this.urllength; } + @Override public int virtualAge() { - return virtualAge; + return this.virtualAge; } + @Override public int wordsintext() { - return wordsintext; + return this.wordsintext; } + @Override public int wordsintitle() { - return wordsintitle; + return this.wordsintitle; } + @Override public double termFrequency() { if (this.termFrequency == 0.0) this.termFrequency = (((double) this.hitcount()) / ((double) (this.wordsintext() + this.wordsintitle() + 1))); return this.termFrequency; } - + public final void min(final ImageReferenceVars other) { if (other == null) return; int v; @@ -317,7 +339,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere if (this.wordsintitle > (v = other.wordsintitle)) this.wordsintitle = v; if (this.termFrequency > (d = other.termFrequency)) this.termFrequency = d; } - + public final void max(final ImageReferenceVars other) { if (other == null) return; int v; @@ -340,11 +362,12 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d; } + @Override public void join(final Reference r) { // joins two entries into one entry - + // combine the distance - ImageReference oe = (ImageReference) r; + ImageReference oe = (ImageReference) r; for (Integer i: r.positions()) this.positions.add(i); this.posinphrase = (this.posofphrase == oe.posofphrase()) ? Math.min(this.posinphrase, oe.posinphrase()) : 0; this.posofphrase = Math.min(this.posofphrase, oe.posofphrase()); @@ -362,5 +385,5 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere public void addPosition(int position) { this.positions.add(position); } - + } diff --git a/source/net/yacy/kelondro/data/word/WordReferenceVars.java b/source/net/yacy/kelondro/data/word/WordReferenceVars.java index 86730abe8..4f1377285 100644 --- a/source/net/yacy/kelondro/data/word/WordReferenceVars.java +++ b/source/net/yacy/kelondro/data/word/WordReferenceVars.java @@ -28,8 +28,8 @@ package net.yacy.kelondro.data.word; import java.util.Collection; import java.util.Comparator; +import java.util.Queue; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; @@ -66,7 +66,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc urlcomps, urllength, wordsintext, wordsintitle; private int virtualAge; - private final ConcurrentLinkedQueue positions; + private final Queue positions; public double termFrequency; public WordReferenceVars( @@ -77,7 +77,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc final int hitcount, // how often appears this word in the text final int wordcount, // total number of words final int phrasecount, // total number of phrases - final ConcurrentLinkedQueue ps, // positions of words that are joined into the reference + final Queue ps, // positions of words that are joined into the reference final int posinphrase, // position of word in its phrase final int posofphrase, // number of the phrase where word appears final long lastmodified, // last-modified time of the document where word appears @@ -101,7 +101,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc this.llocal = outlinksSame; this.lother = outlinksOther; this.phrasesintext = phrasecount; - this.positions = new ConcurrentLinkedQueue(); + this.positions = new LinkedBlockingQueue(); if (ps.size() > 0) for (final Integer i: ps) this.positions.add(i); this.posinphrase = posinphrase; this.posofphrase = posofphrase; @@ -124,7 +124,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc this.llocal = e.llocal(); this.lother = e.lother(); this.phrasesintext = e.phrasesintext(); - this.positions = new ConcurrentLinkedQueue(); + this.positions = new LinkedBlockingQueue(); if (e.positions().size() > 0) for (final Integer i: e.positions()) this.positions.add(i); this.posinphrase = e.posinphrase(); this.posofphrase = e.posofphrase(); @@ -195,46 +195,57 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc this.termFrequency = this.termFrequency + v.termFrequency; } + @Override public Bitfield flags() { return this.flags; } + @Override public byte[] getLanguage() { return this.language; } + @Override public char getType() { return this.type; } + @Override public int hitcount() { return this.hitcount; } + @Override public long lastModified() { return this.lastModified; } + @Override public int llocal() { return this.llocal; } + @Override public int lother() { return this.lother; } + @Override public int phrasesintext() { return this.phrasesintext; } + @Override public int posinphrase() { return this.posinphrase; } + @Override public Collection positions() { return this.positions; } + @Override public int posofphrase() { return this.posofphrase; } @@ -261,14 +272,17 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc ); } + @Override public Entry toKelondroEntry() { return toRowEntry().toKelondroEntry(); } + @Override public String toPropertyForm() { return toRowEntry().toPropertyForm(); } + @Override public byte[] urlhash() { return this.urlHash; } @@ -279,28 +293,34 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc return this.hostHash; } + @Override public int urlcomps() { return this.urlcomps; } + @Override public int urllength() { return this.urllength; } + @Override public int virtualAge() { if (this.virtualAge > 0) return this.virtualAge; this.virtualAge = MicroDate.microDateDays(this.lastModified); return this.virtualAge; } + @Override public int wordsintext() { return this.wordsintext; } + @Override public int wordsintitle() { return this.wordsintitle; } + @Override public double termFrequency() { if (this.termFrequency == 0.0) this.termFrequency = (((double) hitcount()) / ((double) (wordsintext() + wordsintitle() + 1))); return this.termFrequency; @@ -350,6 +370,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d; } + @Override public void join(final Reference r) { // joins two entries into one entry @@ -378,10 +399,12 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc return ByteArray.hashCode(this.urlHash); } + @Override public int compareTo(final WordReferenceVars o) { return Base64Order.enhancedCoder.compare(this.urlHash, o.urlhash()); } + @Override public int compare(final WordReferenceVars o1, final WordReferenceVars o2) { return o1.compareTo(o2); } diff --git a/source/net/yacy/kelondro/util/EventTracker.java b/source/net/yacy/kelondro/util/EventTracker.java index 2207511d9..af338b8bd 100644 --- a/source/net/yacy/kelondro/util/EventTracker.java +++ b/source/net/yacy/kelondro/util/EventTracker.java @@ -28,8 +28,9 @@ package net.yacy.kelondro.util; import java.util.Iterator; import java.util.Map; +import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.LinkedBlockingQueue; import net.yacy.peers.graphics.ProfilingGraph; @@ -49,7 +50,7 @@ public class EventTracker { SEARCH; } - private final static Map> historyMaps = new ConcurrentHashMap>(); + private final static Map> historyMaps = new ConcurrentHashMap>(); private final static Map eventAccess = new ConcurrentHashMap(); // value: last time when this was accessed public final static void delete(final EClass eventName) { @@ -73,11 +74,11 @@ public class EventTracker { } // get event history container - ConcurrentLinkedQueue history = historyMaps.get(eventName); + Queue history = historyMaps.get(eventName); // create history if (history == null) { - history = new ConcurrentLinkedQueue(); + history = new LinkedBlockingQueue(); // update entry history.offer(new Event(eventPayload)); @@ -109,7 +110,7 @@ public class EventTracker { } public final static Iterator getHistory(final EClass eventName) { - final ConcurrentLinkedQueue list = historyMaps.get(eventName); + final Queue list = historyMaps.get(eventName); if (list == null) return null; return list.iterator(); }