From 783cf6fbc7a0c33822df1e15506f2d071796ed24 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 27 Jan 2015 16:53:09 +0100 Subject: [PATCH] the LinkedBlockingQueue is much faster than the ArrayBlockingQueue (strange but this is the result of a test: ArrayBlockingQueue: 39461 lines / second; LinkedBlockingQueue: 60774 lines / second) --- .../yacy/cora/language/synonyms/SynonymLibrary.java | 2 +- source/net/yacy/cora/lod/vocabulary/Tagging.java | 13 +++++++------ source/net/yacy/cora/storage/Files.java | 5 +++-- .../yacy/crawler/retrieval/URLRewriterLibrary.java | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/net/yacy/cora/language/synonyms/SynonymLibrary.java b/source/net/yacy/cora/language/synonyms/SynonymLibrary.java index e0657cad3..4992a2571 100644 --- a/source/net/yacy/cora/language/synonyms/SynonymLibrary.java +++ b/source/net/yacy/cora/language/synonyms/SynonymLibrary.java @@ -53,7 +53,7 @@ public class SynonymLibrary { File ff = new File(path, f); String line; try { - BlockingQueue list = Files.concurentLineReader(ff, 1000); + BlockingQueue list = Files.concurentLineReader(ff); while ((line = list.take()) != Files.POISON_LINE) { line = line.trim(); if (line.length() == 0 || line.charAt(0) == '#') continue; diff --git a/source/net/yacy/cora/lod/vocabulary/Tagging.java b/source/net/yacy/cora/lod/vocabulary/Tagging.java index 49fe5e700..1227e35bc 100644 --- a/source/net/yacy/cora/lod/vocabulary/Tagging.java +++ b/source/net/yacy/cora/lod/vocabulary/Tagging.java @@ -207,7 +207,7 @@ public class Tagging { ConcurrentLog.info("Tagging", "Started Vocabulary Initialization for " + this.propFile); long start = System.currentTimeMillis(); long count = 0; - BlockingQueue list = Files.concurentLineReader(this.propFile, 1000); + BlockingQueue list = Files.concurentLineReader(this.propFile); String term, v; String[] tags; int p; @@ -268,7 +268,8 @@ public class Tagging { } } catch (final InterruptedException e) { } - ConcurrentLog.info("Tagging", "Finished Vocabulary Initialization for " + this.propFile + "; " + count + " lines; " + (System.currentTimeMillis() - start) + " milliseconds"); + long time = Math.max(1, System.currentTimeMillis() - start); + ConcurrentLog.info("Tagging", "Finished Vocabulary Initialization for " + this.propFile + "; " + count + " lines; " + time + " milliseconds; " + (1000L * count / time) + " lines / second"); } public boolean isFacet() { @@ -292,7 +293,7 @@ public class Tagging { if (this.propFile == null) return; File tmp = tmpFile(); BufferedWriter w = new BufferedWriter(new FileWriter(tmp)); - BlockingQueue list = Files.concurentLineReader(this.propFile, 1000); + BlockingQueue list = Files.concurentLineReader(this.propFile); if (this.namespace != null && !this.namespace.equals(DEFAULT_NAMESPACE)) w.write("#namespace:" + this.namespace + "\n"); if (this.objectspace != null && this.objectspace.length() > 0) w.write("#objectspace:" + this.objectspace + "\n"); String line; @@ -325,7 +326,7 @@ public class Tagging { if (this.propFile == null) return; File tmp = tmpFile(); BufferedWriter w = new BufferedWriter(new FileWriter(tmp)); - BlockingQueue list = Files.concurentLineReader(this.propFile, 1000); + BlockingQueue list = Files.concurentLineReader(this.propFile); if (this.namespace != null && !this.namespace.equals(DEFAULT_NAMESPACE)) w.write("#namespace:" + this.namespace + "\n"); if (this.objectspace != null && this.objectspace.length() > 0) w.write("#objectspace:" + this.objectspace + "\n"); String line; @@ -366,7 +367,7 @@ public class Tagging { this.objectspace = os; File tmp = tmpFile(); BufferedWriter w = new BufferedWriter(new FileWriter(tmp)); - BlockingQueue list = Files.concurentLineReader(this.propFile, 1000); + BlockingQueue list = Files.concurentLineReader(this.propFile); if (this.namespace != null && !this.namespace.equals(DEFAULT_NAMESPACE)) w.write("#namespace:" + this.namespace + "\n"); if (this.objectspace != null && this.objectspace.length() > 0) w.write("#objectspace:" + this.objectspace + "\n"); String line; @@ -429,7 +430,7 @@ public class Tagging { Map map = new LinkedHashMap(); BlockingQueue list; try { - list=Files.concurentLineReader(this.propFile, 1000); + list=Files.concurentLineReader(this.propFile); } catch (final IOException e1) { return map; } diff --git a/source/net/yacy/cora/storage/Files.java b/source/net/yacy/cora/storage/Files.java index 9903d6066..441bfab98 100644 --- a/source/net/yacy/cora/storage/Files.java +++ b/source/net/yacy/cora/storage/Files.java @@ -36,6 +36,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; import java.util.zip.GZIPInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; @@ -75,8 +76,8 @@ public class Files { * @throws IOException */ public final static String POISON_LINE = "__@POISON__"; - public static BlockingQueue concurentLineReader(final File f, final int maxQueueSize) throws IOException { - final BlockingQueue q = new ArrayBlockingQueue(maxQueueSize); + public static BlockingQueue concurentLineReader(final File f) throws IOException { + final BlockingQueue q = new LinkedBlockingQueue(); final InputStream is = read(f); final BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8")); Thread t = new Thread() { diff --git a/source/net/yacy/crawler/retrieval/URLRewriterLibrary.java b/source/net/yacy/crawler/retrieval/URLRewriterLibrary.java index 10102d4fb..70e0fd433 100644 --- a/source/net/yacy/crawler/retrieval/URLRewriterLibrary.java +++ b/source/net/yacy/crawler/retrieval/URLRewriterLibrary.java @@ -49,7 +49,7 @@ public class URLRewriterLibrary { for (final String f: files) { File ff = new File(this.rewritingPath, f); try { - BlockingQueue list = Files.concurentLineReader(ff, 1000); + BlockingQueue list = Files.concurentLineReader(ff); String line; while ((line = list.take()) != Files.POISON_LINE) { line = line.trim();