From 1a9cfd8718d08d95cf4c7ddd1de3ce4cd700619c Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 28 Aug 2009 13:28:11 +0000 Subject: [PATCH] some performance hacks (CPU only, not IO) this will cause better computation speed for single- and multi-core; there are enhancements that will speed up old and slow machines as well as multi-core CPUs. Indexing of surrogates has been speed up from 4000 PPM to over 20000 PPM on a simple dual core office computer. Since the enhancements are mostly in core routines, the hack should also speed up search performance. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6276 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/PerformanceSearch_p.java | 29 +++++----- source/de/anomic/document/Condenser.java | 8 +-- source/de/anomic/document/Word.java | 2 +- .../kelondro/index/ObjectIndexCache.java | 2 +- .../anomic/kelondro/index/RowCollection.java | 36 +++++++++--- .../de/anomic/kelondro/table/SplitTable.java | 57 +++++++++++++------ source/de/anomic/kelondro/table/Table.java | 6 +- source/de/anomic/kelondro/text/IndexCell.java | 6 +- .../text/ReferenceContainerCache.java | 56 ++++++++++++------ .../anomic/kelondro/text/ReferenceOrder.java | 11 ++-- source/de/anomic/kelondro/util/SetTools.java | 28 ++++----- source/de/anomic/search/Switchboard.java | 2 + source/de/anomic/server/serverProfiling.java | 30 +++++----- source/de/anomic/ymage/ProfilingGraph.java | 37 ++++++------ 14 files changed, 191 insertions(+), 119 deletions(-) diff --git a/htroot/PerformanceSearch_p.java b/htroot/PerformanceSearch_p.java index 0244161a6..f5cbecb84 100644 --- a/htroot/PerformanceSearch_p.java +++ b/htroot/PerformanceSearch_p.java @@ -24,6 +24,7 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import java.util.ArrayList; import java.util.Date; import java.util.Iterator; @@ -39,25 +40,27 @@ public class PerformanceSearch_p { // return variable that accumulates replacements final serverObjects prop = new serverObjects(); - final Iterator events = serverProfiling.history("SEARCH"); + final ArrayList events = serverProfiling.history("SEARCH"); int c = 0; serverProfiling.Event event; ProfilingGraph.searchEvent search; long lastt = 0; - while (events.hasNext()) { - event = events.next(); - search = (ProfilingGraph.searchEvent) event.payload; - prop.put("table_" + c + "_query", search.queryID); - prop.put("table_" + c + "_event", search.processName); - prop.putNum("table_" + c + "_count", search.resultCount); - prop.putNum("table_" + c + "_delta", event.time - lastt); - prop.put("table_" + c + "_time", (new Date(event.time)).toString()); - prop.putNum("table_" + c + "_duration", search.duration); - c++; - lastt = event.time; + if (events != null) synchronized (events) { + Iterator i = events.iterator(); + while (i.hasNext()) { + event = i.next(); + search = (ProfilingGraph.searchEvent) event.payload; + prop.put("table_" + c + "_query", search.queryID); + prop.put("table_" + c + "_event", search.processName); + prop.putNum("table_" + c + "_count", search.resultCount); + prop.putNum("table_" + c + "_delta", event.time - lastt); + prop.put("table_" + c + "_time", (new Date(event.time)).toString()); + prop.putNum("table_" + c + "_duration", search.duration); + c++; + lastt = event.time; + } } prop.put("table", c); - return prop; } } diff --git a/source/de/anomic/document/Condenser.java b/source/de/anomic/document/Condenser.java index b823960e3..818b8a016 100644 --- a/source/de/anomic/document/Condenser.java +++ b/source/de/anomic/document/Condenser.java @@ -87,7 +87,7 @@ public final class Condenser { private final static int numlength = 5; //private Properties analysis; - private TreeMap words; // a string (the words) to (indexWord) - relation + private Map words; // a string (the words) to (indexWord) - relation private final int wordminsize; private final int wordcut; @@ -108,7 +108,7 @@ public final class Condenser { // added media words are flagged with the appropriate media flag this.wordminsize = 3; this.wordcut = 2; - this.words = new TreeMap(); + this.words = new HashMap(); this.RESULT_FLAGS = new Bitfield(4); // construct flag set for document @@ -260,7 +260,7 @@ public final class Condenser { // subtracts the given stopwords from the word list // the word list shrinkes. This returns the number of shrinked words final int oldsize = words.size(); - words = SetTools.excludeConstructive(words, stopwords); + SetTools.excludeDestructive(words, stopwords); return oldsize - words.size(); } @@ -738,7 +738,7 @@ public final class Condenser { } static StringBuilder readSentence(final Reader reader, final boolean pre) throws IOException { - final StringBuilder s = new StringBuilder(40); + final StringBuilder s = new StringBuilder(80); int nextChar; char c, lc = ' '; // starting with ' ' as last character prevents that the result string starts with a ' ' diff --git a/source/de/anomic/document/Word.java b/source/de/anomic/document/Word.java index 7e72d1c66..1e549b324 100644 --- a/source/de/anomic/document/Word.java +++ b/source/de/anomic/document/Word.java @@ -41,7 +41,7 @@ import de.anomic.yacy.yacySeedDB; public class Word { - public static final int hashCacheSize = (int) (MemoryControl.available() / 10000L); + public static final int hashCacheSize = Math.max(2000, Math.min(100000, (int) (MemoryControl.available() / 20000L))); private static final SimpleARC hashCache = new SimpleARC(hashCacheSize); // object carries statistics for words and sentences diff --git a/source/de/anomic/kelondro/index/ObjectIndexCache.java b/source/de/anomic/kelondro/index/ObjectIndexCache.java index 7139a46f8..130ca531c 100644 --- a/source/de/anomic/kelondro/index/ObjectIndexCache.java +++ b/source/de/anomic/kelondro/index/ObjectIndexCache.java @@ -54,7 +54,7 @@ public class ObjectIndexCache implements ObjectIndex { public synchronized void reset(final int initialspace) { this.index0 = null; // first flush RAM to make room - this.index0 = new RowSet(rowdef, initialspace); + this.index0 = new RowSet(rowdef, initialspace); this.index1 = null; // to show that this is the initialization phase } diff --git a/source/de/anomic/kelondro/index/RowCollection.java b/source/de/anomic/kelondro/index/RowCollection.java index 06556faec..c481dfe19 100644 --- a/source/de/anomic/kelondro/index/RowCollection.java +++ b/source/de/anomic/kelondro/index/RowCollection.java @@ -49,7 +49,7 @@ import de.anomic.yacy.logging.Log; public class RowCollection implements Iterable { - public static final double growfactor = 1.4; + public static final long growfactor100 = 140L; private static final int isortlimit = 20; static final Integer dummy = 0; @@ -142,8 +142,10 @@ public class RowCollection implements Iterable { return (int) (time / day) - 10957; } + private static Column exportColumn0, exportColumn1, exportColumn2, exportColumn3, exportColumn4; + private static Row exportRow(final int chunkcachelength) { - // find out the size of this collection + /* return new Row( "int size-4 {b256}," + "short lastread-2 {b256}," + // as daysSince2000 @@ -153,6 +155,23 @@ public class RowCollection implements Iterable { "byte[] collection-" + chunkcachelength, NaturalOrder.naturalOrder ); + */ + + if (exportColumn0 == null) exportColumn0 = new Column("int size-4 {b256}"); + if (exportColumn1 == null) exportColumn1 = new Column("short lastread-2 {b256}"); + if (exportColumn2 == null) exportColumn2 = new Column("short lastwrote-2 {b256}"); + if (exportColumn3 == null) exportColumn3 = new Column("byte[] orderkey-2"); + if (exportColumn4 == null) exportColumn4 = new Column("int orderbound-4 {b256}"); + /* + * because of a strange bug these objects cannot be initialized as normal + * static final. If I try that, they are not initialized and are assigned null. why? + */ + return new Row(new Column[]{ + exportColumn0, exportColumn1, exportColumn2, exportColumn3, exportColumn4, + new Column("byte[] collection-" + chunkcachelength) + }, + NaturalOrder.naturalOrder + ); } public static final int exportOverheadSize = 14; @@ -183,9 +202,12 @@ public class RowCollection implements Iterable { } protected final void ensureSize(final int elements) { - final int needed = elements * rowdef.objectsize; + assert elements > 0 : "elements = " + elements; + final long needed = elements * rowdef.objectsize; if (chunkcache.length >= needed) return; - byte[] newChunkcache = new byte[(int) (needed * growfactor)]; // increase space + assert needed > 0 : "needed = " + needed; + assert needed * growfactor100 / 100L > 0 : "elements = " + elements + ", new = " + (needed * growfactor100 / 100L); + byte[] newChunkcache = new byte[(int) (needed * growfactor100 / 100L)]; // increase space System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length); chunkcache = newChunkcache; } @@ -193,17 +215,17 @@ public class RowCollection implements Iterable { /** * compute the needed memory in case of a cache extension. That is, if the cache is full and must * be copied into a new cache which is larger. In such a case the Collection needs more than the double size - * than is necessary to store the data. This method coputes the extra memory that is needed to perform this task. + * than is necessary to store the data. This method computes the extra memory that is needed to perform this task. * @return */ public final long memoryNeededForGrow() { - return (long) ((((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor); + return (long) ((((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor100 / 100L); } public synchronized void trim(final boolean plusGrowFactor) { if (chunkcache.length == 0) return; int needed = chunkcount * rowdef.objectsize; - if (plusGrowFactor) needed = (int) (needed * growfactor); + if (plusGrowFactor) needed = (int) (((long) needed) * growfactor100 / 100L); if (needed >= chunkcache.length) return; // in case that the growfactor causes that the cache would // grow instead of shrink, simply ignore the growfactor diff --git a/source/de/anomic/kelondro/table/SplitTable.java b/source/de/anomic/kelondro/table/SplitTable.java index eaa7f68e2..2e8b95f97 100644 --- a/source/de/anomic/kelondro/table/SplitTable.java +++ b/source/de/anomic/kelondro/table/SplitTable.java @@ -114,14 +114,6 @@ public class SplitTable implements ObjectIndex { public void init() { current = null; - // init the thread pool for the keeperOf executor service - this.executor = new ThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() + 1, - Runtime.getRuntime().availableProcessors() + 1, 10, - TimeUnit.SECONDS, - new LinkedBlockingQueue(), - new NamePrefixThreadFactory(prefix)); - // initialized tables map this.tables = new HashMap(); if (!(path.exists())) path.mkdirs(); @@ -199,6 +191,16 @@ public class SplitTable implements ObjectIndex { tables.put(maxf, table); } } + + // init the thread pool for the keeperOf executor service + this.executor = new ThreadPoolExecutor( + Math.max(tables.size(), Runtime.getRuntime().availableProcessors()) + 1, + Math.max(tables.size(), Runtime.getRuntime().availableProcessors()) + 1, 10, + TimeUnit.SECONDS, + new LinkedBlockingQueue(), + new NamePrefixThreadFactory(prefix)); + + } public void clear() throws IOException { @@ -304,20 +306,46 @@ public class SplitTable implements ObjectIndex { keeper.put(row); } + private static final class ReadyCheck { + private boolean r; + public ReadyCheck() { + this.r = false; + } + public void isReady() { + this.r = true; + } + public boolean check() { + return this.r; + } + } + public synchronized ObjectIndex keeperOf(final byte[] key) { + + if (tables.size() < 2) { + // no concurrency if not needed + for (final ObjectIndex table : tables.values()) { + if (table.has(key)) return table; + } + return null; + } + // because the index is stored only in one table, // and the index is completely in RAM, a concurrency will create // not concurrent File accesses - //long start = System.currentTimeMillis(); // start a concurrent query to database tables final CompletionService cs = new ExecutorCompletionService(executor); int accepted = 0; + final ReadyCheck ready = new ReadyCheck(); for (final ObjectIndex table : tables.values()) { + if (ready.check()) break; // found already a table try { cs.submit(new Callable() { public ObjectIndex call() { - if (table.has(key)) return table; + if (table.has(key)) { + ready.isReady(); + return table; + } return dummyIndex; } }); @@ -333,15 +361,11 @@ public class SplitTable implements ObjectIndex { try { for (int i = 0; i < accepted; i++) { final Future f = cs.take(); - //hash(System.out.println("**********accepted = " + accepted + ", i =" + i); if (f == null) continue; final ObjectIndex index = f.get(); - if (index != dummyIndex) { - //System.out.println("*DEBUG SplitTable success.time = " + (System.currentTimeMillis() - start) + " ms"); - return index; - } + if (index == dummyIndex) continue; + return index; } - //System.out.println("*DEBUG SplitTable fail.time = " + (System.currentTimeMillis() - start) + " ms"); return null; } catch (final InterruptedException e) { Thread.currentThread().interrupt(); @@ -349,7 +373,6 @@ public class SplitTable implements ObjectIndex { e.printStackTrace(); throw new RuntimeException(e.getCause()); } - //System.out.println("*DEBUG SplitTable fail.time = " + (System.currentTimeMillis() - start) + " ms"); return null; } diff --git a/source/de/anomic/kelondro/table/Table.java b/source/de/anomic/kelondro/table/Table.java index 50ec67cb9..05b7c30e6 100644 --- a/source/de/anomic/kelondro/table/Table.java +++ b/source/de/anomic/kelondro/table/Table.java @@ -266,9 +266,9 @@ public class Table implements ObjectIndex { final HashMap map = new HashMap(); map.put("tableSize", Integer.toString(index.size())); map.put("tableKeyChunkSize", Integer.toString(index.row().objectsize)); - map.put("tableKeyMem", Integer.toString((int) (index.row().objectsize * index.size() * RowCollection.growfactor))); + map.put("tableKeyMem", Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L))); map.put("tableValueChunkSize", (table == null) ? "0" : Integer.toString(table.row().objectsize)); - map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (table.row().objectsize * table.size() * RowCollection.growfactor))); + map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L))); return map; } @@ -277,7 +277,7 @@ public class Table implements ObjectIndex { } public static int staticRAMIndexNeed(final File f, final Row rowdef) { - return (int) ((rowdef.primaryKeyLength + 4) * tableSize(f, rowdef.objectsize) * RowCollection.growfactor); + return (int) (((long)(rowdef.primaryKeyLength + 4)) * ((long) tableSize(f, rowdef.objectsize)) * RowCollection.growfactor100 / 100L); } public synchronized void addUnique(final Entry row) throws IOException { diff --git a/source/de/anomic/kelondro/text/IndexCell.java b/source/de/anomic/kelondro/text/IndexCell.java index 4c654784b..5bc22e6f1 100644 --- a/source/de/anomic/kelondro/text/IndexCell.java +++ b/source/de/anomic/kelondro/text/IndexCell.java @@ -52,7 +52,7 @@ import de.anomic.server.serverProfiling; public final class IndexCell extends AbstractBufferedIndex implements BufferedIndex { - private static final long cleanupCycle = 10000; + private static final long cleanupCycle = 60000; // class variables private final ReferenceContainerArray array; @@ -101,7 +101,7 @@ public final class IndexCell extends AbstractBu */ public void add(ReferenceContainer newEntries) throws IOException { this.ram.add(newEntries); - if (this.ram.size() % 100 == 0) { + if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) { serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true); cleanCache(); } @@ -109,7 +109,7 @@ public final class IndexCell extends AbstractBu public void add(byte[] termHash, ReferenceType entry) throws IOException { this.ram.add(termHash, entry); - if (this.ram.size() % 100 == 0) { + if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) { serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true); cleanCache(); } diff --git a/source/de/anomic/kelondro/text/ReferenceContainerCache.java b/source/de/anomic/kelondro/text/ReferenceContainerCache.java index 5843b4e97..44d06f3b3 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerCache.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerCache.java @@ -28,11 +28,11 @@ package de.anomic.kelondro.text; import java.io.File; import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; import java.util.Iterator; import java.util.Map; import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import de.anomic.kelondro.blob.HeapReader; @@ -48,8 +48,15 @@ import de.anomic.yacy.logging.Log; public final class ReferenceContainerCache extends AbstractIndex implements Index, IndexReader, Iterable> { + public class ContainerOrder implements Comparator> { + public int compare(ReferenceContainer arg0, ReferenceContainer arg1) { + return termOrder.compare(arg0.getTermHash(), arg1.getTermHash()); + } + } + private final Row payloadrow; - private final ByteOrder termOrder; + protected final ByteOrder termOrder; + private final ContainerOrder containerOrder; protected Map> cache; /** @@ -63,6 +70,7 @@ public final class ReferenceContainerCache exte super(factory); this.payloadrow = payloadrow; this.termOrder = termOrder; + this.containerOrder = new ContainerOrder(); this.cache = null; } @@ -102,17 +110,15 @@ public final class ReferenceContainerCache exte final long startTime = System.currentTimeMillis(); // sort the map - SortedMap> cachecopy = sortedClone(); + ReferenceContainer[] cachecopy = sortedClone(); // write wCache long wordcount = 0, urlcount = 0; byte[] wordHash = null, lwh; - ReferenceContainer container; - for (final Map.Entry> entry: cachecopy.entrySet()) { + for (final ReferenceContainer container: cachecopy) { // get entries lwh = wordHash; - wordHash = entry.getKey(); - container = entry.getValue(); + wordHash = container.getTermHash(); // check consistency: entries must be ordered assert (lwh == null || this.ordering().compare(wordHash, lwh) > 0); @@ -139,6 +145,19 @@ public final class ReferenceContainerCache exte } } + @SuppressWarnings("unchecked") + public ReferenceContainer[] sortedClone() { + ReferenceContainer[] cachecopy = new ReferenceContainer[cache.size()]; + synchronized (cache) { + int p = 0; + for (final Map.Entry> entry: cache.entrySet()) { + cachecopy[p++] = entry.getValue(); + } + } + Arrays.sort(cachecopy, this.containerOrder); + return cachecopy; + } + /* public SortedMap> sortedClone() { SortedMap> cachecopy; synchronized (cache) { @@ -149,7 +168,7 @@ public final class ReferenceContainerCache exte } return cachecopy; } - + */ public int size() { return (this.cache == null) ? 0 : this.cache.size(); } @@ -252,14 +271,15 @@ public final class ReferenceContainerCache exte // plus the mentioned features private final boolean rot; - private Iterator> iterator; + private ReferenceContainer[] cachecopy; + private int p; private byte[] latestTermHash; public heapCacheIterator(byte[] startWordHash, final boolean rot) { this.rot = rot; if (startWordHash != null && startWordHash.length == 0) startWordHash = null; - SortedMap> cachecopy = sortedClone(); - this.iterator = (startWordHash == null) ? cachecopy.values().iterator() : cachecopy.tailMap(startWordHash).values().iterator(); + this.cachecopy = sortedClone(); + this.p = 0; this.latestTermHash = null; // The collection's iterator will return the values in the order that their corresponding keys appear in the tree. } @@ -270,12 +290,12 @@ public final class ReferenceContainerCache exte public boolean hasNext() { if (rot) return true; - return iterator.hasNext(); + return this.p < this.cachecopy.length; } public ReferenceContainer next() { - if (iterator.hasNext()) { - ReferenceContainer c = iterator.next(); + if (this.p < this.cachecopy.length) { + ReferenceContainer c = this.cachecopy[this.p++]; this.latestTermHash = c.getTermHash(); return c.topLevelClone(); } @@ -283,14 +303,14 @@ public final class ReferenceContainerCache exte if (!rot) { return null; } - iterator = cache.values().iterator(); - ReferenceContainer c = iterator.next(); + p = 0; + ReferenceContainer c = this.cachecopy[this.p++]; this.latestTermHash = c.getTermHash(); return c.topLevelClone(); } public void remove() { - iterator.remove(); + System.arraycopy(this.cachecopy, this.p, this.cachecopy, this.p - 1, this.cachecopy.length - p); cache.remove(new ByteArray(this.latestTermHash)); } diff --git a/source/de/anomic/kelondro/text/ReferenceOrder.java b/source/de/anomic/kelondro/text/ReferenceOrder.java index bbd0f39f0..1416cb041 100644 --- a/source/de/anomic/kelondro/text/ReferenceOrder.java +++ b/source/de/anomic/kelondro/text/ReferenceOrder.java @@ -43,11 +43,12 @@ import de.anomic.search.RankingProcess; import de.anomic.yacy.yacyURL; public class ReferenceOrder { - private WordReferenceVars min, max; - private final RankingProfile ranking; - private final ScoreCluster doms; // collected for "authority" heuristic - private int maxdomcount; - private String language; + + protected int maxdomcount; + protected WordReferenceVars min, max; + protected final ScoreCluster doms; // collected for "authority" heuristic + private final RankingProfile ranking; + private String language; public ReferenceOrder(final RankingProfile profile, String language) { this.min = null; diff --git a/source/de/anomic/kelondro/util/SetTools.java b/source/de/anomic/kelondro/util/SetTools.java index 9c8207c9d..56f6af567 100644 --- a/source/de/anomic/kelondro/util/SetTools.java +++ b/source/de/anomic/kelondro/util/SetTools.java @@ -38,7 +38,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; -import java.util.Map.Entry; public class SetTools { @@ -294,17 +293,18 @@ public class SetTools { // ------------------------------------------------------------------------------------------------ // exclude - public static TreeMap excludeConstructive(final TreeMap map, final TreeSet set) { - // comparators must be equal + /* + public static TreeMap excludeConstructive(final TreeMap map, final Set set) { if (map == null) return null; if (set == null) return map; if ((map.size() == 0) || (set.size() == 0)) return map; - if (map.comparator() != set.comparator()) return excludeConstructiveByTestMapInSet(map, set); + assert !(set instanceof TreeSet) || map.comparator() == ((TreeSet) set).comparator(); + // if (map.comparator() != set.comparator()) return excludeConstructiveByTestMapInSet(map, set); return excludeConstructiveByTestMapInSet(map, set); // return excludeConstructiveByEnumeration(map, set); } - private static TreeMap excludeConstructiveByTestMapInSet(final TreeMap map, final TreeSet set) { + private static TreeMap excludeConstructiveByTestMapInSet(final TreeMap map, final Set set) { final TreeMap result = new TreeMap(map.comparator()); A o; for (Entry entry: map.entrySet()) { @@ -313,12 +313,13 @@ public class SetTools { } return result; } + */ - public static void excludeDestructive(final TreeMap map, final TreeSet set) { + public static void excludeDestructive(final Map map, final Set set) { // comparators must be equal if (map == null) return; if (set == null) return; - if (map.comparator() != set.comparator()) return; + assert !(map instanceof TreeMap && set instanceof TreeSet) || ((TreeMap) map).comparator() == ((TreeSet) set).comparator(); if ((map.size() == 0) || (set.size() == 0)) return; if (map.size() < set.size()) @@ -327,22 +328,21 @@ public class SetTools { excludeDestructiveByTestSetInMap(map, set); } - private static void excludeDestructiveByTestMapInSet(final TreeMap map, final TreeSet set) { + private static void excludeDestructiveByTestMapInSet(final Map map, final Set set) { final Iterator mi = map.keySet().iterator(); while (mi.hasNext()) if (set.contains(mi.next())) mi.remove(); } - private static void excludeDestructiveByTestSetInMap(final TreeMap map, final TreeSet set) { + private static void excludeDestructiveByTestSetInMap(final Map map, final Set set) { final Iterator si = set.iterator(); while (si.hasNext()) map.remove(si.next()); } // and the same again with set-set - public static void excludeDestructive(final TreeSet set1, final TreeSet set2) { - // comparators must be equal + public static void excludeDestructive(final Set set1, final Set set2) { if (set1 == null) return; if (set2 == null) return; - if (set1.comparator() != set2.comparator()) return; + assert !(set1 instanceof TreeSet && set2 instanceof TreeSet) || ((TreeSet) set1).comparator() == ((TreeSet) set2).comparator(); if ((set1.size() == 0) || (set2.size() == 0)) return; if (set1.size() < set2.size()) @@ -351,12 +351,12 @@ public class SetTools { excludeDestructiveByTestLargeInSmall(set1, set2); } - private static void excludeDestructiveByTestSmallInLarge(final TreeSet small, final TreeSet large) { + private static void excludeDestructiveByTestSmallInLarge(final Set small, final Set large) { final Iterator mi = small.iterator(); while (mi.hasNext()) if (large.contains(mi.next())) mi.remove(); } - private static void excludeDestructiveByTestLargeInSmall(final TreeSet large, final TreeSet small) { + private static void excludeDestructiveByTestLargeInSmall(final Set large, final Set small) { final Iterator si = small.iterator(); while (si.hasNext()) large.remove(si.next()); } diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index ea9ba4ce3..da1eab1b4 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -1208,6 +1208,8 @@ public final class Switchboard extends serverAbstractSwitch implements serverSwi Response response; while ((surrogate = reader.take()) != DCEntry.poison) { // check if url is in accepted domain + assert surrogate != null; + assert crawlStacker != null; final String urlRejectReason = crawlStacker.urlInAcceptedDomain(surrogate.url()); if (urlRejectReason != null) { if (this.log.isFine()) this.log.logInfo("Rejected URL '" + surrogate.url() + "': " + urlRejectReason); diff --git a/source/de/anomic/server/serverProfiling.java b/source/de/anomic/server/serverProfiling.java index 94fb9399b..c2e964ef4 100644 --- a/source/de/anomic/server/serverProfiling.java +++ b/source/de/anomic/server/serverProfiling.java @@ -26,16 +26,16 @@ package de.anomic.server; +import java.util.ArrayList; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; import de.anomic.kelondro.util.MemoryControl; public class serverProfiling extends Thread { - private static final Map> historyMaps = new ConcurrentHashMap>(); + private static final Map> historyMaps = new ConcurrentHashMap>(); private static final Map eventAccess = new ConcurrentHashMap(); // value: last time when this was accessed private static serverProfiling systemProfiler = null; @@ -84,26 +84,26 @@ public class serverProfiling extends Thread { return; // protect against too heavy load } } - ConcurrentLinkedQueue history = historyMaps.get(eventName); - if (history != null) { + ArrayList history = historyMaps.get(eventName); + if (history != null) synchronized (history) { // update entry history.add(new Event(eventPayload)); // clean up too old entries int tp = history.size() - 30000; - while (tp-- > 0) history.poll(); + while (tp-- > 0) history.remove(0); if (history.size() % 10 == 0) { // reduce number of System.currentTimeMillis() calls Event e; final long now = System.currentTimeMillis(); while (history.size() > 0) { - e = history.peek(); + e = history.get(0); if (now - e.time < 600000) break; - history.poll(); + history.remove(0); } } } else { - history = new ConcurrentLinkedQueue(); + history = new ArrayList(100); // update entry history.add(new Event(eventPayload)); @@ -113,16 +113,20 @@ public class serverProfiling extends Thread { } } - public static Iterator history(final String eventName) { - return (historyMaps.containsKey(eventName) ? (historyMaps.get(eventName)) : new ConcurrentLinkedQueue()).iterator(); + public static ArrayList history(final String eventName) { + return historyMaps.get(eventName); } public static int countEvents(final String eventName, long time) { - Iterator i = history(eventName); + ArrayList event = history(eventName); + if (event == null) return 0; long now = System.currentTimeMillis(); int count = 0; - while (i.hasNext()) { - if (now - i.next().time < time) count++; + synchronized (event) { + Iterator i = event.iterator(); + while (i.hasNext()) { + if (now - i.next().time < time) count++; + } } return count; } diff --git a/source/de/anomic/ymage/ProfilingGraph.java b/source/de/anomic/ymage/ProfilingGraph.java index e82fac3ca..152e90ad9 100644 --- a/source/de/anomic/ymage/ProfilingGraph.java +++ b/source/de/anomic/ymage/ProfilingGraph.java @@ -26,8 +26,8 @@ package de.anomic.ymage; +import java.util.ArrayList; import java.util.ConcurrentModificationException; -import java.util.Iterator; import de.anomic.server.serverProfiling; import de.anomic.server.serverProfiling.Event; @@ -37,13 +37,14 @@ public class ProfilingGraph { private static ymageChart bufferChart = null; public static long maxPayload(final String eventname, final long min) { - final Iterator i = serverProfiling.history(eventname); - serverProfiling.Event event; + final ArrayList list = serverProfiling.history(eventname); + if (list == null) return min; long max = min, l; - while (i.hasNext()) { - event = i.next(); - l = ((Long) event.payload).longValue(); - if (l > max) max = l; + synchronized (list) { + for (serverProfiling.Event event: list) { + l = ((Long) event.payload).longValue(); + if (l > max) max = l; + } } return max; } @@ -79,7 +80,6 @@ public class ProfilingGraph { final long now = System.currentTimeMillis(); long bytes; int x0, x1, y0, y1, ppm, words; - serverProfiling.Event event; try { // draw urls /* @@ -98,10 +98,9 @@ public class ProfilingGraph { } */ // draw memory - Iterator i = serverProfiling.history("memory"); + ArrayList events = serverProfiling.history("memory"); x0 = 1; y0 = 0; - while (i.hasNext()) { - event = i.next(); + if (events != null) synchronized (events) {for (serverProfiling.Event event: events) { time = event.time - now; bytes = ((Long) event.payload).longValue(); x1 = (int) (time/1000); @@ -111,13 +110,12 @@ public class ProfilingGraph { chart.setColor("0000FF"); if (x0 < 0) chart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_RIGHT, x0, y0, x1, y1); x0 = x1; y0 = y1; - } + }} // draw wordcache - i = serverProfiling.history("wordcache"); + events = serverProfiling.history("wordcache"); x0 = 1; y0 = 0; - while (i.hasNext()) { - event = i.next(); + if (events != null) synchronized (events) {for (serverProfiling.Event event: events) { time = event.time - now; words = (int) ((Long) event.payload).longValue(); x1 = (int) (time/1000); @@ -127,13 +125,12 @@ public class ProfilingGraph { chart.setColor("008800"); if (x0 < 0) chart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_LEFT, x0, y0, x1, y1); x0 = x1; y0 = y1; - } + }} // draw ppm - i = serverProfiling.history("ppm"); + events = serverProfiling.history("ppm"); x0 = 1; y0 = 0; - while (i.hasNext()) { - event = i.next(); + if (events != null) synchronized (events) {for (serverProfiling.Event event: events) { time = event.time - now; ppm = (int) ((Long) event.payload).longValue(); x1 = (int) (time/1000); @@ -143,7 +140,7 @@ public class ProfilingGraph { chart.setColor("AA2222"); chart.chartDot(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_ANOT0, x1, y1, 2, ppm + " PPM", 0); x0 = x1; y0 = y1; - } + }} bufferChart = chart; } catch (final ConcurrentModificationException cme) {