From 993b9bc1a8b36df4d5abffe742dea4055a9ad078 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 3 Mar 2011 11:30:04 +0000 Subject: [PATCH] memory/performance hacks, less synchronization, better concurrency git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7544 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/search/SearchEvent.java | 31 ++++++++++--------- source/de/anomic/yacy/yacySeedDB.java | 8 +++-- source/net/yacy/kelondro/blob/ArrayStack.java | 22 ++++++++----- .../yacy/kelondro/data/meta/DigestURI.java | 4 ++- source/net/yacy/kelondro/data/word/Word.java | 2 +- source/net/yacy/kelondro/index/HandleSet.java | 12 +++---- source/net/yacy/kelondro/util/FileUtils.java | 3 ++ 7 files changed, 48 insertions(+), 34 deletions(-) diff --git a/source/de/anomic/search/SearchEvent.java b/source/de/anomic/search/SearchEvent.java index 5639369f9..47d747bcc 100644 --- a/source/de/anomic/search/SearchEvent.java +++ b/source/de/anomic/search/SearchEvent.java @@ -389,28 +389,29 @@ public final class SearchEvent { * @param singleAbstract // a mapping from url-hashes to a string of peer-hashes */ public void addAbstract(String wordhash, TreeMap singleAbstract) { + SortedMap oldAbstract; synchronized (abstractsCache) { - SortedMap oldAbstract = abstractsCache.get(wordhash); + oldAbstract = abstractsCache.get(wordhash); if (oldAbstract == null) { // new abstracts in the cache abstractsCache.put(wordhash, singleAbstract); - } else { - // extend the abstracts in the cache: join the single abstracts - for (final Map.Entry oneref: singleAbstract.entrySet()) { - final String urlhash = oneref.getKey(); - final String peerlistNew = oneref.getValue(); - synchronized (oldAbstract) { - final String peerlistOld = oldAbstract.get(urlhash); - if (peerlistOld == null) { - oldAbstract.put(urlhash, peerlistNew); - } else { - oldAbstract.put(urlhash, peerlistOld + peerlistNew); - } - } + return; + } + } + // extend the abstracts in the cache: join the single abstracts + for (final Map.Entry oneref: singleAbstract.entrySet()) { + final String urlhash = oneref.getKey(); + final String peerlistNew = oneref.getValue(); + synchronized (oldAbstract) { + final String peerlistOld = oldAbstract.get(urlhash); + if (peerlistOld == null) { + oldAbstract.put(urlhash, peerlistNew); + } else { + oldAbstract.put(urlhash, peerlistOld + peerlistNew); } - // abstractsCache.put(wordhash, oldAbstract); } } + // abstractsCache.put(wordhash, oldAbstract); // put not necessary since it is sufficient to just change the value content (it stays assigned) } public void commitAbstract() { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 72994c92b..429f7df01 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -1020,8 +1020,12 @@ public final class yacySeedDB implements AlternativeDomainNames { dna0 = it.next(); assert dna0 != null; if (dna0 == null) continue; - dna = new ConcurrentHashMap(); - dna.putAll(dna0); + if (dna0 instanceof ConcurrentHashMap) { + dna = (ConcurrentHashMap) dna0; + } else { + dna = new ConcurrentHashMap(); + dna.putAll(dna0); + } final String hash = dna.remove("key"); assert hash != null; if (hash == null) continue; // bad seed diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 9adf8f5a6..ea03424a0 100755 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -802,16 +802,22 @@ public class ArrayStack implements BLOB { blobItem bi = blobs.get(0); bi.blob.delete(key); } else { - Thread[] t = new Thread[blobs.size()]; + Thread[] t = new Thread[blobs.size() - 1]; int i = 0; for (blobItem bi: blobs) { - final blobItem bi0 = bi; - t[i] = new Thread() { - public void run() { - try { bi0.blob.delete(key); } catch (IOException e) {} - } - }; - t[i].start(); + if (i < t.length) { + // run this in a concurrent thread + final blobItem bi0 = bi; + t[i] = new Thread() { + public void run() { + try { bi0.blob.delete(key); } catch (IOException e) {} + } + }; + t[i].start(); + } else { + // no additional thread, run in this thread + try { bi.blob.delete(key); } catch (IOException e) {} + } i++; } for (Thread s: t) try {s.join();} catch (InterruptedException e) {} diff --git a/source/net/yacy/kelondro/data/meta/DigestURI.java b/source/net/yacy/kelondro/data/meta/DigestURI.java index c6f6021fa..136427abb 100644 --- a/source/net/yacy/kelondro/data/meta/DigestURI.java +++ b/source/net/yacy/kelondro/data/meta/DigestURI.java @@ -225,7 +225,9 @@ public class DigestURI extends MultiProtocolURI implements Serializable { } private static char subdomPortPath(final String subdom, final int port, final String rootpath) { - return Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(subdom + ":" + port + ":" + rootpath)).charAt(0); + StringBuilder sb = new StringBuilder(subdom.length() + rootpath.length() + 8); + sb.append(subdom).append(':').append(Integer.toString(port)).append(':').append(rootpath); + return Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(sb.toString())).charAt(0); } private static final char rootURLFlag0 = subdomPortPath("", 80, ""); diff --git a/source/net/yacy/kelondro/data/word/Word.java b/source/net/yacy/kelondro/data/word/Word.java index 79abbe8b8..334c95ea4 100644 --- a/source/net/yacy/kelondro/data/word/Word.java +++ b/source/net/yacy/kelondro/data/word/Word.java @@ -56,7 +56,7 @@ public class Word { public static final int commonHashLength = 12; private static final int hashCacheSize = Math.max(100000, Math.min(10000000, (int) (MemoryControl.available() / 20000L))); - private static final ARC hashCache = new ConcurrentARC(hashCacheSize, Runtime.getRuntime().availableProcessors() + 1); + private static final ARC hashCache = new ConcurrentARC(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors()); // object carries statistics for words and sentences public int count; // number of occurrences diff --git a/source/net/yacy/kelondro/index/HandleSet.java b/source/net/yacy/kelondro/index/HandleSet.java index c9baa41d8..921a74473 100644 --- a/source/net/yacy/kelondro/index/HandleSet.java +++ b/source/net/yacy/kelondro/index/HandleSet.java @@ -164,21 +164,19 @@ public final class HandleSet implements Iterable, Cloneable { * @throws IOException * @throws RowSpaceExceededException */ - public final synchronized boolean put(final byte[] key) throws RowSpaceExceededException { + public final boolean put(final byte[] key) throws RowSpaceExceededException { assert (key != null); - final Row.Entry newentry = index.row().newEntry(); - newentry.setCol(0, key); + final Row.Entry newentry = index.row().newEntry(key); return index.put(newentry); } - public final synchronized void putUnique(final byte[] key) throws RowSpaceExceededException { + public final void putUnique(final byte[] key) throws RowSpaceExceededException { assert (key != null); - final Row.Entry newentry = this.rowdef.newEntry(); - newentry.setCol(0, key); + final Row.Entry newentry = index.row().newEntry(key); index.addUnique(newentry); } - public final synchronized boolean remove(final byte[] key) { + public final boolean remove(final byte[] key) { assert (key != null); Row.Entry indexentry; indexentry = index.remove(key); diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index ddaf09781..b779ca4c2 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -750,6 +750,9 @@ public final class FileUtils { } } catch (IOException e) { nextLine = null; + } catch (OutOfMemoryError e) { + Log.logException(e); + nextLine = null; } return line; }