From 57d5529a01c6167de12b64a229ef3645292d91f7 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 28 Sep 2011 21:16:40 +0000 Subject: [PATCH] performance hacks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7977 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../net/yacy/kelondro/blob/HeapModifier.java | 4 ++-- .../net/yacy/kelondro/blob/MapDataMining.java | 4 ++-- .../net/yacy/kelondro/logging/ThreadDump.java | 22 +++++++++++-------- .../kelondro/rwi/ReferenceContainerCache.java | 18 ++++++++------- source/net/yacy/peers/yacySeed.java | 8 +++---- 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/source/net/yacy/kelondro/blob/HeapModifier.java b/source/net/yacy/kelondro/blob/HeapModifier.java index 4f6da3219..79fa1fb86 100644 --- a/source/net/yacy/kelondro/blob/HeapModifier.java +++ b/source/net/yacy/kelondro/blob/HeapModifier.java @@ -109,7 +109,7 @@ public class HeapModifier extends HeapReader implements BLOB { if (seek < 0) return; // check consistency of the index - assert (checkKey(key, seek)) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + seek; + //assert (checkKey(key, seek)) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + seek; // access the file and read the container this.file.seek(seek); @@ -262,7 +262,7 @@ public class HeapModifier extends HeapReader implements BLOB { if (pos < 0) return 0; // check consistency of the index - assert checkKey(key, pos) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + pos; + //assert checkKey(key, pos) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + pos; // access the file and read the container file.seek(pos); diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index 737ef218c..e25d884e8 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -259,14 +259,14 @@ public class MapDataMining extends MapHeap { if (key == null) return; // update elementCount - if ((sortfields != null) || (longaccfields != null) || (floataccfields != null)) { + if (sortfields != null || longaccfields != null || floataccfields != null) { Map map; try { map = super.get(key, false); if (map != null) { // update accumulators (subtract) - if ((longaccfields != null) || (floataccfields != null)) updateAcc(map, false); + if (longaccfields != null || floataccfields != null) updateAcc(map, false); // remove from sortCluster if (sortfields != null) deleteSortCluster(UTF8.String(key)); diff --git a/source/net/yacy/kelondro/logging/ThreadDump.java b/source/net/yacy/kelondro/logging/ThreadDump.java index 81755cb8c..df625c19b 100644 --- a/source/net/yacy/kelondro/logging/ThreadDump.java +++ b/source/net/yacy/kelondro/logging/ThreadDump.java @@ -117,19 +117,21 @@ public class ThreadDump extends HashMap> imp // try to get the thread dump from yacy.log which is available when YaCy is started with // startYACY.sh -l - if (!canProduceLockedBy(logFile)) return; - long sizeBefore = logFile.length(); - - // get the current process PID - int pid = OS.getPID(); - - // call kill -3 on the pid - if (pid >= 0) try {OS.execSynchronous("kill -3 " + pid);} catch (IOException e) {} + long sizeBefore = 0; + if (canProduceLockedBy(logFile)) { + sizeBefore = logFile.length(); + + // get the current process PID + int pid = OS.getPID(); + + // call kill -3 on the pid + if (pid >= 0) try {OS.execSynchronous("kill -3 " + pid);} catch (IOException e) {} + } // read the log from the dump long sizeAfter = logFile.length(); if (sizeAfter <= sizeBefore) return; - + RandomAccessFile raf = new RandomAccessFile(logFile, "r"); raf.seek(sizeBefore); byte[] b = new byte[(int) (sizeAfter - sizeBefore)]; @@ -412,7 +414,9 @@ public class ThreadDump extends HashMap> imp } } //dump.print(); + assert dump != null; Map locks = dump.countLocks(); + assert locks != null; System.out.println("*** Thread Dump Lock report; dump size = " + dump.size() + ", locks = " + locks.size()); for (int i = 0; i < dump.size() + 10; i++) { for (Map.Entry entry: locks.entrySet()) { diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index 92dff501d..1cb3f5102 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -61,7 +61,7 @@ public final class ReferenceContainerCache exte private final int termSize; private final ByteOrder termOrder; private final ContainerOrder containerOrder; - private Map> cache; + private ConcurrentHashMap> cache; /** * open an existing heap file in undefined mode @@ -511,20 +511,22 @@ public final class ReferenceContainerCache exte container.put(newEntry); // synchronization: check if the entry is still empty and set new value - synchronized (this.cache) { - final ReferenceContainer containerNew = this.cache.put(tha, container); + final ReferenceContainer container0 = this.cache.put(tha, container); + if (container0 != null) synchronized (this.cache) { + // no luck here, we get a lock exclusively to sort this out + final ReferenceContainer containerNew = this.cache.put(tha, container0); if (containerNew == null) return; - if (container == containerNew) { + if (container0 == containerNew) { // The containers are the same, so nothing needs to be done return; } // Now merge the smaller container into the lager. // The other way around can become very slow - if (container.size() >= containerNew.size()) { - container.putAllRecent(containerNew); - this.cache.put(tha, container); + if (container0.size() >= containerNew.size()) { + container0.putAllRecent(containerNew); + this.cache.put(tha, container0); } else { - containerNew.putAllRecent(container); + containerNew.putAllRecent(container0); this.cache.put(tha, containerNew); } } diff --git a/source/net/yacy/peers/yacySeed.java b/source/net/yacy/peers/yacySeed.java index 9ed508c11..a7a25cf96 100644 --- a/source/net/yacy/peers/yacySeed.java +++ b/source/net/yacy/peers/yacySeed.java @@ -178,9 +178,6 @@ public class yacySeed implements Cloneable, Comparable, Comparator theDna) { // create a seed with a pre-defined hash map assert theHash != null; @@ -234,7 +231,7 @@ public class yacySeed implements Cloneable, Comparable, Comparator, Comparator, Comparator, Comparator 0) return this.birthdate; long b; try { + GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes b = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.BDATE, "20040101000000")).getTime(); } catch (final ParseException e) { b = System.currentTimeMillis();