performance hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7977 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/net/yacy/kelondro/blob/HeapModifier.java

@@ -109,7 +109,7 @@ public class HeapModifier extends HeapReader implements BLOB {
             if (seek < 0) return;
             
             // check consistency of the index
-            assert (checkKey(key, seek)) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + seek;
+            //assert (checkKey(key, seek)) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + seek;
             
             // access the file and read the container
             this.file.seek(seek);
@@ -262,7 +262,7 @@ public class HeapModifier extends HeapReader implements BLOB {
             if (pos < 0) return 0;
             
             // check consistency of the index
-            assert checkKey(key, pos) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + pos;
+            //assert checkKey(key, pos) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + pos;
             
             // access the file and read the container
             file.seek(pos);

source/net/yacy/kelondro/blob/MapDataMining.java

@@ -259,14 +259,14 @@ public class MapDataMining extends MapHeap {
         if (key == null) return;
         
         // update elementCount
-        if ((sortfields != null) || (longaccfields != null) || (floataccfields != null)) {
+        if (sortfields != null || longaccfields != null || floataccfields != null) {
             Map<String, String> map;
             try {
                 map = super.get(key, false);
                 if (map != null) {
 
                     // update accumulators (subtract)
-                    if ((longaccfields != null) || (floataccfields != null)) updateAcc(map, false);
+                    if (longaccfields != null || floataccfields != null) updateAcc(map, false);
 
                     // remove from sortCluster
                     if (sortfields != null) deleteSortCluster(UTF8.String(key));

source/net/yacy/kelondro/logging/ThreadDump.java

@@ -117,19 +117,21 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
         
         // try to get the thread dump from yacy.log which is available when YaCy is started with
         // startYACY.sh -l
-        if (!canProduceLockedBy(logFile)) return;
-        long sizeBefore = logFile.length();
-        
-        // get the current process PID
-        int pid = OS.getPID();
-        
-        // call kill -3 on the pid
-        if (pid >= 0) try {OS.execSynchronous("kill -3 " + pid);} catch (IOException e) {}
+        long sizeBefore = 0;
+        if (canProduceLockedBy(logFile)) {
+            sizeBefore = logFile.length();
+            
+            // get the current process PID
+            int pid = OS.getPID();
+            
+            // call kill -3 on the pid
+            if (pid >= 0) try {OS.execSynchronous("kill -3 " + pid);} catch (IOException e) {}
+        }
 
         // read the log from the dump
         long sizeAfter = logFile.length();
         if (sizeAfter <= sizeBefore) return;
-
+        
         RandomAccessFile raf = new RandomAccessFile(logFile, "r");
         raf.seek(sizeBefore);
         byte[] b = new byte[(int) (sizeAfter - sizeBefore)];
@@ -412,7 +414,9 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
             }
         }
         //dump.print();
+        assert dump != null;
         Map<StackTrace, Integer> locks = dump.countLocks();
+        assert locks != null;
         System.out.println("*** Thread Dump Lock report; dump size = " + dump.size() + ", locks = " + locks.size());
         for (int i = 0; i < dump.size() + 10; i++) {
             for (Map.Entry<StackTrace, Integer> entry: locks.entrySet()) {

source/net/yacy/kelondro/rwi/ReferenceContainerCache.java

@@ -61,7 +61,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
     private final int termSize;
     private final ByteOrder termOrder;
     private final ContainerOrder<ReferenceType> containerOrder;
-    private Map<ByteArray, ReferenceContainer<ReferenceType>> cache;
+    private ConcurrentHashMap<ByteArray, ReferenceContainer<ReferenceType>> cache;
 
     /**
      * open an existing heap file in undefined mode
@@ -511,20 +511,22 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
         container.put(newEntry);
 
         // synchronization: check if the entry is still empty and set new value
-        synchronized (this.cache) {
-            final ReferenceContainer<ReferenceType> containerNew = this.cache.put(tha, container);
+        final ReferenceContainer<ReferenceType> container0 = this.cache.put(tha, container);
+        if (container0 != null) synchronized (this.cache) {
+            // no luck here, we get a lock exclusively to sort this out
+            final ReferenceContainer<ReferenceType> containerNew = this.cache.put(tha, container0);
             if (containerNew == null) return;
-            if (container == containerNew) {
+            if (container0 == containerNew) {
                 // The containers are the same, so nothing needs to be done
                 return;
             }
             // Now merge the smaller container into the lager.
             // The other way around can become very slow
-            if (container.size() >= containerNew.size()) {
-                container.putAllRecent(containerNew);
-       	        this.cache.put(tha, container);
+            if (container0.size() >= containerNew.size()) {
+                container0.putAllRecent(containerNew);
+       	        this.cache.put(tha, container0);
             } else {
-                containerNew.putAllRecent(container);
+                containerNew.putAllRecent(container0);
                 this.cache.put(tha, containerNew);
             }
         }

source/net/yacy/peers/yacySeed.java

@@ -178,9 +178,6 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
     private String alternativeIP = null;
     private long birthdate; // keep this value in ram since it is often used and may cause lockings in concurrent situations.
 
-    // use our own formatter to prevent concurrency locks with other processes
-    private final static GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
-
     public yacySeed(final String theHash, final ConcurrentMap<String, String> theDna) {
         // create a seed with a pre-defined hash map
         assert theHash != null;
@@ -234,7 +231,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
         this.dna.put(yacySeed.URL_IN, yacySeed.ZERO);    // received URLs
 
         // default first filling
-        this.dna.put(yacySeed.BDATE, my_SHORT_SECOND_FORMATTER.format());
+        this.dna.put(yacySeed.BDATE, GenericFormatter.SHORT_SECOND_FORMATTER.format());
         this.dna.put(yacySeed.LASTSEEN, this.dna.get(yacySeed.BDATE)); // just as initial setting
         this.dna.put(yacySeed.UTC, GenericFormatter.UTCDiffString());
         this.dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); // virgin/junior/senior/principal
@@ -524,6 +521,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
         // because java thinks it must apply the UTC offset to the current time,
         // to create a string that looks like our current time, it adds the local UTC offset to the
         // time. To create a corrected UTC Date string, we first subtract the local UTC offset.
+        GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
         final String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
         //System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls);
         this.dna.put(yacySeed.LASTSEEN, ls );
@@ -534,6 +532,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
      */
     public final long getLastSeenUTC() {
         try {
+            GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
             final long t = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.LASTSEEN, "20040101000000")).getTime();
             // getTime creates a UTC time number. But in this case java thinks, that the given
             // time string is a local time, which has a local UTC offset applied.
@@ -563,6 +562,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
         if (this.birthdate > 0) return this.birthdate;
         long b;
         try {
+            GenericFormatter my_SHORT_SECOND_FORMATTER  = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
             b = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.BDATE, "20040101000000")).getTime();
         } catch (final ParseException e) {
             b = System.currentTimeMillis();