performance hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7116 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/net/yacy/cora/document/MultiProtocolURI.java

@@ -587,13 +587,27 @@ public class MultiProtocolURI implements Serializable {
         // this is the path plus quest plus ref
         // if there is no quest and no ref the result is identical to getPath
         // this is defined according to http://java.sun.com/j2se/1.4.2/docs/api/java/net/URL.html#getFile()
-        if (quest == null) return (excludeReference || ref == null) ? path : path + "#" + ref;
+        if (quest == null) {
+            if (excludeReference || ref == null) return path;
+            StringBuilder sb = new StringBuilder(120);
+            sb.append(path);
+            sb.append('#');
+            sb.append(ref);
+            return sb.toString();
+        }
         String q = quest;
         if (removeSessionID) {
             for (String sid: sessionIDnames) {
                 if (q.toLowerCase().startsWith(sid.toLowerCase() + "=")) {
                     int p = q.indexOf('&');
-                    if (p < 0) return (excludeReference || ref == null) ? path : path + "#" + ref;
+                    if (p < 0) {
+                        if (excludeReference || ref == null) return path;
+                        StringBuilder sb = new StringBuilder(120);
+                        sb.append(path);
+                        sb.append('#');
+                        sb.append(ref);
+                        return sb.toString();
+                    }
                     q = q.substring(p + 1);
                     continue;
                 }
@@ -607,7 +621,14 @@ public class MultiProtocolURI implements Serializable {
                 }
             }
         }
-        return (excludeReference || ref == null) ? path + "?" + q : path + "?" + q + "#" + ref;
+        StringBuilder sb = new StringBuilder(120);
+        sb.append(path);
+        sb.append('?');
+        sb.append(q);
+        if (excludeReference || ref == null) return sb.toString();
+        sb.append('#');
+        sb.append(ref);
+        return sb.toString();
     }
     
     public String getFileName() {

source/net/yacy/cora/storage/SimpleARC.java

@@ -135,7 +135,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
     /**
      * iterator implements the Iterable interface
      */
-    public Iterator<Map.Entry<K, V>> iterator() {
+    public final Iterator<Map.Entry<K, V>> iterator() {
         return entrySet().iterator();
     }
 
@@ -147,7 +147,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
      * @return a set view of the mappings contained in this map
      */
     @Override
-    public Set<Map.Entry<K, V>> entrySet() {
+    public final synchronized Set<Map.Entry<K, V>> entrySet() {
         Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
         for (Map.Entry<K, V> entry: this.levelA.entrySet()) m.add(entry);
         for (Map.Entry<K, V> entry: this.levelB.entrySet()) m.add(entry);
@@ -158,7 +158,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
      * a hash code for this ARC
      * @return the hash code of one of the ARC partial hash tables
      */
-    public int hashCode() {
+    public final int hashCode() {
         return this.levelA.hashCode();
     }
 }

source/net/yacy/document/parser/html/ContentScraper.java

@@ -284,7 +284,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
         return cleanLine(super.stripAll(new String(scraper.content.getChars())));
     }
     
-    private static String cleanLine(String s) {
+    private final static String cleanLine(String s) {
         StringBuilder sb = new StringBuilder(s.length());
         char c, l = ' ';
         for (int i = 0; i < s.length(); i++) {

source/net/yacy/kelondro/blob/Compressor.java

@@ -228,24 +228,28 @@ public class Compressor implements BLOB {
         }
     }
 
-    public synchronized byte[] get(byte[] key) throws IOException, RowSpaceExceededException {
+    public byte[] get(byte[] key) throws IOException, RowSpaceExceededException {
         // depending on the source of the result, we additionally do entry compression
         // because if a document was read once, we think that it will not be retrieved another time again soon
-        byte[] b = buffer.remove(new String(key));
+        String keys = new String(key);
-        if (b != null) {
+        byte[] b = null;
-            // compress the entry now and put it to the backend
+        synchronized (this) {
-            try {
+            b = buffer.remove(keys);
-                this.writeQueue.put(new Entity(new String(key), b));
+            if (b != null) {
-                this.bufferlength = this.bufferlength - b.length;
+                // compress the entry now and put it to the backend
-                return b;
+                try {
-            } catch (InterruptedException e) {
+                    this.writeQueue.put(new Entity(new String(key), b));
-                Log.logException(e);
+                    this.bufferlength = this.bufferlength - b.length;
-                buffer.put(new String(key), b);
+                    return b;
+                } catch (InterruptedException e) {
+                    Log.logException(e);
+                    buffer.put(keys, b);
+                }
             }
+            
+            // return from the backend
+            b = this.backend.get(key);
         }
-        
-        // return from the backend
-        b = this.backend.get(key);
         if (b == null) return null;
         return decompress(b);
     }
@@ -263,8 +267,7 @@ public class Compressor implements BLOB {
     }
     
     public synchronized boolean containsKey(byte[] key) {
-        return 
+        return this.buffer.containsKey(new String(key)) || this.backend.containsKey(key);
-          this.buffer.containsKey(new String(key)) || this.backend.containsKey(key);
     }
 
     public int keylength() {

source/net/yacy/kelondro/data/word/Word.java

@@ -54,7 +54,7 @@ public class Word {
      */
     public static final int commonHashLength = 12;
     
-    private static final int hashCacheSize = Math.max(2048, Math.min(100000, (int) (MemoryControl.available() / 20000L)));
+    private static final int hashCacheSize = Math.max(10000, Math.min(100000, (int) (MemoryControl.available() / 20000L)));
     private static final ARC<String, byte[]> hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, Runtime.getRuntime().availableProcessors());
     
     // object carries statistics for words and sentences

source/net/yacy/repository/LoaderDispatcher.java

@@ -177,7 +177,7 @@ public final class LoaderDispatcher {
      * @return the loaded entity in a Response object
      * @throws IOException
      */
-    public Response loadInternal(final Request request, CrawlProfile.CacheStrategy cacheStrategy, long maxFileSize) throws IOException {
+    private Response loadInternal(final Request request, CrawlProfile.CacheStrategy cacheStrategy, long maxFileSize) throws IOException {
         // get the protocol of the next URL
         final DigestURI url = request.url();
         final String protocol = url.getProtocol();