more performance hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4676 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/IndexCreateIndexingQueue_p.java

@@ -177,6 +177,7 @@ public class IndexCreateIndexingQueue_p {
             int j=0;
             for (int i = switchboard.crawlQueues.errorURL.stackSize() - 1; i >= (switchboard.crawlQueues.errorURL.stackSize() - showRejectedCount); i--) {
                     entry = switchboard.crawlQueues.errorURL.top(i);
+                    if (entry == null) continue;
                     url = entry.url();
                     if (url == null) continue;
                     

source/de/anomic/http/HttpResponse.java

@@ -27,12 +27,12 @@ package de.anomic.http;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
-import java.io.Writer;
 
 import de.anomic.server.serverFileUtils;
 
@@ -103,8 +103,7 @@ public interface HttpResponse {
          * @throws IOException
          * @throws UnsupportedEncodingException
          */
-        public static void writeContent(HttpResponse res, BufferedOutputStream hfos, BufferedOutputStream byteStream) throws IOException,
+        public static void writeContent(HttpResponse res, BufferedOutputStream hfos, BufferedOutputStream byteStream) throws IOException, UnsupportedEncodingException {
-                UnsupportedEncodingException {
             try {
                 InputStream data = res.getDataAsStream();
                 if (byteStream == null) {
@@ -112,21 +111,20 @@ public interface HttpResponse {
                 } else {
                     serverFileUtils.copyToStreams(new BufferedInputStream(data), hfos, byteStream);
                 }
-                
-                
             } finally {
                 res.closeStream();
             }
         }
         
-        public static void writeContent(HttpResponse res, Writer hfos, OutputStream byteStream) throws IOException,
+        public static void writeContent(HttpResponse res, BufferedWriter hfos, OutputStream byteStream) throws IOException, UnsupportedEncodingException {
-        UnsupportedEncodingException {
             try {
                 InputStream data = res.getDataAsStream();
                 String charSet = httpHeader.getCharSet(res.getResponseHeader());
-                Writer[] writers = (byteStream == null ? new Writer[] { hfos } : new Writer[] { hfos,
+                if (byteStream == null) {
-                        new OutputStreamWriter(byteStream, charSet) });
+                    serverFileUtils.copyToWriter(new BufferedInputStream(data), hfos, charSet);
-                serverFileUtils.copyToWriters(data, writers, charSet);
+                } else {
+                    serverFileUtils.copyToWriters(new BufferedInputStream(data), hfos, new BufferedWriter(new OutputStreamWriter(byteStream, charSet)) , charSet);
+                }
             } finally {
                 res.closeStream();
             }

source/de/anomic/http/httpTemplate.java

@@ -283,9 +283,6 @@ public final class httpTemplate {
                                 num=0;
                             }
                             //System.out.println(multi_key + ": " + num); //DEBUG
-                        }else{
-                            //0 interations - no display
-                            //System.out.println("_"+new String(multi_key)+" is null or does not exist"); //DEBUG
                         }
 
                         //Enumeration enx = pattern.keys(); while (enx.hasMoreElements()) System.out.println("KEY=" + enx.nextElement()); // DEBUG
@@ -336,8 +333,6 @@ public final class httpTemplate {
                         byName=true;
                         patternName=patternId.getBytes("UTF-8");
                     }
-                }else{
-                    //System.out.println("Pattern \""+new String(prefix + key)+"\" is not set"); //DEBUG
                 }
 
                 int currentPattern=0;

source/de/anomic/http/httpdProxyHandler.java

@@ -64,6 +64,7 @@ package de.anomic.http;
 
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -634,7 +635,7 @@ public final class httpdProxyHandler {
                 {
                     // ok, we don't write actually into a file, only to RAM, and schedule writing the file.
                     ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
-                    Saver.writeContent(res, hfos, byteStream);
+                    Saver.writeContent(res, new BufferedWriter(hfos), byteStream);
                     // cached bytes
                     byte[] cacheArray;
                     if(byteStream.size() > 0) {
@@ -669,7 +670,7 @@ public final class httpdProxyHandler {
                     // the file is too big to cache it in the ram, or the size is unknown
                     // write to file right here.
                     cacheFile.getParentFile().mkdirs();
-                    Saver.writeContent(res, hfos, new FileOutputStream(cacheFile));
+                    Saver.writeContent(res, new BufferedWriter(hfos), new FileOutputStream(cacheFile));
                     if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize();
                     theLogger.logFine("for write-file of " + url + ": contentLength = " + contentLength + ", sizeBeforeDelete = " + sizeBeforeDelete);
                     plasmaHTCache.writeFileAnnouncement(cacheFile);
@@ -699,7 +700,7 @@ public final class httpdProxyHandler {
                         " StoreHTCache=" + storeHTCache + 
                         " SupportetContent=" + isSupportedContent);
 
-                Saver.writeContent(res, hfos, null);
+                Saver.writeContent(res, new BufferedWriter(hfos), null);
                 if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize();
                 if (sizeBeforeDelete == -1) {
                     // no old file and no load. just data passing

source/de/anomic/kelondro/kelondroLock.java

@@ -95,7 +95,7 @@ public class kelondroLock {
         if (locked()) {
             lock = false;
             releaseTime = 0;
-            notify();
+            notifyAll();
         }
     }
 

source/de/anomic/kelondro/kelondroStack.java

@@ -145,7 +145,7 @@ public final class kelondroStack extends kelondroFullRecords {
         }
     }
 
-    public int size() {
+    public synchronized int size() {
         return super.size();
     }
     

source/de/anomic/server/serverFileUtils.java

@@ -562,34 +562,36 @@ public final class serverFileUtils {
      * @return
      * @throws IOException
      */
-    public static int copyToWriters(final InputStream data, final Writer[] writers, final String charSet) throws IOException {
+    public static int copyToWriter(final BufferedInputStream data, final BufferedWriter writer, final String charSet) throws IOException {
         // the docs say: "For top efficiency, consider wrapping an InputStreamReader within a BufferedReader."
-        final BufferedReader sourceReader = new BufferedReader(new InputStreamReader(data, charSet));
+        final Reader sourceReader = new InputStreamReader(data, charSet);
-        
-        // check if buffer is used. From the documentation:
-        // "For top efficiency, consider wrapping an OutputStreamWriter within a BufferedWriter so as to avoid frequent
-        // converter invocations"
-        int i = 0;
-        for(final Writer writer: writers) {
-            if (!(writer instanceof BufferedWriter)) {
-                // add buffer
-                writers[i] = new BufferedWriter(writer);
-            }
-            i++;
-        }
         
         int count = 0;
         // copy bytes
         int b;
         while((b = sourceReader.read()) != -1) {
             count++;
-            for(final Writer writer: writers) {
             writer.write(b);
         }
-        }
-        for(final Writer writer: writers) {
         writer.flush();
+        return count;
+    }
+    public static int copyToWriters(final BufferedInputStream data, final BufferedWriter writer0, final BufferedWriter writer1, final String charSet) throws IOException {
+        // the docs say: "For top efficiency, consider wrapping an InputStreamReader within a BufferedReader."
+        assert writer0 != null;
+        assert writer1 != null;
+        final Reader sourceReader = new InputStreamReader(data, charSet);
+        
+        int count = 0;
+        // copy bytes
+        int b;
+        while((b = sourceReader.read()) != -1) {
+            count++;
+            writer0.write(b);
+            writer1.write(b);
         }
+        writer0.flush();
+        writer1.flush();
         return count;
     }
 }