performance hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5288 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/JakartaCommonsHttpClient.java

@@ -342,7 +342,7 @@ public class JakartaCommonsHttpClient {
      */
     private RequestEntity zipRequest(final RequestEntity data) throws IOException {
         // cache data and gzip it
-        final ByteArrayOutputStream zippedBytes = new ByteArrayOutputStream();
+        final ByteArrayOutputStream zippedBytes = new ByteArrayOutputStream(512);
         final GZIPOutputStream toZip = new GZIPOutputStream(zippedBytes);
         data.writeRequest(toZip);
         toZip.finish();

source/de/anomic/http/httpTemplate.java

@@ -246,7 +246,7 @@ public final class httpTemplate {
 
         while (transferUntil(pis, out, hasha)) {
             bb = pis.read();
-            keyStream = new ByteArrayOutputStream();
+            keyStream = new ByteArrayOutputStream(512);
 
             if( (bb & 0xFF) == lcbr ){ //multi
                 if( transferUntil(pis, keyStream, mClose) ){ //close tag
@@ -256,7 +256,7 @@ public final class httpTemplate {
                         pis.unread(bb);
                     }
                     multi_key = keyStream.toByteArray(); //IMPORTANT: no prefix here
-                    keyStream = new ByteArrayOutputStream(); //reset stream
+                    keyStream.reset(); //reset stream
 
                     /* DEBUG - print key + value
 			try{
@@ -318,7 +318,7 @@ public final class httpTemplate {
 		}
                  */
 
-                keyStream=new ByteArrayOutputStream(); //clear
+                keyStream.reset(); //clear
 
                 boolean byName=false;
                 int whichPattern=0;
@@ -337,7 +337,7 @@ public final class httpTemplate {
 
                 int currentPattern=0;
                 boolean found=false;
-                keyStream = new ByteArrayOutputStream(); //reset stream
+                keyStream.reset(); //reset stream
                 if(byName){
                     //TODO: better Error Handling
                     transferUntil(pis, keyStream,appendBytes("%%".getBytes("UTF-8"),patternName,null,null));
@@ -345,7 +345,7 @@ public final class httpTemplate {
                         serverLog.logSevere("TEMPLATE", "No such Template: %%"+new String(patternName));
                         return structure.getBytes();
                     }
-                    keyStream=new ByteArrayOutputStream();
+                    keyStream.reset();
                     transferUntil(pis, keyStream, "::".getBytes());
                     pis2 = new PushbackInputStream(new ByteArrayInputStream(keyStream.toByteArray()));
                     structure.append(writeTemplate(pis2, out, pattern, dflt, newPrefix(prefix,key)));
@@ -376,7 +376,7 @@ public final class httpTemplate {
                                     others++;
                                     text.append("#(".getBytes("UTF-8")).append(keyStream.toByteArray()).append(")#".getBytes("UTF-8"));
                                 }
-                                keyStream = new ByteArrayOutputStream(); //reset stream
+                                keyStream.reset(); //reset stream
                                 continue;
                             } //is not #(
                             pis.unread(bb);//is processed in next loop
@@ -436,7 +436,7 @@ public final class httpTemplate {
                 }
             }else if( (bb & 0xFF) == ps){ //include
                 final serverByteBuffer include = new serverByteBuffer();                
-                keyStream = new ByteArrayOutputStream(); //reset stream
+                keyStream.reset(); //reset stream
                 if(transferUntil(pis, keyStream, iClose)){
                     byte[] filename = keyStream.toByteArray();
                     //if(filename.startsWith( Character.toString((char)lbr) ) && filename.endsWith( Character.toString((char)rbr) )){ //simple pattern for filename

source/de/anomic/http/httpd.java

@@ -747,7 +747,7 @@ public final class httpd implements serverHandler, Cloneable {
             assert bytesRead == buffer.length;
         // parsing post request bodies which are gzip content-encoded
         } else {
-            ByteArrayOutputStream bout = new ByteArrayOutputStream();
+            ByteArrayOutputStream bout = new ByteArrayOutputStream(512);
             serverFileUtils.copy(in,bout);
             buffer = bout.toByteArray();
             bout.close(); bout = null;
@@ -1206,7 +1206,7 @@ public final class httpd implements serverHandler, Cloneable {
             
             httpTemplate.writeTemplate(
                     fis = new FileInputStream(new File(htRootPath, "/proxymsg/error.html")), 
-                    o = new ByteArrayOutputStream(), 
+                    o = new ByteArrayOutputStream(512), 
                     tp, 
                     "-UNRESOLVED_PATTERN-".getBytes()
             );

source/de/anomic/http/httpdProxyHandler.java

@@ -600,7 +600,8 @@ public final class httpdProxyHandler {
                     ((storeHTCache) || (isSupportedContent))
             ) {
                 // we don't write actually into a file, only to RAM, and schedule writing the file.
-                final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+                int l = res.getResponseHeader().size();
+                final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);
                 if(isBinary) {
                     final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});
                     serverFileUtils.copy(res.getDataAsStream(), toClientAndMemory);
@@ -1014,17 +1015,17 @@ public final class httpdProxyHandler {
             // "if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader"
             // it seems that it is a HTTP/1.1 connection which stays open (the inputStream) and endlessly waits for
             // input so we have to end it to do the request
-            final long requestLength = requestHeader.getContentLength();
+            final int contentLength = requestHeader.getContentLength();
-            if(requestLength > -1) {
+            if (contentLength > -1) {
                 final byte[] bodyData;
-                if(requestLength == 0) {
+                if(contentLength == 0) {
                     // no body
                     bodyData = new byte[0];
                 } else {
                     // read content-length bytes into memory
-                    final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+                    bodyData = new byte[contentLength];
-                    serverFileUtils.copy(body, buffer, requestLength);
+                    int r = body.read(bodyData, 0, contentLength);
-                    bodyData = buffer.toByteArray();
+                    if (r < contentLength) throw new IOException("not all read: " + r + " from " + contentLength);
                 }
                 body = new ByteArrayInputStream(bodyData);
             }

source/de/anomic/kelondro/kelondroAbstractRA.java

@@ -75,7 +75,7 @@ abstract class kelondroAbstractRA implements kelondroRA {
     }
     
     public byte[] readFully() throws IOException {
-        final ByteArrayOutputStream dest = new ByteArrayOutputStream();
+        final ByteArrayOutputStream dest = new ByteArrayOutputStream(512);
         final byte[] buffer = new byte[1024];
         
         int c, total = 0;

source/de/anomic/kelondro/kelondroEcoFS.java

@@ -69,8 +69,8 @@ public class kelondroEcoFS {
     /**
      * stay below hard disc cache (is that necessary?)
      */
-    private static final int maxReadCache = 8 * 1024;
+    private static final int maxReadCache = 512 * 1024;
-    private static final int maxWriteBuffer = 4 * 1024;
+    private static final int maxWriteBuffer = 512 * 1024;
     
     
     public kelondroEcoFS(final File tablefile, final int recordsize) throws IOException {
@@ -267,7 +267,7 @@ public class kelondroEcoFS {
 
     public synchronized void put(final long index, final byte[] b, final int start) throws IOException {
         assert b.length - start >= this.recordsize;
-        final long s = size();
+        final long s = filesize() + this.buffercount;
         if (index > s) throw new IndexOutOfBoundsException("kelondroEcoFS.put(" + index + ") outside bounds (" + this.size() + ")");
         
         // check if this is an empty entry

source/de/anomic/plasma/parser/mimeType/odtDetector.java

@@ -82,7 +82,7 @@ public class odtDetector implements MagicDetector {
             
             // read in the content of the file
             final InputStream zippedContent = zipFile.getInputStream(mimeTypeInfo); 
-            final String realMimeType = new String(serverFileUtils.read(zippedContent, mimeTypeInfo.getSize()));
+            final String realMimeType = new String(serverFileUtils.read(zippedContent, (int) mimeTypeInfo.getSize()));
             
             return new String[]{realMimeType};
         } catch (final Exception e) {

source/de/anomic/server/serverFileUtils.java

@@ -249,15 +249,17 @@ public final class serverFileUtils {
         return read(source,-1);
     }
     
-    public static byte[] read(final InputStream source, final long count) throws IOException {
+    public static byte[] read(final InputStream source, final int count) throws IOException {
-        final ByteArrayOutputStream baos = (count > 0) 
+        if (count > 0) {
-                                   ? new ByteArrayOutputStream((int)count) 
+            byte[] b = new byte[count];
-                                   : new ByteArrayOutputStream();
+            source.read(b, 0, count);
-        copy(source, baos, count);
+            return b;
-        baos.close();
+        } else {
-        
+            final ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
-        // convert Stream into array
+            copy(source, baos, count);
-        return baos.toByteArray();        
+            baos.close();
+            return baos.toByteArray();
+        }
     }
 
     public static byte[] read(final File source) throws IOException {
@@ -331,7 +333,7 @@ public final class serverFileUtils {
             System.out.println("DEBUG: uncompressGZipArray - uncompressing source");
             try {
                 final ByteArrayInputStream byteInput = new ByteArrayInputStream(source);
-                final ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
+                final ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(source.length / 5);
                 final GZIPInputStream zippedContent = new GZIPInputStream(byteInput);
                 final byte[] data = new byte[1024];
                 int read = 0;

source/de/anomic/tools/gzip.java

@@ -75,7 +75,7 @@ public class gzip {
     public static byte[] gzipString(final String in) {
 	try {
 	    final InputStream  fin  = new ByteArrayInputStream(in.getBytes("UTF8"));
-	    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+	    final ByteArrayOutputStream baos = new ByteArrayOutputStream(in.length() / 3);
 	    final OutputStream fout = new GZIPOutputStream(baos, 128);
 	    copy(fout, fin, 128);
 	    fin.close();
@@ -90,7 +90,7 @@ public class gzip {
 	
     public static String gunzipString(final byte[] in) throws IOException {
 	    final InputStream  fin  = new GZIPInputStream(new ByteArrayInputStream(in));
-	    final ByteArrayOutputStream fout = new ByteArrayOutputStream();
+	    final ByteArrayOutputStream fout = new ByteArrayOutputStream(in.length / 3);
 	    copy(fout, fin, 128);
 	    fin.close();
 	    fout.close();