From 6941bf42b1425603a2422a86348941b7dd9b3dbf Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 20 Oct 2008 14:07:09 +0000 Subject: [PATCH] performance hacks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5288 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../anomic/http/JakartaCommonsHttpClient.java | 2 +- source/de/anomic/http/httpTemplate.java | 14 ++++++------ source/de/anomic/http/httpd.java | 4 ++-- source/de/anomic/http/httpdProxyHandler.java | 15 +++++++------ .../anomic/kelondro/kelondroAbstractRA.java | 2 +- source/de/anomic/kelondro/kelondroEcoFS.java | 6 ++--- .../plasma/parser/mimeType/odtDetector.java | 2 +- source/de/anomic/server/serverFileUtils.java | 22 ++++++++++--------- source/de/anomic/tools/gzip.java | 4 ++-- 9 files changed, 37 insertions(+), 34 deletions(-) diff --git a/source/de/anomic/http/JakartaCommonsHttpClient.java b/source/de/anomic/http/JakartaCommonsHttpClient.java index e5374d685..d49ae3116 100644 --- a/source/de/anomic/http/JakartaCommonsHttpClient.java +++ b/source/de/anomic/http/JakartaCommonsHttpClient.java @@ -342,7 +342,7 @@ public class JakartaCommonsHttpClient { */ private RequestEntity zipRequest(final RequestEntity data) throws IOException { // cache data and gzip it - final ByteArrayOutputStream zippedBytes = new ByteArrayOutputStream(); + final ByteArrayOutputStream zippedBytes = new ByteArrayOutputStream(512); final GZIPOutputStream toZip = new GZIPOutputStream(zippedBytes); data.writeRequest(toZip); toZip.finish(); diff --git a/source/de/anomic/http/httpTemplate.java b/source/de/anomic/http/httpTemplate.java index 6fba685be..4a9b9d589 100644 --- a/source/de/anomic/http/httpTemplate.java +++ b/source/de/anomic/http/httpTemplate.java @@ -246,7 +246,7 @@ public final class httpTemplate { while (transferUntil(pis, out, hasha)) { bb = pis.read(); - keyStream = new ByteArrayOutputStream(); + keyStream = new ByteArrayOutputStream(512); if( (bb & 0xFF) == lcbr ){ //multi if( transferUntil(pis, keyStream, mClose) ){ //close tag @@ -256,7 +256,7 @@ public final class httpTemplate { pis.unread(bb); } multi_key = keyStream.toByteArray(); //IMPORTANT: no prefix here - keyStream = new ByteArrayOutputStream(); //reset stream + keyStream.reset(); //reset stream /* DEBUG - print key + value try{ @@ -318,7 +318,7 @@ public final class httpTemplate { } */ - keyStream=new ByteArrayOutputStream(); //clear + keyStream.reset(); //clear boolean byName=false; int whichPattern=0; @@ -337,7 +337,7 @@ public final class httpTemplate { int currentPattern=0; boolean found=false; - keyStream = new ByteArrayOutputStream(); //reset stream + keyStream.reset(); //reset stream if(byName){ //TODO: better Error Handling transferUntil(pis, keyStream,appendBytes("%%".getBytes("UTF-8"),patternName,null,null)); @@ -345,7 +345,7 @@ public final class httpTemplate { serverLog.logSevere("TEMPLATE", "No such Template: %%"+new String(patternName)); return structure.getBytes(); } - keyStream=new ByteArrayOutputStream(); + keyStream.reset(); transferUntil(pis, keyStream, "::".getBytes()); pis2 = new PushbackInputStream(new ByteArrayInputStream(keyStream.toByteArray())); structure.append(writeTemplate(pis2, out, pattern, dflt, newPrefix(prefix,key))); @@ -376,7 +376,7 @@ public final class httpTemplate { others++; text.append("#(".getBytes("UTF-8")).append(keyStream.toByteArray()).append(")#".getBytes("UTF-8")); } - keyStream = new ByteArrayOutputStream(); //reset stream + keyStream.reset(); //reset stream continue; } //is not #( pis.unread(bb);//is processed in next loop @@ -436,7 +436,7 @@ public final class httpTemplate { } }else if( (bb & 0xFF) == ps){ //include final serverByteBuffer include = new serverByteBuffer(); - keyStream = new ByteArrayOutputStream(); //reset stream + keyStream.reset(); //reset stream if(transferUntil(pis, keyStream, iClose)){ byte[] filename = keyStream.toByteArray(); //if(filename.startsWith( Character.toString((char)lbr) ) && filename.endsWith( Character.toString((char)rbr) )){ //simple pattern for filename diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index 4ed49e63e..a94bbc9ee 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -747,7 +747,7 @@ public final class httpd implements serverHandler, Cloneable { assert bytesRead == buffer.length; // parsing post request bodies which are gzip content-encoded } else { - ByteArrayOutputStream bout = new ByteArrayOutputStream(); + ByteArrayOutputStream bout = new ByteArrayOutputStream(512); serverFileUtils.copy(in,bout); buffer = bout.toByteArray(); bout.close(); bout = null; @@ -1206,7 +1206,7 @@ public final class httpd implements serverHandler, Cloneable { httpTemplate.writeTemplate( fis = new FileInputStream(new File(htRootPath, "/proxymsg/error.html")), - o = new ByteArrayOutputStream(), + o = new ByteArrayOutputStream(512), tp, "-UNRESOLVED_PATTERN-".getBytes() ); diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 395743abf..b7104b230 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -600,7 +600,8 @@ public final class httpdProxyHandler { ((storeHTCache) || (isSupportedContent)) ) { // we don't write actually into a file, only to RAM, and schedule writing the file. - final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + int l = res.getResponseHeader().size(); + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l); if(isBinary) { final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream}); serverFileUtils.copy(res.getDataAsStream(), toClientAndMemory); @@ -1014,17 +1015,17 @@ public final class httpdProxyHandler { // "if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader" // it seems that it is a HTTP/1.1 connection which stays open (the inputStream) and endlessly waits for // input so we have to end it to do the request - final long requestLength = requestHeader.getContentLength(); - if(requestLength > -1) { + final int contentLength = requestHeader.getContentLength(); + if (contentLength > -1) { final byte[] bodyData; - if(requestLength == 0) { + if(contentLength == 0) { // no body bodyData = new byte[0]; } else { // read content-length bytes into memory - final ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - serverFileUtils.copy(body, buffer, requestLength); - bodyData = buffer.toByteArray(); + bodyData = new byte[contentLength]; + int r = body.read(bodyData, 0, contentLength); + if (r < contentLength) throw new IOException("not all read: " + r + " from " + contentLength); } body = new ByteArrayInputStream(bodyData); } diff --git a/source/de/anomic/kelondro/kelondroAbstractRA.java b/source/de/anomic/kelondro/kelondroAbstractRA.java index 4ddabc84b..3088dd5c5 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRA.java +++ b/source/de/anomic/kelondro/kelondroAbstractRA.java @@ -75,7 +75,7 @@ abstract class kelondroAbstractRA implements kelondroRA { } public byte[] readFully() throws IOException { - final ByteArrayOutputStream dest = new ByteArrayOutputStream(); + final ByteArrayOutputStream dest = new ByteArrayOutputStream(512); final byte[] buffer = new byte[1024]; int c, total = 0; diff --git a/source/de/anomic/kelondro/kelondroEcoFS.java b/source/de/anomic/kelondro/kelondroEcoFS.java index 3247593d9..3463dc629 100644 --- a/source/de/anomic/kelondro/kelondroEcoFS.java +++ b/source/de/anomic/kelondro/kelondroEcoFS.java @@ -69,8 +69,8 @@ public class kelondroEcoFS { /** * stay below hard disc cache (is that necessary?) */ - private static final int maxReadCache = 8 * 1024; - private static final int maxWriteBuffer = 4 * 1024; + private static final int maxReadCache = 512 * 1024; + private static final int maxWriteBuffer = 512 * 1024; public kelondroEcoFS(final File tablefile, final int recordsize) throws IOException { @@ -267,7 +267,7 @@ public class kelondroEcoFS { public synchronized void put(final long index, final byte[] b, final int start) throws IOException { assert b.length - start >= this.recordsize; - final long s = size(); + final long s = filesize() + this.buffercount; if (index > s) throw new IndexOutOfBoundsException("kelondroEcoFS.put(" + index + ") outside bounds (" + this.size() + ")"); // check if this is an empty entry diff --git a/source/de/anomic/plasma/parser/mimeType/odtDetector.java b/source/de/anomic/plasma/parser/mimeType/odtDetector.java index 01c23e806..088a35780 100644 --- a/source/de/anomic/plasma/parser/mimeType/odtDetector.java +++ b/source/de/anomic/plasma/parser/mimeType/odtDetector.java @@ -82,7 +82,7 @@ public class odtDetector implements MagicDetector { // read in the content of the file final InputStream zippedContent = zipFile.getInputStream(mimeTypeInfo); - final String realMimeType = new String(serverFileUtils.read(zippedContent, mimeTypeInfo.getSize())); + final String realMimeType = new String(serverFileUtils.read(zippedContent, (int) mimeTypeInfo.getSize())); return new String[]{realMimeType}; } catch (final Exception e) { diff --git a/source/de/anomic/server/serverFileUtils.java b/source/de/anomic/server/serverFileUtils.java index 52fda469b..857f9ef03 100644 --- a/source/de/anomic/server/serverFileUtils.java +++ b/source/de/anomic/server/serverFileUtils.java @@ -249,15 +249,17 @@ public final class serverFileUtils { return read(source,-1); } - public static byte[] read(final InputStream source, final long count) throws IOException { - final ByteArrayOutputStream baos = (count > 0) - ? new ByteArrayOutputStream((int)count) - : new ByteArrayOutputStream(); - copy(source, baos, count); - baos.close(); - - // convert Stream into array - return baos.toByteArray(); + public static byte[] read(final InputStream source, final int count) throws IOException { + if (count > 0) { + byte[] b = new byte[count]; + source.read(b, 0, count); + return b; + } else { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(512); + copy(source, baos, count); + baos.close(); + return baos.toByteArray(); + } } public static byte[] read(final File source) throws IOException { @@ -331,7 +333,7 @@ public final class serverFileUtils { System.out.println("DEBUG: uncompressGZipArray - uncompressing source"); try { final ByteArrayInputStream byteInput = new ByteArrayInputStream(source); - final ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + final ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(source.length / 5); final GZIPInputStream zippedContent = new GZIPInputStream(byteInput); final byte[] data = new byte[1024]; int read = 0; diff --git a/source/de/anomic/tools/gzip.java b/source/de/anomic/tools/gzip.java index 565d03886..d1ab151a1 100644 --- a/source/de/anomic/tools/gzip.java +++ b/source/de/anomic/tools/gzip.java @@ -75,7 +75,7 @@ public class gzip { public static byte[] gzipString(final String in) { try { final InputStream fin = new ByteArrayInputStream(in.getBytes("UTF8")); - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final ByteArrayOutputStream baos = new ByteArrayOutputStream(in.length() / 3); final OutputStream fout = new GZIPOutputStream(baos, 128); copy(fout, fin, 128); fin.close(); @@ -90,7 +90,7 @@ public class gzip { public static String gunzipString(final byte[] in) throws IOException { final InputStream fin = new GZIPInputStream(new ByteArrayInputStream(in)); - final ByteArrayOutputStream fout = new ByteArrayOutputStream(); + final ByteArrayOutputStream fout = new ByteArrayOutputStream(in.length / 3); copy(fout, fin, 128); fin.close(); fout.close();