From 444dce7e817e535e033219fb4191574101973da2 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 10 Apr 2008 15:28:58 +0000 Subject: [PATCH] more performance hacks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4676 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/IndexCreateIndexingQueue_p.java | 1 + source/de/anomic/http/HttpResponse.java | 18 +++++----- source/de/anomic/http/httpTemplate.java | 5 --- source/de/anomic/http/httpdProxyHandler.java | 7 ++-- source/de/anomic/kelondro/kelondroLock.java | 2 +- source/de/anomic/kelondro/kelondroStack.java | 2 +- source/de/anomic/server/serverFileUtils.java | 38 ++++++++++---------- 7 files changed, 35 insertions(+), 38 deletions(-) diff --git a/htroot/IndexCreateIndexingQueue_p.java b/htroot/IndexCreateIndexingQueue_p.java index 8ec803f27..915f06940 100644 --- a/htroot/IndexCreateIndexingQueue_p.java +++ b/htroot/IndexCreateIndexingQueue_p.java @@ -177,6 +177,7 @@ public class IndexCreateIndexingQueue_p { int j=0; for (int i = switchboard.crawlQueues.errorURL.stackSize() - 1; i >= (switchboard.crawlQueues.errorURL.stackSize() - showRejectedCount); i--) { entry = switchboard.crawlQueues.errorURL.top(i); + if (entry == null) continue; url = entry.url(); if (url == null) continue; diff --git a/source/de/anomic/http/HttpResponse.java b/source/de/anomic/http/HttpResponse.java index 6fb230b00..8a3ea03f3 100644 --- a/source/de/anomic/http/HttpResponse.java +++ b/source/de/anomic/http/HttpResponse.java @@ -27,12 +27,12 @@ package de.anomic.http; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; -import java.io.Writer; import de.anomic.server.serverFileUtils; @@ -103,8 +103,7 @@ public interface HttpResponse { * @throws IOException * @throws UnsupportedEncodingException */ - public static void writeContent(HttpResponse res, BufferedOutputStream hfos, BufferedOutputStream byteStream) throws IOException, - UnsupportedEncodingException { + public static void writeContent(HttpResponse res, BufferedOutputStream hfos, BufferedOutputStream byteStream) throws IOException, UnsupportedEncodingException { try { InputStream data = res.getDataAsStream(); if (byteStream == null) { @@ -112,21 +111,20 @@ public interface HttpResponse { } else { serverFileUtils.copyToStreams(new BufferedInputStream(data), hfos, byteStream); } - - } finally { res.closeStream(); } } - public static void writeContent(HttpResponse res, Writer hfos, OutputStream byteStream) throws IOException, - UnsupportedEncodingException { + public static void writeContent(HttpResponse res, BufferedWriter hfos, OutputStream byteStream) throws IOException, UnsupportedEncodingException { try { InputStream data = res.getDataAsStream(); String charSet = httpHeader.getCharSet(res.getResponseHeader()); - Writer[] writers = (byteStream == null ? new Writer[] { hfos } : new Writer[] { hfos, - new OutputStreamWriter(byteStream, charSet) }); - serverFileUtils.copyToWriters(data, writers, charSet); + if (byteStream == null) { + serverFileUtils.copyToWriter(new BufferedInputStream(data), hfos, charSet); + } else { + serverFileUtils.copyToWriters(new BufferedInputStream(data), hfos, new BufferedWriter(new OutputStreamWriter(byteStream, charSet)) , charSet); + } } finally { res.closeStream(); } diff --git a/source/de/anomic/http/httpTemplate.java b/source/de/anomic/http/httpTemplate.java index 393c5f465..e1e30309f 100644 --- a/source/de/anomic/http/httpTemplate.java +++ b/source/de/anomic/http/httpTemplate.java @@ -283,9 +283,6 @@ public final class httpTemplate { num=0; } //System.out.println(multi_key + ": " + num); //DEBUG - }else{ - //0 interations - no display - //System.out.println("_"+new String(multi_key)+" is null or does not exist"); //DEBUG } //Enumeration enx = pattern.keys(); while (enx.hasMoreElements()) System.out.println("KEY=" + enx.nextElement()); // DEBUG @@ -336,8 +333,6 @@ public final class httpTemplate { byName=true; patternName=patternId.getBytes("UTF-8"); } - }else{ - //System.out.println("Pattern \""+new String(prefix + key)+"\" is not set"); //DEBUG } int currentPattern=0; diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index b3bd6d2a9..6ad26e510 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -64,6 +64,7 @@ package de.anomic.http; import java.io.BufferedOutputStream; import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -634,7 +635,7 @@ public final class httpdProxyHandler { { // ok, we don't write actually into a file, only to RAM, and schedule writing the file. ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - Saver.writeContent(res, hfos, byteStream); + Saver.writeContent(res, new BufferedWriter(hfos), byteStream); // cached bytes byte[] cacheArray; if(byteStream.size() > 0) { @@ -669,7 +670,7 @@ public final class httpdProxyHandler { // the file is too big to cache it in the ram, or the size is unknown // write to file right here. cacheFile.getParentFile().mkdirs(); - Saver.writeContent(res, hfos, new FileOutputStream(cacheFile)); + Saver.writeContent(res, new BufferedWriter(hfos), new FileOutputStream(cacheFile)); if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize(); theLogger.logFine("for write-file of " + url + ": contentLength = " + contentLength + ", sizeBeforeDelete = " + sizeBeforeDelete); plasmaHTCache.writeFileAnnouncement(cacheFile); @@ -699,7 +700,7 @@ public final class httpdProxyHandler { " StoreHTCache=" + storeHTCache + " SupportetContent=" + isSupportedContent); - Saver.writeContent(res, hfos, null); + Saver.writeContent(res, new BufferedWriter(hfos), null); if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize(); if (sizeBeforeDelete == -1) { // no old file and no load. just data passing diff --git a/source/de/anomic/kelondro/kelondroLock.java b/source/de/anomic/kelondro/kelondroLock.java index c5209be53..3253b162c 100644 --- a/source/de/anomic/kelondro/kelondroLock.java +++ b/source/de/anomic/kelondro/kelondroLock.java @@ -95,7 +95,7 @@ public class kelondroLock { if (locked()) { lock = false; releaseTime = 0; - notify(); + notifyAll(); } } diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 6f8ba7a07..f26230238 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -145,7 +145,7 @@ public final class kelondroStack extends kelondroFullRecords { } } - public int size() { + public synchronized int size() { return super.size(); } diff --git a/source/de/anomic/server/serverFileUtils.java b/source/de/anomic/server/serverFileUtils.java index e68949e65..83da9c20c 100644 --- a/source/de/anomic/server/serverFileUtils.java +++ b/source/de/anomic/server/serverFileUtils.java @@ -562,34 +562,36 @@ public final class serverFileUtils { * @return * @throws IOException */ - public static int copyToWriters(final InputStream data, final Writer[] writers, final String charSet) throws IOException { + public static int copyToWriter(final BufferedInputStream data, final BufferedWriter writer, final String charSet) throws IOException { // the docs say: "For top efficiency, consider wrapping an InputStreamReader within a BufferedReader." - final BufferedReader sourceReader = new BufferedReader(new InputStreamReader(data, charSet)); + final Reader sourceReader = new InputStreamReader(data, charSet); - // check if buffer is used. From the documentation: - // "For top efficiency, consider wrapping an OutputStreamWriter within a BufferedWriter so as to avoid frequent - // converter invocations" - int i = 0; - for(final Writer writer: writers) { - if (!(writer instanceof BufferedWriter)) { - // add buffer - writers[i] = new BufferedWriter(writer); - } - i++; + int count = 0; + // copy bytes + int b; + while((b = sourceReader.read()) != -1) { + count++; + writer.write(b); } + writer.flush(); + return count; + } + public static int copyToWriters(final BufferedInputStream data, final BufferedWriter writer0, final BufferedWriter writer1, final String charSet) throws IOException { + // the docs say: "For top efficiency, consider wrapping an InputStreamReader within a BufferedReader." + assert writer0 != null; + assert writer1 != null; + final Reader sourceReader = new InputStreamReader(data, charSet); int count = 0; // copy bytes int b; while((b = sourceReader.read()) != -1) { count++; - for(final Writer writer: writers) { - writer.write(b); - } - } - for(final Writer writer: writers) { - writer.flush(); + writer0.write(b); + writer1.write(b); } + writer0.flush(); + writer1.flush(); return count; } }