From 2c1c3bb6ebf4b8d3f3eb311428bd96a97438c552 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 8 Apr 2008 21:36:33 +0000 Subject: [PATCH] =?UTF-8?q?-=20some=20refactoring=20(sorry=20Daniel,=20hab?= =?UTF-8?q?=20in=20deinem=20Code=20rumgew=C3=BCtet)=20-=20fixed=20broken?= =?UTF-8?q?=20downloads=20(flush=20was=20missing)=20-=20different=20proble?= =?UTF-8?q?m=20handling=20when=20download=20is=20corrupted=20-=20different?= =?UTF-8?q?=20default=20values=20in=20yacy.init?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4669 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- defaults/yacy.init | 19 ++++++------ htroot/ConfigUpdate_p.java | 8 ++--- source/de/anomic/http/HttpResponse.java | 31 +++++++++++-------- .../anomic/http/JakartaCommonsHttpClient.java | 4 +-- .../http/JakartaCommonsHttpResponse.java | 2 +- source/de/anomic/http/httpdProxyHandler.java | 5 +-- .../de/anomic/index/indexContainerHeap.java | 2 +- .../de/anomic/plasma/plasmaSwitchboard.java | 10 +++--- source/de/anomic/server/serverCore.java | 17 +++++----- source/de/anomic/server/serverFileUtils.java | 6 ++++ source/de/anomic/yacy/yacyVersion.java | 12 ++++--- 11 files changed, 64 insertions(+), 52 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 3bd1bec15..b15082f0a 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -553,8 +553,8 @@ filterOutStopwordsFromTopwords=true 40_peerseedcycle_busysleep=1200000 40_peerseedcycle_memprereq=2097152 50_localcrawl_idlesleep=2000 -50_localcrawl_busysleep=250 -50_localcrawl_busysleep__pro=100 +50_localcrawl_busysleep=50 +50_localcrawl_busysleep__pro=0 50_localcrawl_memprereq=4194304 50_localcrawl_isPaused=false 60_remotecrawlloader_idlesleep=60000 @@ -564,15 +564,15 @@ filterOutStopwordsFromTopwords=true 60_remotecrawlloader_memprereq=2097152 60_remotecrawlloader_isPaused=false 62_remotetriggeredcrawl_idlesleep=10000 -62_remotetriggeredcrawl_busysleep=500 +62_remotetriggeredcrawl_busysleep=1000 62_remotetriggeredcrawl_memprereq=6291456 62_remotetriggeredcrawl_isPaused=false 70_cachemanager_idlesleep=1000 70_cachemanager_busysleep=0 70_cachemanager_memprereq=1048576 80_indexing_idlesleep=1000 -80_indexing_busysleep=200 -80_indexing_busysleep__pro=10 +80_indexing_busysleep=10 +80_indexing_busysleep__pro=0 80_indexing_memprereq=6291456 82_crawlstack_idlesleep=5000 82_crawlstack_busysleep=0 @@ -608,11 +608,10 @@ javastart_priority__pro=0 # wordCacheMaxLow/High is the number of word indexes that shall be held in the # ram cache during indexing. When YaCy is shut down, this cache must be # flushed to disc; this may last some minutes. -wordCacheMaxCount = 20000 -wordCacheInitCount = 20000 -wordCacheMaxCount__pro = 20000 -wordCacheInitCount__pro = 20000 -wordFlushSize__pro = 500 +wordCacheMaxCount = 30000 +wordCacheInitCount = 30000 +wordCacheMaxCount__pro = 60000 +wordCacheInitCount__pro = 60000 # Specifies if yacy can be used as transparent http proxy. # diff --git a/htroot/ConfigUpdate_p.java b/htroot/ConfigUpdate_p.java index a557c193c..8ec5894cb 100644 --- a/htroot/ConfigUpdate_p.java +++ b/htroot/ConfigUpdate_p.java @@ -86,17 +86,17 @@ public class ConfigUpdate_p { yacyVersion updateVersion = yacyVersion.rulebasedUpdateInfo(true); if (updateVersion == null) { prop.put("candeploy_autoUpdate", "2"); // no more recent release found - } else try { + } else { // there is a version that is more recent. Load it and re-start with it sb.getLog().logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.url); - yacyVersion.downloadRelease(updateVersion); + boolean downloaded = yacyVersion.downloadRelease(updateVersion); prop.put("candeploy_autoUpdate_downloadedRelease", updateVersion.name); File releaseFile = new File(sb.getRootPath(), "DATA/RELEASE/" + updateVersion.name); boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev"); if (devenvironment) { sb.getLog().logInfo("AUTO-UPDATE: omiting update because this is a development environment"); prop.put("candeploy_autoUpdate", "3"); - } else if ((!releaseFile.exists()) || (releaseFile.length() == 0)) { + } else if ((!downloaded) || (!releaseFile.exists()) || (releaseFile.length() == 0)) { sb.getLog().logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found or is too small)"); prop.put("candeploy_autoUpdate", "4"); } else { @@ -105,8 +105,6 @@ public class ConfigUpdate_p { sb.getLog().logInfo("AUTO-UPDATE: deploy and restart initiated"); prop.put("candeploy_autoUpdate", "1"); } - } catch (IOException e) { - sb.getLog().logSevere("AUTO-UPDATE: could not download and install release " + updateVersion.url + ": " + e.getMessage()); } } diff --git a/source/de/anomic/http/HttpResponse.java b/source/de/anomic/http/HttpResponse.java index dbb3e5d94..331649766 100644 --- a/source/de/anomic/http/HttpResponse.java +++ b/source/de/anomic/http/HttpResponse.java @@ -101,22 +101,27 @@ public interface HttpResponse { * @throws IOException * @throws UnsupportedEncodingException */ - public static void writeContent(HttpResponse res, Object hfos, OutputStream byteStream) throws IOException, + public static void writeContent(HttpResponse res, OutputStream hfos, OutputStream byteStream) throws IOException, UnsupportedEncodingException { try { InputStream data = res.getDataAsStream(); - if (hfos instanceof OutputStream) { - OutputStream[] streams = (byteStream == null ? new OutputStream[] { (OutputStream) hfos } - : new OutputStream[] { (OutputStream) hfos, byteStream }); - serverFileUtils.copyToStreams(data, streams); - } else if (hfos instanceof Saver) { - String charSet = httpHeader.getCharSet(res.getResponseHeader()); - Writer[] writers = (byteStream == null ? new Writer[] { (Writer) hfos } : new Writer[] { (Writer) hfos, - new OutputStreamWriter(byteStream, charSet) }); - serverFileUtils.copyToWriters(data, writers, charSet); - } else { - throw new IOException("cannot save data: hfos-type ("+ hfos.getClass().toString() +") not supported!"); - } + OutputStream[] streams = (byteStream == null ? new OutputStream[] { (OutputStream) hfos } + : new OutputStream[] { (OutputStream) hfos, byteStream }); + serverFileUtils.copyToStreams(data, streams); + + } finally { + res.closeStream(); + } + } + + public static void writeContent(HttpResponse res, Writer hfos, OutputStream byteStream) throws IOException, + UnsupportedEncodingException { + try { + InputStream data = res.getDataAsStream(); + String charSet = httpHeader.getCharSet(res.getResponseHeader()); + Writer[] writers = (byteStream == null ? new Writer[] { hfos } : new Writer[] { hfos, + new OutputStreamWriter(byteStream, charSet) }); + serverFileUtils.copyToWriters(data, writers, charSet); } finally { res.closeStream(); } diff --git a/source/de/anomic/http/JakartaCommonsHttpClient.java b/source/de/anomic/http/JakartaCommonsHttpClient.java index d8ae9ae8a..7ac6a7c4f 100644 --- a/source/de/anomic/http/JakartaCommonsHttpClient.java +++ b/source/de/anomic/http/JakartaCommonsHttpClient.java @@ -78,7 +78,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { static { // set user-agent - apacheHttpClient.getParams().setParameter(HttpClientParams.USER_AGENT, + apacheHttpClient.getParams().setParameter(HttpMethodParams.USER_AGENT, "yacy/" + yacyVersion.thisVersion().releaseNr + " (www.yacy.net; " + de.anomic.http.HttpClient.getSystemOST() + ") " + @@ -505,7 +505,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * @return */ public static String getCurrentUserAgent() { - return (String) apacheHttpClient.getParams().getParameter(HttpClientParams.USER_AGENT); + return (String) apacheHttpClient.getParams().getParameter(HttpMethodParams.USER_AGENT); } /** diff --git a/source/de/anomic/http/JakartaCommonsHttpResponse.java b/source/de/anomic/http/JakartaCommonsHttpResponse.java index 0ad2e269f..6f7d2c431 100644 --- a/source/de/anomic/http/JakartaCommonsHttpResponse.java +++ b/source/de/anomic/http/JakartaCommonsHttpResponse.java @@ -54,7 +54,7 @@ public class JakartaCommonsHttpResponse implements HttpResponse { * @param method * @throws IOException */ - public JakartaCommonsHttpResponse(final HttpMethod method) throws IOException { + public JakartaCommonsHttpResponse(final HttpMethod method) { super(); this.method = method; diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 2d4d231f8..4290e7e42 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -71,6 +71,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.Writer; import java.net.BindException; @@ -481,7 +482,7 @@ public final class httpdProxyHandler { GZIPOutputStream gzippedOut = null; httpChunkedOutputStream chunkedOut = null; - Object hfos = null; + Writer hfos = null; HttpResponse res = null; try { @@ -590,7 +591,7 @@ public final class httpdProxyHandler { } else { // simply pass through without parsing theLogger.logFine("create passthrough for URL " + url + ", extension '" + ext + "', mime-type '" + responseHeader.mime() + "'"); - hfos = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond); + hfos = new OutputStreamWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), httpHeader.getCharSet(res.getResponseHeader())); } // handle incoming cookies diff --git a/source/de/anomic/index/indexContainerHeap.java b/source/de/anomic/index/indexContainerHeap.java index 4019cbb1a..c2c81cc7f 100755 --- a/source/de/anomic/index/indexContainerHeap.java +++ b/source/de/anomic/index/indexContainerHeap.java @@ -61,7 +61,7 @@ public final class indexContainerHeap { private kelondroRow payloadrow; private serverLog log; private kelondroBytesLongMap index; - private SortedMap cache; + SortedMap cache; private File backupFile; private boolean readOnlyMode; // index xor cache is used. If one is not null, then the other must be null diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index dad0699ca..bab13f471 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -2003,29 +2003,27 @@ public final class plasmaSwitchboard extends serverAbstractSwitch interfaces = NetworkInterface.getNetworkInterfaces(); if (interfaces != null) { @@ -402,7 +402,7 @@ public final class serverCore extends serverAbstractBusyThread implements server // prepare for new connection // idleThreadCheck(); this.switchboard.handleBusyState(this.busySessions.size()); - if (log.isFinest()) this.log.logFinest("* waiting for connections, " + this.busySessions.size() + " sessions running"); + this.log.logFinest("* waiting for connections, " + this.busySessions.size() + " sessions running"); announceThreadBlockApply(); @@ -480,7 +480,8 @@ public final class serverCore extends serverAbstractBusyThread implements server Thread.interrupted(); // shut down all busySessions - if (this.busySessions != null) for (Session session: this.busySessions) { + for (Session session: this.busySessions) { + try {session.notifyAll();} catch (IllegalMonitorStateException e) {e.printStackTrace();} try {session.interrupt();} catch (SecurityException e ) {e.printStackTrace();} } @@ -539,7 +540,7 @@ public final class serverCore extends serverAbstractBusyThread implements server public final class Session extends Thread { boolean destroyed = false; - private boolean running = false; + private boolean runningsession = false; private boolean stopped = false; private long start; // startup time @@ -566,7 +567,7 @@ public final class serverCore extends serverAbstractBusyThread implements server this.hashIndex = sessionCounter; sessionCounter++; - if (!this.running) { + if (!this.runningsession) { // this.setDaemon(true); this.start(); } else { @@ -638,7 +639,7 @@ public final class serverCore extends serverAbstractBusyThread implements server */ public void log(boolean outgoing, String request) { - if (log.isFine()) log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " + + serverCore.this.log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " + "[" + ((busySessions == null)? -1 : busySessions.size()) + ", " + this.commandCounter + ((outgoing) ? "] > " : "] < ") + request); @@ -667,7 +668,7 @@ public final class serverCore extends serverAbstractBusyThread implements server * @return whether the {@link Thread} is currently running */ public boolean isRunning() { - return this.running; + return this.runningsession; } /** @@ -676,7 +677,7 @@ public final class serverCore extends serverAbstractBusyThread implements server * @see java.lang.Thread#run() */ public void run() { - this.running = true; + this.runningsession = true; try { // setting the session startup time diff --git a/source/de/anomic/server/serverFileUtils.java b/source/de/anomic/server/serverFileUtils.java index 75be2a2b2..f3ac76f4d 100644 --- a/source/de/anomic/server/serverFileUtils.java +++ b/source/de/anomic/server/serverFileUtils.java @@ -542,6 +542,9 @@ public final class serverFileUtils { out.write(b); } } + for(final OutputStream out: outs) { + out.flush(); + } return count; } @@ -578,6 +581,9 @@ public final class serverFileUtils { writer.write(b); } } + for(final Writer writer: writers) { + writer.flush(); + } return count; } } diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index c6b12efda..e755b612d 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -324,23 +324,27 @@ public final class yacyVersion implements Comparator, Comparable