- some refactoring (sorry Daniel, hab in deinem Code rumgewütet)

- fixed broken downloads (flush was missing)
- different problem handling when download is corrupted
- different default values in yacy.init

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4669 6c8d7289-2bf4-0310-a012-ef5d649a1542

defaults/yacy.init

@@ -553,8 +553,8 @@ filterOutStopwordsFromTopwords=true
 40_peerseedcycle_busysleep=1200000
 40_peerseedcycle_memprereq=2097152
 50_localcrawl_idlesleep=2000
-50_localcrawl_busysleep=250
-50_localcrawl_busysleep__pro=100
+50_localcrawl_busysleep=50
+50_localcrawl_busysleep__pro=0
 50_localcrawl_memprereq=4194304
 50_localcrawl_isPaused=false
 60_remotecrawlloader_idlesleep=60000
@@ -564,15 +564,15 @@ filterOutStopwordsFromTopwords=true
 60_remotecrawlloader_memprereq=2097152
 60_remotecrawlloader_isPaused=false
 62_remotetriggeredcrawl_idlesleep=10000
-62_remotetriggeredcrawl_busysleep=500
+62_remotetriggeredcrawl_busysleep=1000
 62_remotetriggeredcrawl_memprereq=6291456
 62_remotetriggeredcrawl_isPaused=false
 70_cachemanager_idlesleep=1000
 70_cachemanager_busysleep=0
 70_cachemanager_memprereq=1048576
 80_indexing_idlesleep=1000
-80_indexing_busysleep=200
-80_indexing_busysleep__pro=10
+80_indexing_busysleep=10
+80_indexing_busysleep__pro=0
 80_indexing_memprereq=6291456
 82_crawlstack_idlesleep=5000
 82_crawlstack_busysleep=0
@@ -608,11 +608,10 @@ javastart_priority__pro=0
 # wordCacheMaxLow/High is the number of word indexes that shall be held in the
 # ram cache during indexing. When YaCy is shut down, this cache must be
 # flushed to disc; this may last some minutes.
-wordCacheMaxCount = 20000
-wordCacheInitCount = 20000
-wordCacheMaxCount__pro = 20000
-wordCacheInitCount__pro = 20000
-wordFlushSize__pro = 500
+wordCacheMaxCount = 30000
+wordCacheInitCount = 30000
+wordCacheMaxCount__pro = 60000
+wordCacheInitCount__pro = 60000
 
 # Specifies if yacy can be used as transparent http proxy.
 # 

htroot/ConfigUpdate_p.java

@@ -86,17 +86,17 @@ public class ConfigUpdate_p {
                 yacyVersion updateVersion = yacyVersion.rulebasedUpdateInfo(true);
                 if (updateVersion == null) {
                     prop.put("candeploy_autoUpdate", "2"); // no more recent release found
-                } else try {
+                } else {
                     // there is a version that is more recent. Load it and re-start with it
                     sb.getLog().logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.url);
-                    yacyVersion.downloadRelease(updateVersion);
+                    boolean downloaded = yacyVersion.downloadRelease(updateVersion);
                     prop.put("candeploy_autoUpdate_downloadedRelease", updateVersion.name);
                     File releaseFile = new File(sb.getRootPath(), "DATA/RELEASE/" + updateVersion.name);
                     boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev");
                     if (devenvironment) {
                         sb.getLog().logInfo("AUTO-UPDATE: omiting update because this is a development environment");
                         prop.put("candeploy_autoUpdate", "3");
-                    } else if ((!releaseFile.exists()) || (releaseFile.length() == 0)) {
+                    } else if ((!downloaded) || (!releaseFile.exists()) || (releaseFile.length() == 0)) {
                         sb.getLog().logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found or is too small)");
                         prop.put("candeploy_autoUpdate", "4");
                     } else {
@@ -105,8 +105,6 @@ public class ConfigUpdate_p {
                         sb.getLog().logInfo("AUTO-UPDATE: deploy and restart initiated");
                         prop.put("candeploy_autoUpdate", "1");
                     }
-                } catch (IOException e) {
-                    sb.getLog().logSevere("AUTO-UPDATE: could not download and install release " + updateVersion.url + ": " + e.getMessage());
                 }
             }
          

source/de/anomic/http/HttpResponse.java

@@ -101,22 +101,27 @@ public interface HttpResponse {
          * @throws IOException
          * @throws UnsupportedEncodingException
          */
-        public static void writeContent(HttpResponse res, Object hfos, OutputStream byteStream) throws IOException,
+        public static void writeContent(HttpResponse res, OutputStream hfos, OutputStream byteStream) throws IOException,
                 UnsupportedEncodingException {
             try {
                 InputStream data = res.getDataAsStream();
-                if (hfos instanceof OutputStream) {
                 OutputStream[] streams = (byteStream == null ? new OutputStream[] { (OutputStream) hfos }
                         : new OutputStream[] { (OutputStream) hfos, byteStream });
                 serverFileUtils.copyToStreams(data, streams);
-                } else if (hfos instanceof Saver) {
+                
+            } finally {
+                res.closeStream();
+            }
+        }
+        
+        public static void writeContent(HttpResponse res, Writer hfos, OutputStream byteStream) throws IOException,
+        UnsupportedEncodingException {
+            try {
+                InputStream data = res.getDataAsStream();
                 String charSet = httpHeader.getCharSet(res.getResponseHeader());
-                    Writer[] writers = (byteStream == null ? new Writer[] { (Writer) hfos } : new Writer[] { (Writer) hfos,
+                Writer[] writers = (byteStream == null ? new Writer[] { hfos } : new Writer[] { hfos,
                         new OutputStreamWriter(byteStream, charSet) });
                 serverFileUtils.copyToWriters(data, writers, charSet);
-                } else {
-                    throw new IOException("cannot save data: hfos-type ("+  hfos.getClass().toString() +") not supported!");
-                }
             } finally {
                 res.closeStream();
             }

source/de/anomic/http/JakartaCommonsHttpClient.java

@@ -78,7 +78,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
 
     static {
         // set user-agent
-        apacheHttpClient.getParams().setParameter(HttpClientParams.USER_AGENT,
+        apacheHttpClient.getParams().setParameter(HttpMethodParams.USER_AGENT,
                                                   "yacy/" + yacyVersion.thisVersion().releaseNr +
                                                           " (www.yacy.net; " +
                                                           de.anomic.http.HttpClient.getSystemOST() + ") " +
@@ -505,7 +505,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
      * @return
      */
     public static String getCurrentUserAgent() {
-        return (String) apacheHttpClient.getParams().getParameter(HttpClientParams.USER_AGENT);
+        return (String) apacheHttpClient.getParams().getParameter(HttpMethodParams.USER_AGENT);
     }
 
     /**

source/de/anomic/http/JakartaCommonsHttpResponse.java

@@ -54,7 +54,7 @@ public class JakartaCommonsHttpResponse implements HttpResponse {
      * @param method
      * @throws IOException
      */
-    public JakartaCommonsHttpResponse(final HttpMethod method) throws IOException {
+    public JakartaCommonsHttpResponse(final HttpMethod method) {
         super();
 
         this.method = method;

source/de/anomic/http/httpdProxyHandler.java

@@ -71,6 +71,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.io.Writer;
 import java.net.BindException;
@@ -481,7 +482,7 @@ public final class httpdProxyHandler {
         
         GZIPOutputStream gzippedOut = null; 
         httpChunkedOutputStream chunkedOut = null;
-        Object hfos = null;
+        Writer hfos = null;
         
         HttpResponse res = null;                
         try {
@@ -590,7 +591,7 @@ public final class httpdProxyHandler {
             } else {
                 // simply pass through without parsing
                 theLogger.logFine("create passthrough for URL " + url + ", extension '" + ext + "', mime-type '" + responseHeader.mime() + "'");
-                hfos = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
+                hfos = new OutputStreamWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), httpHeader.getCharSet(res.getResponseHeader()));
             }
             
             // handle incoming cookies

source/de/anomic/index/indexContainerHeap.java

@@ -61,7 +61,7 @@ public final class indexContainerHeap {
     private kelondroRow payloadrow;
     private serverLog log;
     private kelondroBytesLongMap index;
-    private SortedMap<String, indexContainer> cache;
+    SortedMap<String, indexContainer> cache;
     private File backupFile;
     private boolean readOnlyMode;
     // index xor cache is used. If one is not null, then the other must be null

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -2003,29 +2003,27 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
                 		deleteQueue.add(seed.hash);
                 	}
                 }
-                for(int i=0;i<deleteQueue.size();++i) yacyCore.seedDB.removePotential((String)deleteQueue.get(i));
+                for (int i = 0; i < deleteQueue.size(); ++i) yacyCore.seedDB.removePotential((String)deleteQueue.get(i));
             }
             
             // check if update is available and
             // if auto-update is activated perform an automatic installation and restart
             yacyVersion updateVersion = yacyVersion.rulebasedUpdateInfo(false);
-            if (updateVersion != null) try {
+            if (updateVersion != null) {
                 // there is a version that is more recent. Load it and re-start with it
                 log.logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.url);
-                yacyVersion.downloadRelease(updateVersion);
+                boolean downloaded = yacyVersion.downloadRelease(updateVersion);
                 File releaseFile = new File(sb.getRootPath(), "DATA/RELEASE/" + updateVersion.name);
                 boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev");
                 if (devenvironment) {
                     log.logInfo("AUTO-UPDATE: omiting update because this is a development environment");
-                } else if ((!releaseFile.exists()) || (releaseFile.length() == 0)) {
+                } else if ((!downloaded) || (!releaseFile.exists()) || (releaseFile.length() == 0)) {
                     log.logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found or is too small)");
                 } else {
                     yacyVersion.deployRelease(updateVersion.name);
                     terminate(5000);
                     log.logInfo("AUTO-UPDATE: deploy and restart initiated");
                 }
-            } catch (IOException e) {
-                log.logSevere("AUTO-UPDATE: could not download and install release " + updateVersion.url + ": " + e.getMessage());
             }
             
             // initiate broadcast about peer startup to spread supporter url

source/de/anomic/server/serverCore.java

@@ -300,7 +300,7 @@ public final class serverCore extends serverAbstractBusyThread implements server
             if (bindIP.startsWith("#")) {
                 String interfaceName = bindIP.substring(1);
                 String hostName = null;
-                if (this.log.isFine()) this.log.logFine("Trying to determine IP address of interface '" + interfaceName + "'.");                    
+                this.log.logFine("Trying to determine IP address of interface '" + interfaceName + "'.");                    
 
                 Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
                 if (interfaces != null) {
@@ -402,7 +402,7 @@ public final class serverCore extends serverAbstractBusyThread implements server
             // prepare for new connection
             // idleThreadCheck();
             this.switchboard.handleBusyState(this.busySessions.size());
-            if (log.isFinest()) this.log.logFinest("* waiting for connections, " + this.busySessions.size() + " sessions running");
+            this.log.logFinest("* waiting for connections, " + this.busySessions.size() + " sessions running");
                         
             announceThreadBlockApply();
             
@@ -480,7 +480,8 @@ public final class serverCore extends serverAbstractBusyThread implements server
         Thread.interrupted();
         
         // shut down all busySessions
-        if (this.busySessions != null) for (Session session: this.busySessions) {
+        for (Session session: this.busySessions) {
+            try {session.notifyAll();} catch (IllegalMonitorStateException e) {e.printStackTrace();}
             try {session.interrupt();} catch (SecurityException e ) {e.printStackTrace();}
         }
         
@@ -539,7 +540,7 @@ public final class serverCore extends serverAbstractBusyThread implements server
     public final class Session extends Thread {
 
         boolean destroyed = false;
-        private boolean running = false;
+        private boolean runningsession = false;
         private boolean stopped = false;
         
         private long start;                // startup time
@@ -566,7 +567,7 @@ public final class serverCore extends serverAbstractBusyThread implements server
             this.hashIndex = sessionCounter;
             sessionCounter++;
             
-            if (!this.running)  {
+            if (!this.runningsession)  {
                // this.setDaemon(true);
                this.start();
             }  else { 
@@ -638,7 +639,7 @@ public final class serverCore extends serverAbstractBusyThread implements server
     	*/
     
     	public void log(boolean outgoing, String request) {
-    	    if (log.isFine()) log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " +
+    	    serverCore.this.log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " +
     		     "[" + ((busySessions == null)? -1 : busySessions.size()) + ", " + this.commandCounter +
     		     ((outgoing) ? "] > " : "] < ") +
     		     request);
@@ -667,7 +668,7 @@ public final class serverCore extends serverAbstractBusyThread implements server
          * @return whether the {@link Thread} is currently running
          */
         public boolean isRunning() {
-            return this.running;
+            return this.runningsession;
         }
         
         /**
@@ -676,7 +677,7 @@ public final class serverCore extends serverAbstractBusyThread implements server
          * @see java.lang.Thread#run()
          */
         public void run()  {
-            this.running = true;
+            this.runningsession = true;
             
             try {
                 // setting the session startup time

source/de/anomic/server/serverFileUtils.java

@@ -542,6 +542,9 @@ public final class serverFileUtils {
                 out.write(b);
             }
         }
+        for(final OutputStream out: outs) {
+            out.flush();
+        }
         return count;
     }
 
@@ -578,6 +581,9 @@ public final class serverFileUtils {
                 writer.write(b);
             }
         }
+        for(final Writer writer: writers) {
+            writer.flush();
+        }
         return count;
     }
 }

source/de/anomic/yacy/yacyVersion.java

@@ -324,23 +324,27 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
         return new DevMain(devreleases, mainreleases);
     }
     
-    public static void downloadRelease(yacyVersion release) throws IOException {
+    public static boolean downloadRelease(yacyVersion release) {
         File storagePath = plasmaSwitchboard.getSwitchboard().releasePath;
         // load file
         File download = new File(storagePath, release.url.getFileName());
-        HttpClient client = HttpFactory.newClient(null, 30000);
+        HttpClient client = HttpFactory.newClient(null, 60000);
         HttpResponse res = null;
         try {
             res = client.GET(release.url.toString());
             Saver.writeContent(res, new FileOutputStream(download), null);
+            if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + release.url + " failed");
+        } catch (IOException e) {
+            serverLog.logSevere("yacyVersion", "download of " + release.name + " failed: " + e.getMessage());
+            if (download.exists()) download.delete();
         } finally {
-            if(res != null) {
+            if (res != null) {
                 // release connection
                 res.closeStream();
             }
         }
-        if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + release.url + " failed");
         plasmaSwitchboard.getSwitchboard().setConfig("update.time.download", System.currentTimeMillis());
+        return download.exists();
     }