*) shrinking httpc linebuffer when httpc is returned to pool. This is done to free memory

*) Making Seed-Upload configuration more verbose.
*) Some Changes in SOAP Search API (not finished yet).

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@158 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/SettingsAck_p.html

@@ -55,6 +55,7 @@ You are now permanently <b>online</b>. After a short while you should see the ef
 <b>Seed Settings changed.#(success)#::You are now a principal peer.#(/success)#</b><br>
 ::
 <p><b>Seed Settings changed, but something is wrong.</b></p>
+<p><font color="red">#[errormsg]#</font></p>
 Seed Uploading was deactivated automatically. 
 Please return to the settings page and modify the data.<br>
 ::

htroot/SettingsAck_p.java

@@ -275,16 +275,14 @@ public class SettingsAck_p {
                     env.setConfig("seedUploadMethod", newSeedUploadMethod);
                     env.setConfig("seedURL", newSeedURLStr);
                     
-                    boolean success = yacyCore.saveSeedList(env);
-                    if (!success) {
-                        prop.put("info", 14);
-                        env.setConfig("seedUploadMethod","none");
-                    } else {
+                    // trying to upload the seed-list file
+                    yacyCore.saveSeedList(env);
+                    
+                    // we have successfully uploaded the seed-list file
                     prop.put("info_seedUploadMethod",newSeedUploadMethod);
                     prop.put("info_seedURL",newSeedURLStr);
                     prop.put("info_success",(newSeedUploadMethod.equalsIgnoreCase("none")?0:1));
                     prop.put("info", 19);                                     
-                    }                  
                 } else {
                     prop.put("info_seedUploadMethod",newSeedUploadMethod);
                     prop.put("info_seedURL",newSeedURLStr);
@@ -294,6 +292,8 @@ public class SettingsAck_p {
                 }
             } catch (Exception e) {
                 prop.put("info",14);
+                prop.put("info_errormsg",e.getMessage().replaceAll("\n","<br>"));                
+                env.setConfig("seedUploadMethod","none"); 
             }
             return prop;            
         }
@@ -308,6 +308,7 @@ public class SettingsAck_p {
             // getting the uploader module name
             String uploaderName = (String) uploaderKeys.nextElement();
             
+            
             // determining if the user has reconfigured the settings of this uploader
             if (post.containsKey("seed" + uploaderName + "Settings")) {
                 nothingChanged = true;
@@ -324,14 +325,20 @@ public class SettingsAck_p {
                     }
                 }   
                 if (!nothingChanged) {
-                    if (env.getConfig("seedUploadMethod","none").equals(theUploader)) {
-                        boolean success = yacyCore.saveSeedList(env);
-                        if (!success) {
-                            prop.put("info", 14);
-                            env.setConfig("seedUploadMethod","none");
-                        } else {
+                    // if the seed upload method is equal to the seed uploader whose settings
+                    // were changed, we now try to upload the seed list with the new settings
+                    if (env.getConfig("seedUploadMethod","none").equalsIgnoreCase(uploaderName)) {
+                        try {
+                            yacyCore.saveSeedList(env);
+                            
+                            // we have successfully uploaded the seed file
                             prop.put("info", 13);
                             prop.put("info_success",1);
+                        } catch (Exception e) {
+                            // if uploading failed we print out an error message
+                            prop.put("info", 14);
+                            prop.put("info_errormsg",e.getMessage().replaceAll("\n","<br>"));
+                            env.setConfig("seedUploadMethod","none");                            
                         }                       
                     } else {
                         prop.put("info", 13);

source/de/anomic/http/httpc.java

@@ -285,6 +285,10 @@ public final class httpc {
             this.remoteProxyUse = false;
             this.savedRemoteHost = null;
             this.requestPath = null;
+            
+            
+            // shrink readlinebuffer if it is to large
+            this.readLineBuffer.reset(80);
         } catch (Exception e) {
             // we could ignore this ...
         }
@@ -306,6 +310,7 @@ public final class httpc {
         this.remoteProxyUse = false;
         this.timeout = timeout;
         this.savedRemoteHost = server;
+
         try {
             this.host = server + ((port == 80) ? "" : (":" + port));
             String hostip;

source/de/anomic/http/httpd.java

@@ -52,6 +52,7 @@ package de.anomic.http;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PushbackInputStream;
+import java.lang.reflect.Constructor;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.util.Arrays;
@@ -262,7 +263,9 @@ public final class httpd implements serverHandler {
             if (this.prop.containsKey("PATH") && this.prop.getProperty("PATH").startsWith("/soap")) {
                 if (soapHandler == null) {
                     try {
-                        soapHandler  = (httpdHandler) Class.forName("de.anomic.soap.httpdSoapHandler").newInstance();
+                        Class soapHandlerClass = Class.forName("de.anomic.soap.httpdSoapHandler");
+                        Constructor classConstructor = soapHandlerClass.getConstructor( new Class[] { serverSwitch.class } );
+                        soapHandler  = (httpdHandler) classConstructor.newInstance(new Object[] { this.switchboard });
                     } catch (Exception e) {
                         throw new IOException("Unable to load the soap handler");
                     }

source/de/anomic/http/httpdHandler.java

@@ -57,6 +57,8 @@ import java.io.OutputStream;
 import java.io.PushbackInputStream;
 import java.util.Properties;
 
+import de.anomic.server.serverSwitch;
+
 public interface httpdHandler {
 
     void doGet(Properties conProp, httpHeader header, OutputStream response) throws IOException;

source/de/anomic/server/serverByteBuffer.java

@@ -307,6 +307,18 @@ public final class serverByteBuffer extends OutputStream {
         this.offset = 0;
     }        
     
+    public void reset(int newSize) {  
+        this.resize(newSize);
+        this.reset();
+    }         
+     
+    public void resize(int newSize) {
+        if(newSize < 0) throw new IllegalArgumentException("Illegal array size: " + newSize);
+        byte[] v = new byte[newSize];
+        System.arraycopy(this.buffer,0,v,0,newSize > this.buffer.length ? this.buffer.length : newSize);
+        this.buffer = v;          
+    }
+    
     public byte toByteArray()[] {
         byte newbuf[] = new byte[this.length];
         System.arraycopy(this.buffer, 0, newbuf, 0, this.length);

source/de/anomic/soap/httpdSoapHandler.java

@@ -60,7 +60,7 @@ import de.anomic.server.serverSwitch;
  *  
  * @author Martin Thelian
  */
-final class httpdSoapHandler extends httpdAbstractHandler implements httpdHandler 
+public final class httpdSoapHandler extends httpdAbstractHandler implements httpdHandler 
 {
  
     /* ===============================================================
@@ -431,7 +431,7 @@ final class httpdSoapHandler extends httpdAbstractHandler implements httpdHandle
             msgContext.setProperty(Constants.MC_RELATIVE_PATH, path.toString());
             msgContext.setProperty(Constants.MC_JWS_CLASSDIR,  "jwsClasses");
             msgContext.setProperty(Constants.MC_HOME_DIR,      "."); 
-            msgContext.setProperty(MessageContext.TRANS_URL, "http://" + requestHeader.get("Host") + this.switchboard.getConfig("port","8080") + "/soap/index");
+            msgContext.setProperty(MessageContext.TRANS_URL, "http://" + requestHeader.get("Host") + ((((String)requestHeader.get("Host")).indexOf(":") > -1)?"":this.switchboard.getConfig("port","8080")) + "/soap/index");
             
             msgContext.setProperty(MESSAGE_CONTEXT_HTTP_ROOT_PATH  ,this.htRootPath.toString());
             msgContext.setProperty(MESSAGE_CONTEXT_SERVER_SWITCH,this.switchboard);

source/de/anomic/soap/httpdSoapService.java

@@ -4,21 +4,26 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.StringReader;
 import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
 import org.apache.axis.AxisFault;
 import org.apache.axis.Message;
 import org.apache.axis.MessageContext;
 import org.apache.axis.message.SOAPEnvelope;
 import org.apache.axis.message.SOAPHeaderElement;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
 
 import de.anomic.http.httpHeader;
 import de.anomic.http.httpTemplate;
 import de.anomic.server.serverClassLoader;
-import de.anomic.server.serverCodings;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 
@@ -27,7 +32,7 @@ import de.anomic.server.serverSwitch;
  * 
  * @author Martin Thelian
  */
-final public class httpdSoapService
+public class httpdSoapService
 {
     /* ================================================================
      * Constants needed to set the template that should be used to 
@@ -36,7 +41,7 @@ final public class httpdSoapService
     /**
      * Constant: template for searching
      */
-    private static final String TEMPLATE_SEARCH = "index.rss";
+    private static final String TEMPLATE_SEARCH = "index.soap";
     /**
      * Constant: template for the network status page
      */    
@@ -81,7 +86,14 @@ final public class httpdSoapService
      * 
      * @throws AxisFault if the service could not be executed propery. 
      */
-    public String search(String searchString) 
+    public Document search(
+                String searchString,
+                String searchMode,
+                String searchOrder,
+                int maxSearchCount,
+                int maxSearchTime,
+                String urlMaskFilter            
+            ) 
         throws AxisFault
     {        
         try
@@ -89,14 +101,20 @@ final public class httpdSoapService
             // extracting the message context
             extractMessageContext();
             
+            if ((searchMode == null) || !(searchMode.equalsIgnoreCase("global") || searchMode.equalsIgnoreCase("locale"))) {
+                searchMode = "global";
+            }
+            
+            if (urlMaskFilter == null) urlMaskFilter = ".*";
+            
             // setting the searching properties
             serverObjects args = new serverObjects();
             args.put("order","Quality-Date");
             args.put("Enter","Search");
-            args.put("count","10");
+            args.put("count",Integer.toString(maxSearchCount));
             args.put("resource","global");
-            args.put("time","10");
-            args.put("urlmaskfilter",".*");
+            args.put("time",Integer.toString(maxSearchTime));
+            args.put("urlmaskfilter",urlMaskFilter);
             
             args.put("search",searchString);
             
@@ -104,7 +122,7 @@ final public class httpdSoapService
             String result = writeTemplate(TEMPLATE_SEARCH, args);
             
             // sending back the result to the client
-            return result;
+            return this.convertContentToXML(result);
         }
         catch (Exception e)
         {
@@ -336,4 +354,25 @@ final public class httpdSoapService
         return m;
         }        
     
+    private Document convertContentToXML(String contentString) 
+    throws Exception
+    {
+        Document doc = null;
+        try
+        {
+            DocumentBuilderFactory newDocBuilderFactory = DocumentBuilderFactory.newInstance();
+            DocumentBuilder newDocBuilder = newDocBuilderFactory.newDocumentBuilder();          
+            
+            InputSource is = new InputSource(new StringReader(contentString));
+            doc = newDocBuilder.parse(is);                      
+        }       
+        catch (Exception e)
+        {
+            String errorMessage = "Unable to parse the search result XML data. " + e.getClass().getName() + ". " + e.getMessage();
+            throw new Exception(errorMessage);
+        }       
+        
+        return doc;
+    }    
+    
 }

source/de/anomic/yacy/seedUpload/yacySeedUploadFile.java

@@ -12,19 +12,19 @@ public class yacySeedUploadFile implements yacySeedUploader {
     
     public static final String CONFIG_FILE_PATH = "seedFilePath";
 
-    public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) {
+    public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) throws Exception {
         
-        String logt;
+        String logt, seedFilePath = "";
         try {
-            String seedFilePath = sb.getConfig(CONFIG_FILE_PATH,null);
-            if (seedFilePath == null) return "Error: Path to seed file is not configured properly";
+            seedFilePath = sb.getConfig(CONFIG_FILE_PATH,"");
+            if (seedFilePath.length() == 0) throw new Exception("Path to seed file is not configured properly");
             
             File publicSeedFile = new File(seedFilePath);            
             serverFileUtils.copy(seedFile,publicSeedFile);
             
             return "Seed-List file stored successfully";
         } catch (Exception e) {
-            return "Error: " + e.getMessage();
+            throw new Exception("Unable to store the seed-list file into the filesystem using path '" + seedFilePath + "'. " + e.getMessage());
         }
     }
 

source/de/anomic/yacy/seedUpload/yacySeedUploadScp.java

@@ -24,7 +24,7 @@ public class yacySeedUploadScp implements yacySeedUploader {
     public static final String CONFIG_SCP_PASSWORD = "seedScpPassword";
     public static final String CONFIG_SCP_PATH = "seedScpPath";
     
-    public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) {
+    public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) throws Exception {
         try {        
             if (sb == null) throw new NullPointerException("Reference to serverSwitch nut not be null.");
             if (seedDB == null) throw new NullPointerException("Reference to seedDB must not be null.");
@@ -36,14 +36,13 @@ public class yacySeedUploadScp implements yacySeedUploader {
             String  seedScpPath = sb.getConfig(CONFIG_SCP_PATH,null);       
             
             if ((seedScpServer != null) && (seedScpAccount != null) && (seedScpPassword != null) && (seedScpPath != null)) {
-                String log = sshc.put(seedScpServer, seedFile, seedScpPath, seedScpAccount, seedScpPassword);
-                return "Seed file uploaded successfully using scp. " + log;
+                return sshc.put(seedScpServer, seedFile, seedScpPath, seedScpAccount, seedScpPassword);
             } 
             return "Seed upload settings not configured properly. password-len=" +
             seedScpPassword.length() + ", filePath=" +
             seedScpPath;
         } catch (Exception e) {
-            return "Error: " + e.getMessage();
+            throw e;
         }
     }
     
@@ -60,7 +59,7 @@ public class yacySeedUploadScp implements yacySeedUploader {
 class sshc {
     public static String put(String host,
             File localFile, String remoteName,
-               String account, String password) {
+            String account, String password) throws Exception {
         
         Session session = null;
         try {    
@@ -130,15 +129,11 @@ class sshc {
             
             checkAck(in);       
             
-        return "";
+            return "SCP: File uploaded successfully.";
         } catch (Exception e) {
-        System.out.println("ERROR: scp put failed:" + e.getMessage());
-        e.printStackTrace();
-        return "";
-    } finally
-    {
-        if ((session != null) && (session.isConnected()))
-            session.disconnect();
+            throw new Exception("SCP: File uploading failed: " + e.getMessage());
+        } finally {
+            if ((session != null) && (session.isConnected())) session.disconnect();
         }
     }
     
@@ -237,7 +232,7 @@ implements UserInfo, UIKeyboardInteractive {
             }
         }
         return result.toString();        
-}
+    }
     
     String arrayToString2(boolean[] a, String separator) {
         StringBuffer result = new StringBuffer();// start with first element
@@ -249,6 +244,6 @@ implements UserInfo, UIKeyboardInteractive {
             }
         }
         return result.toString();        
-}
+    }
 }
 

source/de/anomic/yacy/yacyCore.java

@@ -483,14 +483,22 @@ public class yacyCore {
     }
     
     public boolean saveSeedList() {
-	return saveSeedList(this.switchboard);
+        try {
+            saveSeedList(this.switchboard);
+            return true;
+        } catch (Exception e) {
+            return false;
+        }
     }
     
-    public static boolean saveSeedList(serverSwitch sb) {
+    public static void saveSeedList(serverSwitch sb) 
+    throws Exception {
         String logt;
         
         // be shure that we have something to say
-        if (seedDB.mySeed.getAddress() == null) return false;        
+        if (seedDB.mySeed.getAddress() == null) {
+            throw new Exception ("We have no valid IP address until now");        
+        }
         
         // getting the configured seed uploader
         String seedUploadMethod = sb.getConfig("seedUploadMethod","");
@@ -505,17 +513,22 @@ public class yacyCore {
         }        
 
         //  determine the seed uploader that should be used ...       
-        if (seedUploadMethod.equalsIgnoreCase("none"))  return true;
+        if (seedUploadMethod.equalsIgnoreCase("none")) return;
 
         yacySeedUploader uploader = getSeedUploader(seedUploadMethod);
-        if (uploader == null) return false;
+        if (uploader == null) {
+            throw new Exception("Unable to get the proper uploader-class for seed uploading method '" + seedUploadMethod + "'.");
+        }
         
         // ensure that the seed file url is configured properly
         URL seedURL;
         try{
-            seedURL = new URL(sb.getConfig("seedURL",""));
+            String seedURLStr = sb.getConfig("seedURL","");
+            if (seedURLStr.length() == 0) throw new MalformedURLException("The seed-file url must not be empty.");
+            if (!seedURLStr.toLowerCase().startsWith("http://")) throw new MalformedURLException("Unsupported protocol.");
+            seedURL = new URL(seedURLStr);
         }catch(MalformedURLException e){
-            return false;
+            throw new Exception("Malformed seed file URL '" + sb.getConfig("seedURL","") + "'. " + e.getMessage());
         }              
         
         // upload the seed-list using the configured uploader class
@@ -531,19 +544,19 @@ public class yacyCore {
                 if (logt.indexOf("Error") >= 0) {
                     seedDB.mySeed.put("PeerType", prevStatus);
                     log.logError("seed upload failed using " + uploader.getClass().getName() + " (error): " + logt.substring(logt.indexOf("Error") + 6));
-                    return false;                    
+                    throw new Exception("Seed-list uploading failed using uploader '" + uploader.getClass().getName() + "'\n(error): " + logt.substring(logt.indexOf("Error") + 6));                    
                 }
                 log.logInfo(logt);
             }
             
             // finally, set the principal status
             sb.setConfig("yacyStatus","principal");
-            return true;
+            return;
         } catch (Exception e) {
             seedDB.mySeed.put("PeerType", prevStatus);
             sb.setConfig("yacyStatus", prevStatus);
-            log.logInfo("seed upload failed (IO error): " + e.getMessage());
-            return false;
+            log.logInfo("Seed upload failed (IO error): " + e.getMessage());
+            throw new Exception("Seed-list uploading failed using uploader '" + uploader.getClass().getName() + "'\n(error): " + e.getMessage());
         }
     }
 

source/de/anomic/yacy/yacySeedDB.java

@@ -520,7 +520,7 @@ public class yacySeedDB {
 //          String  seedFTPAccount,
 //          String  seedFTPPassword,
 //          File    seedFTPPath,
-			      URL     seedURL) throws IOException {
+            URL     seedURL) throws Exception {
         
         // upload a seed file, if possible
         if (seedURL == null) throw new NullPointerException("UPLOAD - Error: URL not given");
@@ -538,10 +538,9 @@ public class yacySeedDB {
             // check also if the result can be retrieved again
             if (checkCache(uv, seedURL))
                 log = log + "UPLOAD CHECK - Success: the result vectors are equal" + serverCore.crlfString;
-            else
-                log = log + "UPLOAD CHECK - Error: the result vector is different" + serverCore.crlfString;
-        } catch (IOException e) {
-            log = log + "UPLOAD CHECK - Error: IO problem " + e.getMessage() + serverCore.crlfString;
+            else {
+                throw new Exception("UPLOAD CHECK - Error: the result vector is different" + serverCore.crlfString);
+            }
         } finally {
             if (seedFile != null) seedFile.delete();
         }

source/de/anomic/yacy/yacySeedUploader.java

@@ -5,7 +5,7 @@ import java.io.File;
 import de.anomic.server.serverSwitch;
 
 public interface yacySeedUploader {
-    public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile);
+    public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) throws Exception;
     public String[] getConfigurationOptions();
     public String[] getLibxDependences();
 }