*) shrinking httpc linebuffer when httpc is returned to pool. This is done to free memory

*) Making Seed-Upload configuration more verbose.
*) Some Changes in SOAP Search API (not finished yet).

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@158 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent b625aa91fd
commit 0e1d9e9722

@ -55,6 +55,7 @@ You are now permanently <b>online</b>. After a short while you should see the ef
<b>Seed Settings changed.#(success)#::You are now a principal peer.#(/success)#</b><br> <b>Seed Settings changed.#(success)#::You are now a principal peer.#(/success)#</b><br>
:: ::
<p><b>Seed Settings changed, but something is wrong.</b></p> <p><b>Seed Settings changed, but something is wrong.</b></p>
<p><font color="red">#[errormsg]#</font></p>
Seed Uploading was deactivated automatically. Seed Uploading was deactivated automatically.
Please return to the settings page and modify the data.<br> Please return to the settings page and modify the data.<br>
:: ::

@ -275,16 +275,14 @@ public class SettingsAck_p {
env.setConfig("seedUploadMethod", newSeedUploadMethod); env.setConfig("seedUploadMethod", newSeedUploadMethod);
env.setConfig("seedURL", newSeedURLStr); env.setConfig("seedURL", newSeedURLStr);
boolean success = yacyCore.saveSeedList(env); // trying to upload the seed-list file
if (!success) { yacyCore.saveSeedList(env);
prop.put("info", 14);
env.setConfig("seedUploadMethod","none"); // we have successfully uploaded the seed-list file
} else {
prop.put("info_seedUploadMethod",newSeedUploadMethod); prop.put("info_seedUploadMethod",newSeedUploadMethod);
prop.put("info_seedURL",newSeedURLStr); prop.put("info_seedURL",newSeedURLStr);
prop.put("info_success",(newSeedUploadMethod.equalsIgnoreCase("none")?0:1)); prop.put("info_success",(newSeedUploadMethod.equalsIgnoreCase("none")?0:1));
prop.put("info", 19); prop.put("info", 19);
}
} else { } else {
prop.put("info_seedUploadMethod",newSeedUploadMethod); prop.put("info_seedUploadMethod",newSeedUploadMethod);
prop.put("info_seedURL",newSeedURLStr); prop.put("info_seedURL",newSeedURLStr);
@ -294,6 +292,8 @@ public class SettingsAck_p {
} }
} catch (Exception e) { } catch (Exception e) {
prop.put("info",14); prop.put("info",14);
prop.put("info_errormsg",e.getMessage().replaceAll("\n","<br>"));
env.setConfig("seedUploadMethod","none");
} }
return prop; return prop;
} }
@ -308,6 +308,7 @@ public class SettingsAck_p {
// getting the uploader module name // getting the uploader module name
String uploaderName = (String) uploaderKeys.nextElement(); String uploaderName = (String) uploaderKeys.nextElement();
// determining if the user has reconfigured the settings of this uploader // determining if the user has reconfigured the settings of this uploader
if (post.containsKey("seed" + uploaderName + "Settings")) { if (post.containsKey("seed" + uploaderName + "Settings")) {
nothingChanged = true; nothingChanged = true;
@ -324,14 +325,20 @@ public class SettingsAck_p {
} }
} }
if (!nothingChanged) { if (!nothingChanged) {
if (env.getConfig("seedUploadMethod","none").equals(theUploader)) { // if the seed upload method is equal to the seed uploader whose settings
boolean success = yacyCore.saveSeedList(env); // were changed, we now try to upload the seed list with the new settings
if (!success) { if (env.getConfig("seedUploadMethod","none").equalsIgnoreCase(uploaderName)) {
prop.put("info", 14); try {
env.setConfig("seedUploadMethod","none"); yacyCore.saveSeedList(env);
} else {
// we have successfully uploaded the seed file
prop.put("info", 13); prop.put("info", 13);
prop.put("info_success",1); prop.put("info_success",1);
} catch (Exception e) {
// if uploading failed we print out an error message
prop.put("info", 14);
prop.put("info_errormsg",e.getMessage().replaceAll("\n","<br>"));
env.setConfig("seedUploadMethod","none");
} }
} else { } else {
prop.put("info", 13); prop.put("info", 13);

@ -285,6 +285,10 @@ public final class httpc {
this.remoteProxyUse = false; this.remoteProxyUse = false;
this.savedRemoteHost = null; this.savedRemoteHost = null;
this.requestPath = null; this.requestPath = null;
// shrink readlinebuffer if it is to large
this.readLineBuffer.reset(80);
} catch (Exception e) { } catch (Exception e) {
// we could ignore this ... // we could ignore this ...
} }
@ -306,6 +310,7 @@ public final class httpc {
this.remoteProxyUse = false; this.remoteProxyUse = false;
this.timeout = timeout; this.timeout = timeout;
this.savedRemoteHost = server; this.savedRemoteHost = server;
try { try {
this.host = server + ((port == 80) ? "" : (":" + port)); this.host = server + ((port == 80) ? "" : (":" + port));
String hostip; String hostip;

@ -52,6 +52,7 @@ package de.anomic.http;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.PushbackInputStream; import java.io.PushbackInputStream;
import java.lang.reflect.Constructor;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.NetworkInterface; import java.net.NetworkInterface;
import java.util.Arrays; import java.util.Arrays;
@ -262,7 +263,9 @@ public final class httpd implements serverHandler {
if (this.prop.containsKey("PATH") && this.prop.getProperty("PATH").startsWith("/soap")) { if (this.prop.containsKey("PATH") && this.prop.getProperty("PATH").startsWith("/soap")) {
if (soapHandler == null) { if (soapHandler == null) {
try { try {
soapHandler = (httpdHandler) Class.forName("de.anomic.soap.httpdSoapHandler").newInstance(); Class soapHandlerClass = Class.forName("de.anomic.soap.httpdSoapHandler");
Constructor classConstructor = soapHandlerClass.getConstructor( new Class[] { serverSwitch.class } );
soapHandler = (httpdHandler) classConstructor.newInstance(new Object[] { this.switchboard });
} catch (Exception e) { } catch (Exception e) {
throw new IOException("Unable to load the soap handler"); throw new IOException("Unable to load the soap handler");
} }

@ -57,6 +57,8 @@ import java.io.OutputStream;
import java.io.PushbackInputStream; import java.io.PushbackInputStream;
import java.util.Properties; import java.util.Properties;
import de.anomic.server.serverSwitch;
public interface httpdHandler { public interface httpdHandler {
void doGet(Properties conProp, httpHeader header, OutputStream response) throws IOException; void doGet(Properties conProp, httpHeader header, OutputStream response) throws IOException;

@ -307,6 +307,18 @@ public final class serverByteBuffer extends OutputStream {
this.offset = 0; this.offset = 0;
} }
public void reset(int newSize) {
this.resize(newSize);
this.reset();
}
public void resize(int newSize) {
if(newSize < 0) throw new IllegalArgumentException("Illegal array size: " + newSize);
byte[] v = new byte[newSize];
System.arraycopy(this.buffer,0,v,0,newSize > this.buffer.length ? this.buffer.length : newSize);
this.buffer = v;
}
public byte toByteArray()[] { public byte toByteArray()[] {
byte newbuf[] = new byte[this.length]; byte newbuf[] = new byte[this.length];
System.arraycopy(this.buffer, 0, newbuf, 0, this.length); System.arraycopy(this.buffer, 0, newbuf, 0, this.length);

@ -60,7 +60,7 @@ import de.anomic.server.serverSwitch;
* *
* @author Martin Thelian * @author Martin Thelian
*/ */
final class httpdSoapHandler extends httpdAbstractHandler implements httpdHandler public final class httpdSoapHandler extends httpdAbstractHandler implements httpdHandler
{ {
/* =============================================================== /* ===============================================================
@ -431,7 +431,7 @@ final class httpdSoapHandler extends httpdAbstractHandler implements httpdHandle
msgContext.setProperty(Constants.MC_RELATIVE_PATH, path.toString()); msgContext.setProperty(Constants.MC_RELATIVE_PATH, path.toString());
msgContext.setProperty(Constants.MC_JWS_CLASSDIR, "jwsClasses"); msgContext.setProperty(Constants.MC_JWS_CLASSDIR, "jwsClasses");
msgContext.setProperty(Constants.MC_HOME_DIR, "."); msgContext.setProperty(Constants.MC_HOME_DIR, ".");
msgContext.setProperty(MessageContext.TRANS_URL, "http://" + requestHeader.get("Host") + this.switchboard.getConfig("port","8080") + "/soap/index"); msgContext.setProperty(MessageContext.TRANS_URL, "http://" + requestHeader.get("Host") + ((((String)requestHeader.get("Host")).indexOf(":") > -1)?"":this.switchboard.getConfig("port","8080")) + "/soap/index");
msgContext.setProperty(MESSAGE_CONTEXT_HTTP_ROOT_PATH ,this.htRootPath.toString()); msgContext.setProperty(MESSAGE_CONTEXT_HTTP_ROOT_PATH ,this.htRootPath.toString());
msgContext.setProperty(MESSAGE_CONTEXT_SERVER_SWITCH,this.switchboard); msgContext.setProperty(MESSAGE_CONTEXT_SERVER_SWITCH,this.switchboard);

@ -4,21 +4,26 @@ import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.axis.AxisFault; import org.apache.axis.AxisFault;
import org.apache.axis.Message; import org.apache.axis.Message;
import org.apache.axis.MessageContext; import org.apache.axis.MessageContext;
import org.apache.axis.message.SOAPEnvelope; import org.apache.axis.message.SOAPEnvelope;
import org.apache.axis.message.SOAPHeaderElement; import org.apache.axis.message.SOAPHeaderElement;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpTemplate; import de.anomic.http.httpTemplate;
import de.anomic.server.serverClassLoader; import de.anomic.server.serverClassLoader;
import de.anomic.server.serverCodings;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -27,7 +32,7 @@ import de.anomic.server.serverSwitch;
* *
* @author Martin Thelian * @author Martin Thelian
*/ */
final public class httpdSoapService public class httpdSoapService
{ {
/* ================================================================ /* ================================================================
* Constants needed to set the template that should be used to * Constants needed to set the template that should be used to
@ -36,7 +41,7 @@ final public class httpdSoapService
/** /**
* Constant: template for searching * Constant: template for searching
*/ */
private static final String TEMPLATE_SEARCH = "index.rss"; private static final String TEMPLATE_SEARCH = "index.soap";
/** /**
* Constant: template for the network status page * Constant: template for the network status page
*/ */
@ -81,7 +86,14 @@ final public class httpdSoapService
* *
* @throws AxisFault if the service could not be executed propery. * @throws AxisFault if the service could not be executed propery.
*/ */
public String search(String searchString) public Document search(
String searchString,
String searchMode,
String searchOrder,
int maxSearchCount,
int maxSearchTime,
String urlMaskFilter
)
throws AxisFault throws AxisFault
{ {
try try
@ -89,14 +101,20 @@ final public class httpdSoapService
// extracting the message context // extracting the message context
extractMessageContext(); extractMessageContext();
if ((searchMode == null) || !(searchMode.equalsIgnoreCase("global") || searchMode.equalsIgnoreCase("locale"))) {
searchMode = "global";
}
if (urlMaskFilter == null) urlMaskFilter = ".*";
// setting the searching properties // setting the searching properties
serverObjects args = new serverObjects(); serverObjects args = new serverObjects();
args.put("order","Quality-Date"); args.put("order","Quality-Date");
args.put("Enter","Search"); args.put("Enter","Search");
args.put("count","10"); args.put("count",Integer.toString(maxSearchCount));
args.put("resource","global"); args.put("resource","global");
args.put("time","10"); args.put("time",Integer.toString(maxSearchTime));
args.put("urlmaskfilter",".*"); args.put("urlmaskfilter",urlMaskFilter);
args.put("search",searchString); args.put("search",searchString);
@ -104,7 +122,7 @@ final public class httpdSoapService
String result = writeTemplate(TEMPLATE_SEARCH, args); String result = writeTemplate(TEMPLATE_SEARCH, args);
// sending back the result to the client // sending back the result to the client
return result; return this.convertContentToXML(result);
} }
catch (Exception e) catch (Exception e)
{ {
@ -336,4 +354,25 @@ final public class httpdSoapService
return m; return m;
} }
private Document convertContentToXML(String contentString)
throws Exception
{
Document doc = null;
try
{
DocumentBuilderFactory newDocBuilderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder newDocBuilder = newDocBuilderFactory.newDocumentBuilder();
InputSource is = new InputSource(new StringReader(contentString));
doc = newDocBuilder.parse(is);
}
catch (Exception e)
{
String errorMessage = "Unable to parse the search result XML data. " + e.getClass().getName() + ". " + e.getMessage();
throw new Exception(errorMessage);
}
return doc;
}
} }

@ -12,19 +12,19 @@ public class yacySeedUploadFile implements yacySeedUploader {
public static final String CONFIG_FILE_PATH = "seedFilePath"; public static final String CONFIG_FILE_PATH = "seedFilePath";
public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) { public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) throws Exception {
String logt; String logt, seedFilePath = "";
try { try {
String seedFilePath = sb.getConfig(CONFIG_FILE_PATH,null); seedFilePath = sb.getConfig(CONFIG_FILE_PATH,"");
if (seedFilePath == null) return "Error: Path to seed file is not configured properly"; if (seedFilePath.length() == 0) throw new Exception("Path to seed file is not configured properly");
File publicSeedFile = new File(seedFilePath); File publicSeedFile = new File(seedFilePath);
serverFileUtils.copy(seedFile,publicSeedFile); serverFileUtils.copy(seedFile,publicSeedFile);
return "Seed-List file stored successfully"; return "Seed-List file stored successfully";
} catch (Exception e) { } catch (Exception e) {
return "Error: " + e.getMessage(); throw new Exception("Unable to store the seed-list file into the filesystem using path '" + seedFilePath + "'. " + e.getMessage());
} }
} }

@ -24,7 +24,7 @@ public class yacySeedUploadScp implements yacySeedUploader {
public static final String CONFIG_SCP_PASSWORD = "seedScpPassword"; public static final String CONFIG_SCP_PASSWORD = "seedScpPassword";
public static final String CONFIG_SCP_PATH = "seedScpPath"; public static final String CONFIG_SCP_PATH = "seedScpPath";
public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) { public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) throws Exception {
try { try {
if (sb == null) throw new NullPointerException("Reference to serverSwitch nut not be null."); if (sb == null) throw new NullPointerException("Reference to serverSwitch nut not be null.");
if (seedDB == null) throw new NullPointerException("Reference to seedDB must not be null."); if (seedDB == null) throw new NullPointerException("Reference to seedDB must not be null.");
@ -36,14 +36,13 @@ public class yacySeedUploadScp implements yacySeedUploader {
String seedScpPath = sb.getConfig(CONFIG_SCP_PATH,null); String seedScpPath = sb.getConfig(CONFIG_SCP_PATH,null);
if ((seedScpServer != null) && (seedScpAccount != null) && (seedScpPassword != null) && (seedScpPath != null)) { if ((seedScpServer != null) && (seedScpAccount != null) && (seedScpPassword != null) && (seedScpPath != null)) {
String log = sshc.put(seedScpServer, seedFile, seedScpPath, seedScpAccount, seedScpPassword); return sshc.put(seedScpServer, seedFile, seedScpPath, seedScpAccount, seedScpPassword);
return "Seed file uploaded successfully using scp. " + log;
} }
return "Seed upload settings not configured properly. password-len=" + return "Seed upload settings not configured properly. password-len=" +
seedScpPassword.length() + ", filePath=" + seedScpPassword.length() + ", filePath=" +
seedScpPath; seedScpPath;
} catch (Exception e) { } catch (Exception e) {
return "Error: " + e.getMessage(); throw e;
} }
} }
@ -60,7 +59,7 @@ public class yacySeedUploadScp implements yacySeedUploader {
class sshc { class sshc {
public static String put(String host, public static String put(String host,
File localFile, String remoteName, File localFile, String remoteName,
String account, String password) { String account, String password) throws Exception {
Session session = null; Session session = null;
try { try {
@ -130,15 +129,11 @@ class sshc {
checkAck(in); checkAck(in);
return ""; return "SCP: File uploaded successfully.";
} catch (Exception e) { } catch (Exception e) {
System.out.println("ERROR: scp put failed:" + e.getMessage()); throw new Exception("SCP: File uploading failed: " + e.getMessage());
e.printStackTrace(); } finally {
return ""; if ((session != null) && (session.isConnected())) session.disconnect();
} finally
{
if ((session != null) && (session.isConnected()))
session.disconnect();
} }
} }
@ -237,7 +232,7 @@ implements UserInfo, UIKeyboardInteractive {
} }
} }
return result.toString(); return result.toString();
} }
String arrayToString2(boolean[] a, String separator) { String arrayToString2(boolean[] a, String separator) {
StringBuffer result = new StringBuffer();// start with first element StringBuffer result = new StringBuffer();// start with first element
@ -249,6 +244,6 @@ implements UserInfo, UIKeyboardInteractive {
} }
} }
return result.toString(); return result.toString();
} }
} }

@ -483,14 +483,22 @@ public class yacyCore {
} }
public boolean saveSeedList() { public boolean saveSeedList() {
return saveSeedList(this.switchboard); try {
saveSeedList(this.switchboard);
return true;
} catch (Exception e) {
return false;
}
} }
public static boolean saveSeedList(serverSwitch sb) { public static void saveSeedList(serverSwitch sb)
throws Exception {
String logt; String logt;
// be shure that we have something to say // be shure that we have something to say
if (seedDB.mySeed.getAddress() == null) return false; if (seedDB.mySeed.getAddress() == null) {
throw new Exception ("We have no valid IP address until now");
}
// getting the configured seed uploader // getting the configured seed uploader
String seedUploadMethod = sb.getConfig("seedUploadMethod",""); String seedUploadMethod = sb.getConfig("seedUploadMethod","");
@ -505,17 +513,22 @@ public class yacyCore {
} }
// determine the seed uploader that should be used ... // determine the seed uploader that should be used ...
if (seedUploadMethod.equalsIgnoreCase("none")) return true; if (seedUploadMethod.equalsIgnoreCase("none")) return;
yacySeedUploader uploader = getSeedUploader(seedUploadMethod); yacySeedUploader uploader = getSeedUploader(seedUploadMethod);
if (uploader == null) return false; if (uploader == null) {
throw new Exception("Unable to get the proper uploader-class for seed uploading method '" + seedUploadMethod + "'.");
}
// ensure that the seed file url is configured properly // ensure that the seed file url is configured properly
URL seedURL; URL seedURL;
try{ try{
seedURL = new URL(sb.getConfig("seedURL","")); String seedURLStr = sb.getConfig("seedURL","");
if (seedURLStr.length() == 0) throw new MalformedURLException("The seed-file url must not be empty.");
if (!seedURLStr.toLowerCase().startsWith("http://")) throw new MalformedURLException("Unsupported protocol.");
seedURL = new URL(seedURLStr);
}catch(MalformedURLException e){ }catch(MalformedURLException e){
return false; throw new Exception("Malformed seed file URL '" + sb.getConfig("seedURL","") + "'. " + e.getMessage());
} }
// upload the seed-list using the configured uploader class // upload the seed-list using the configured uploader class
@ -531,19 +544,19 @@ public class yacyCore {
if (logt.indexOf("Error") >= 0) { if (logt.indexOf("Error") >= 0) {
seedDB.mySeed.put("PeerType", prevStatus); seedDB.mySeed.put("PeerType", prevStatus);
log.logError("seed upload failed using " + uploader.getClass().getName() + " (error): " + logt.substring(logt.indexOf("Error") + 6)); log.logError("seed upload failed using " + uploader.getClass().getName() + " (error): " + logt.substring(logt.indexOf("Error") + 6));
return false; throw new Exception("Seed-list uploading failed using uploader '" + uploader.getClass().getName() + "'\n(error): " + logt.substring(logt.indexOf("Error") + 6));
} }
log.logInfo(logt); log.logInfo(logt);
} }
// finally, set the principal status // finally, set the principal status
sb.setConfig("yacyStatus","principal"); sb.setConfig("yacyStatus","principal");
return true; return;
} catch (Exception e) { } catch (Exception e) {
seedDB.mySeed.put("PeerType", prevStatus); seedDB.mySeed.put("PeerType", prevStatus);
sb.setConfig("yacyStatus", prevStatus); sb.setConfig("yacyStatus", prevStatus);
log.logInfo("seed upload failed (IO error): " + e.getMessage()); log.logInfo("Seed upload failed (IO error): " + e.getMessage());
return false; throw new Exception("Seed-list uploading failed using uploader '" + uploader.getClass().getName() + "'\n(error): " + e.getMessage());
} }
} }

@ -520,7 +520,7 @@ public class yacySeedDB {
// String seedFTPAccount, // String seedFTPAccount,
// String seedFTPPassword, // String seedFTPPassword,
// File seedFTPPath, // File seedFTPPath,
URL seedURL) throws IOException { URL seedURL) throws Exception {
// upload a seed file, if possible // upload a seed file, if possible
if (seedURL == null) throw new NullPointerException("UPLOAD - Error: URL not given"); if (seedURL == null) throw new NullPointerException("UPLOAD - Error: URL not given");
@ -538,10 +538,9 @@ public class yacySeedDB {
// check also if the result can be retrieved again // check also if the result can be retrieved again
if (checkCache(uv, seedURL)) if (checkCache(uv, seedURL))
log = log + "UPLOAD CHECK - Success: the result vectors are equal" + serverCore.crlfString; log = log + "UPLOAD CHECK - Success: the result vectors are equal" + serverCore.crlfString;
else else {
log = log + "UPLOAD CHECK - Error: the result vector is different" + serverCore.crlfString; throw new Exception("UPLOAD CHECK - Error: the result vector is different" + serverCore.crlfString);
} catch (IOException e) { }
log = log + "UPLOAD CHECK - Error: IO problem " + e.getMessage() + serverCore.crlfString;
} finally { } finally {
if (seedFile != null) seedFile.delete(); if (seedFile != null) seedFile.delete();
} }

@ -5,7 +5,7 @@ import java.io.File;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
public interface yacySeedUploader { public interface yacySeedUploader {
public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile); public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) throws Exception;
public String[] getConfigurationOptions(); public String[] getConfigurationOptions();
public String[] getLibxDependences(); public String[] getLibxDependences();
} }

Loading…
Cancel
Save