diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index 94981575d..469e78151 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -98,7 +98,7 @@ public class ConfigLanguage_p { ArrayList langVector; try{ URL u = new URL(url); - langVector = nxTools.strings(httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig), "UTF-8"); + langVector = nxTools.strings(httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig, null, null), "UTF-8"); }catch(IOException e){ prop.put("status", 1);//unable to get url prop.put("status_url", url); diff --git a/htroot/ConfigSkins_p.java b/htroot/ConfigSkins_p.java index c71a32e46..61405b36c 100644 --- a/htroot/ConfigSkins_p.java +++ b/htroot/ConfigSkins_p.java @@ -127,7 +127,7 @@ public class ConfigSkins_p { ArrayList skinVector; try{ URL u = new URL(url); - skinVector = nxTools.strings(httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig), "UTF-8"); + skinVector = nxTools.strings(httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig, null, null), "UTF-8"); }catch(IOException e){ prop.put("status", 1);//unable to get URL prop.put("status_url", url); diff --git a/htroot/CrawlURLFetch_p.java b/htroot/CrawlURLFetch_p.java index bd13702d6..79c4449f3 100644 --- a/htroot/CrawlURLFetch_p.java +++ b/htroot/CrawlURLFetch_p.java @@ -53,6 +53,7 @@ import de.anomic.net.URL; import de.anomic.plasma.plasmaCrawlProfile; import de.anomic.plasma.plasmaCrawlZURL; import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.server.serverByteBuffer; import de.anomic.server.serverSwitch; import de.anomic.http.httpHeader; import de.anomic.http.httpRemoteProxyConfig; @@ -306,7 +307,9 @@ public class CrawlURLFetch_p { seed.getIP(), 5000, null, null, - theRemoteProxyConfig)); + theRemoteProxyConfig, + null, + null)); if (answer.matches("\\d+")) return Integer.parseInt(answer); else { @@ -519,11 +522,13 @@ public class CrawlURLFetch_p { serverLog.logFine(this.getName(), "downloaded URL-list from " + url + " (" + res.statusCode + ")"); this.lastServerResponse = res.statusCode + " (" + res.statusText + ")"; if (res.status.startsWith("2")) { - byte[] cbs = res.writeContent(); + serverByteBuffer sbb = new serverByteBuffer(); + //byte[] cbs = res.writeContent(); + res.writeContent(sbb, null); String encoding = res.responseHeader.getCharacterEncoding(); if (encoding == null) encoding = "US-ASCII"; - r = parseText(new String(cbs, encoding)); + r = parseText(new String(sbb.getBytes(), encoding)); } httpc.returnInstance(con); } catch (IOException e) { } diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index 220904caa..1a62fd05f 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -132,7 +132,7 @@ public class sharedBlacklist_p { // get List URL u = new URL(downloadURL); - otherBlacklist = nxTools.strings(httpc.wget(u, u.getHost(), 12000, null, null, switchboard.remoteProxyConfig,reqHeader), "UTF-8"); + otherBlacklist = nxTools.strings(httpc.wget(u, u.getHost(), 12000, null, null, switchboard.remoteProxyConfig,reqHeader, null), "UTF-8"); } catch (Exception e) { prop.put("status", STATUS_PEER_UNKNOWN); prop.put("page", 1); @@ -148,7 +148,7 @@ public class sharedBlacklist_p { try { URL u = new URL(downloadURL); - otherBlacklist = nxTools.strings(httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig), "UTF-8"); //get List + otherBlacklist = nxTools.strings(httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig, null, null), "UTF-8"); //get List } catch (Exception e) { prop.put("status", STATUS_URL_PROBLEM); prop.put("status_address",downloadURL); diff --git a/htroot/xml/util/getpageinfo_p.java b/htroot/xml/util/getpageinfo_p.java index dbd8e900f..43525b342 100644 --- a/htroot/xml/util/getpageinfo_p.java +++ b/htroot/xml/util/getpageinfo_p.java @@ -83,7 +83,7 @@ public class getpageinfo_p { if (actions.indexOf("title")>=0) { try { URL u = new URL(url); - String contentString=new String(httpc.wget(u, u.getHost(), 6000, null, null, ((plasmaSwitchboard) env).remoteProxyConfig, null)) ; + String contentString=new String(httpc.wget(u, u.getHost(), 6000, null, null, ((plasmaSwitchboard) env).remoteProxyConfig, null, null)) ; htmlFilterContentScraper scraper = new htmlFilterContentScraper(u); //OutputStream os = new htmlFilterOutputStream(null, scraper, null, false); diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java index d51a67108..04c0c5144 100644 --- a/source/de/anomic/data/robotsParser.java +++ b/source/de/anomic/data/robotsParser.java @@ -59,6 +59,7 @@ import de.anomic.http.httpc; import de.anomic.net.URL; import de.anomic.plasma.plasmaCrawlRobotsTxt; import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.server.serverByteBuffer; import de.anomic.server.logging.serverLog; /* @@ -433,7 +434,9 @@ public final class robotsParser{ } // downloading the content - robotsTxt = res.writeContent(); + serverByteBuffer sbb = new serverByteBuffer(); + res.writeContent(sbb, null); + robotsTxt = sbb.getBytes(); con.close(); downloadEnd = System.currentTimeMillis(); diff --git a/source/de/anomic/htmlFilter/htmlFilterContentScraper.java b/source/de/anomic/htmlFilter/htmlFilterContentScraper.java index 596cb1266..5b66ff794 100644 --- a/source/de/anomic/htmlFilter/htmlFilterContentScraper.java +++ b/source/de/anomic/htmlFilter/htmlFilterContentScraper.java @@ -482,7 +482,9 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen 10000, null, null, - plasmaSwitchboard.getSwitchboard().remoteProxyConfig + plasmaSwitchboard.getSwitchboard().remoteProxyConfig, + null, + null ); if (page == null) throw new IOException("no response from url " + location.toString()); diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index 3f3e0159d..3c5c244b5 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -1101,60 +1101,6 @@ public final class httpc { return new response(false); } - /* -DEBUG: PUT BODY=------------1090358578442 -Content-Disposition: form-data; name="youare" - -Ty2F86ekSWM5 -------------1090358578442 -Content-Disposition: form-data; name="key" - -6EkPPOl7 -------------1090358578442 -Content-Disposition: form-data; name="iam" - -HnTvzwV7SCJR -------------1090358578442 -Content-Disposition: form-data; name="process" - -permission -------------1090358578442 - - */ - - /* -------------0xKhTmLbOuNdArY -Content-Disposition: form-data; name="file1"; filename="dir.gif" -Content-Type: image/gif - -GIF89 -------------0xKhTmLbOuNdArY -Content-Disposition: form-data; name="file2"; filename="" - - -------------0xKhTmLbOuNdArY -Content-Disposition: form-data; name="upload" - -do upload -------------0xKhTmLbOuNdArY-- - -###### Listing Properties ###### -# METHOD=POST -### Header Values: -# EXT=html -# HTTP=HTTP/1.1 -# ACCEPT-ENCODING=gzip, deflate;q=1.0, identity;q=0.5, *;q=0 -# HOST=localhost:8080 -# PATH=/testcgi/doit.html -# CONTENT-LENGTH=474 -# CONTENT-TYPE=multipart/form-data; boundary=----------0xKhTmLbOuNdArY -# ARGC=0 -# CONNECTION=close -# USER-AGENT=Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/103u (KHTML, like Gecko) Safari/100.1 -### Call Properties: -###### End OfList ###### - */ - public static byte[] singleGET( String realhost, String virtualhost, @@ -1165,8 +1111,11 @@ do upload String password, boolean ssl, httpRemoteProxyConfig theRemoteProxyConfig, - httpHeader requestHeader + httpHeader requestHeader, + File download ) throws IOException { + // if download == null, the get result is stored to a byte[]Êand returned, + // othervise the get is streamed to the file and null is returned if (requestHeader == null) requestHeader = new httpHeader(); // setting host authorization header @@ -1184,7 +1133,16 @@ do upload httpc.response res = con.GET(path, requestHeader); if (res.status.startsWith("2")) { - return res.writeContent(); + if (download == null) { + // stream to byte[] + serverByteBuffer sbb = new serverByteBuffer(); + res.writeContent(sbb, null); + return sbb.getBytes(); + } else { + // stream to file and return null + res.writeContent(null, download); + return null; + } } return res.status.getBytes(); } catch (Exception e) { @@ -1201,7 +1159,8 @@ do upload int timeout, String user, String password, - httpRemoteProxyConfig theRemoteProxyConfig + httpRemoteProxyConfig theRemoteProxyConfig, + File download ) throws IOException { int port = u.getPort(); boolean ssl = u.getProtocol().equals("https"); @@ -1209,19 +1168,9 @@ do upload String path = u.getPath(); String query = u.getQuery(); if ((query != null) && (query.length() > 0)) path = path + "?" + query; - return singleGET(u.getHost(), vhost, port, path, timeout, user, password, ssl, theRemoteProxyConfig, null); + return singleGET(u.getHost(), vhost, port, path, timeout, user, password, ssl, theRemoteProxyConfig, null, download); } - /* - public static byte[] singleGET(String url, int timeout) throws IOException { - try { - return singleGET(new URL(url), timeout, null, null, null, 0); - } catch (MalformedURLException e) { - throw new IOException("Malformed URL: " + e.getMessage()); - } - } - */ - public static byte[] singlePOST( String realhost, String virtualhost, @@ -1253,7 +1202,9 @@ do upload //System.out.println("response=" + res.toString()); if (res.status.startsWith("2")) { - return res.writeContent(); + serverByteBuffer sbb = new serverByteBuffer(); + res.writeContent(sbb, null); + return sbb.getBytes(); } return res.status.getBytes(); } catch (Exception e) { @@ -1295,43 +1246,6 @@ do upload files ); } - - public static byte[] singlePOST( - String url, - int timeout, - serverObjects props - ) throws IOException { - try { - URL u = new URL(url); - return singlePOST( - u, - u.getHost(), - timeout, - null, - null, - null, - props, - null - ); - } catch (MalformedURLException e) { - throw new IOException("Malformed URL: " + e.getMessage()); - } - } - - public static byte[] wget( - URL url, - String vhost, - int timeout, - String user, - String password, - httpRemoteProxyConfig theRemoteProxyConfig - ) throws IOException { - return wget(url, vhost,timeout,user,password,theRemoteProxyConfig,null); - } - - public static byte[] wget(URL url) throws IOException{ - return wget(url, url.getHost(), 10000, null, null, null, null); - } public static byte[] wget( URL url, @@ -1340,7 +1254,8 @@ do upload String user, String password, httpRemoteProxyConfig theRemoteProxyConfig, - httpHeader requestHeader + httpHeader requestHeader, + File download ) throws IOException { int port = url.getPort(); @@ -1361,7 +1276,8 @@ do upload password, ssl, theRemoteProxyConfig, - requestHeader + requestHeader, + download ); if (a == null) return null; @@ -1389,6 +1305,7 @@ do upload null, null, (useProxy) ? proxy : null, + null, null ) , "UTF-8"); @@ -1503,7 +1420,7 @@ do upload httpRemoteProxyConfig theRemoteProxyConfig = httpRemoteProxyConfig.init(proxyHost,proxyPort); try { URL u = new URL(url); - text = nxTools.strings(wget(u, u.getHost(), timeout, null, null, theRemoteProxyConfig)); + text = nxTools.strings(wget(u, u.getHost(), timeout, null, null, theRemoteProxyConfig, null, null)); } catch (MalformedURLException e) { System.out.println("The url '" + url + "' is wrong."); } catch (IOException e) { @@ -1864,7 +1781,7 @@ do upload * * @return the found content * @throws IOException - */ + */ /* public byte[] writeContent() throws IOException { // int contentLength = (int) this.responseHeader.contentLength(); // serverByteBuffer sbb = new serverByteBuffer((contentLength==-1)?8192:contentLength); @@ -1872,7 +1789,19 @@ do upload // return sbb.getBytes(); return serverFileUtils.read(this.getContentInputStream()); } - + public void writeContent(File file) throws IOException { + // this writes the input stream to a file + FileOutputStream bufferOS = null; + try { + if (file != null) bufferOS = new FileOutputStream(file); + serverFileUtils.writeX(this.getContentInputStream(), null, bufferOS); + } finally { + if (bufferOS != null) { + bufferOS.close(); + if (file.length() == 0) file.delete(); + } + } + }*/ /** * This method outputs the found content into an byte-array and * additionally outputs it to procOS. @@ -1917,7 +1846,9 @@ do upload FileOutputStream bufferOS = null; try { if (file != null) bufferOS = new FileOutputStream(file); - if (procOS instanceof OutputStream) { + if (procOS == null) { + serverFileUtils.writeX(this.getContentInputStream(), null, bufferOS); + } else if (procOS instanceof OutputStream) { serverFileUtils.writeX(this.getContentInputStream(), (OutputStream) procOS, bufferOS); //writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, bufferOS); } else if (procOS instanceof Writer) { @@ -1929,6 +1860,7 @@ do upload } } finally { if (bufferOS != null) { + bufferOS.flush(); bufferOS.close(); if (file.length() == 0) file.delete(); } diff --git a/source/de/anomic/net/natLib.java b/source/de/anomic/net/natLib.java index 7f3fc9ee0..806f5862a 100644 --- a/source/de/anomic/net/natLib.java +++ b/source/de/anomic/net/natLib.java @@ -64,7 +64,7 @@ public class natLib { rm status.htm */ try { - ArrayList x = nxTools.strings(httpc.wget(new URL("http://192.168.0.1:80/status.htm"), "192.168.0.1", 5000, "admin", password, null)); + ArrayList x = nxTools.strings(httpc.wget(new URL("http://192.168.0.1:80/status.htm"), "192.168.0.1", 5000, "admin", password, null, null, null)); x = nxTools.grep(x, 1, "IP Address"); if ((x == null) || (x.size() == 0)) return null; String line = nxTools.tail1(x); @@ -76,7 +76,7 @@ public class natLib { private static String getWhatIsMyIP() { try { - ArrayList x = nxTools.strings(httpc.wget(new URL("http://www.whatismyip.com/"), "www.whatsmyip.com", 5000, null, null, null)); + ArrayList x = nxTools.strings(httpc.wget(new URL("http://www.whatismyip.com/"), "www.whatsmyip.com", 5000, null, null, null, null, null)); x = nxTools.grep(x, 0, "Your IP is"); String line = nxTools.tail1(x); return nxTools.awk(line, " ", 4); @@ -87,7 +87,7 @@ public class natLib { private static String getStanford() { try { - ArrayList x = nxTools.strings(httpc.wget(new URL("http://www.slac.stanford.edu/cgi-bin/nph-traceroute.pl"), "www.slac.stanford.edu", 5000, null, null, null)); + ArrayList x = nxTools.strings(httpc.wget(new URL("http://www.slac.stanford.edu/cgi-bin/nph-traceroute.pl"), "www.slac.stanford.edu", 5000, null, null, null, null, null)); x = nxTools.grep(x, 0, "firewall protecting your browser"); String line = nxTools.tail1(x); return nxTools.awk(line, " ", 7); @@ -98,7 +98,7 @@ public class natLib { private static String getIPID() { try { - ArrayList x = nxTools.strings(httpc.wget(new URL("http://ipid.shat.net/"), "ipid.shat.net", 5000, null, null, null), "UTF-8"); + ArrayList x = nxTools.strings(httpc.wget(new URL("http://ipid.shat.net/"), "ipid.shat.net", 5000, null, null, null, null, null), "UTF-8"); x = nxTools.grep(x, 2, "Your IP address"); String line = nxTools.tail1(x); return nxTools.awk(nxTools.awk(nxTools.awk(line, " ", 5), ">", 2), "<", 1); diff --git a/source/de/anomic/net/whois.java b/source/de/anomic/net/whois.java index 7a0890e36..dc1337e85 100644 --- a/source/de/anomic/net/whois.java +++ b/source/de/anomic/net/whois.java @@ -40,16 +40,11 @@ package de.anomic.net; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.util.ArrayList; import java.util.Properties; -import de.anomic.http.httpc; -import de.anomic.tools.nxTools; - public class whois { public static Properties Whois(String dom) { @@ -86,32 +81,4 @@ public class whois { if ((info1 != null) && (info2 != null) && (info1.toLowerCase().startsWith("person"))) return "Person: " + info2; return "unknown"; } - - public static String reversedns(String ip) { - try { - ArrayList x = nxTools.strings(httpc.wget(new URL("http://www.dnsstuff.com/tools/ptr.ch?ip=" + ip))); - x = nxTools.grep(x, 0, "PTR record:"); - if ((x == null) || (x.size() == 0)) return null; - String line = nxTools.tail1(x); - int p = line.indexOf(""); - int q = line.indexOf(""); - if ((p > 0) && (q > 0)) return line.substring(p + 3, q); - } catch (Exception e) { - return null; - } - return null; - } - - public static void main(String[] args) { - /* - Properties p = Whois(args[0]); - if (p != null) { - System.out.println(p); - System.out.println("---" + evaluateWhois(p)); - } else { - System.out.println("whois cannot execute"); - } - */ - System.out.println(reversedns("85.212.45.94")); - } } diff --git a/source/de/anomic/plasma/parser/odt/odtParser.java b/source/de/anomic/plasma/parser/odt/odtParser.java index a1a4f9966..c0c7681fa 100644 --- a/source/de/anomic/plasma/parser/odt/odtParser.java +++ b/source/de/anomic/plasma/parser/odt/odtParser.java @@ -250,7 +250,7 @@ public class odtParser extends AbstractParser implements Parser { testParser.setLogger(new serverLog("PARSER.ODT")); // downloading the document content - byte[] content = httpc.singleGET(contentUrl, contentUrl.getHost(), 10000, null, null, null); + byte[] content = httpc.singleGET(contentUrl, contentUrl.getHost(), 10000, null, null, null, null); ByteArrayInputStream input = new ByteArrayInputStream(content); // parsing the document diff --git a/source/de/anomic/plasma/parser/rpm/rpmParser.java b/source/de/anomic/plasma/parser/rpm/rpmParser.java index 383bc3613..1ac0be0dd 100644 --- a/source/de/anomic/plasma/parser/rpm/rpmParser.java +++ b/source/de/anomic/plasma/parser/rpm/rpmParser.java @@ -180,7 +180,7 @@ public class rpmParser extends AbstractParser implements Parser { URL contentUrl = new URL(args[0]); rpmParser testParser = new rpmParser(); - byte[] content = httpc.singleGET(contentUrl, contentUrl.getHost(), 10000, null, null, null); + byte[] content = httpc.singleGET(contentUrl, contentUrl.getHost(), 10000, null, null, null, null); ByteArrayInputStream input = new ByteArrayInputStream(content); testParser.parse(contentUrl, "application/x-rpm", null, input); } catch (Exception e) { diff --git a/source/de/anomic/plasma/parser/vcf/vcfParser.java b/source/de/anomic/plasma/parser/vcf/vcfParser.java index 5419b2635..4a6f2f108 100644 --- a/source/de/anomic/plasma/parser/vcf/vcfParser.java +++ b/source/de/anomic/plasma/parser/vcf/vcfParser.java @@ -271,7 +271,7 @@ public class vcfParser extends AbstractParser implements Parser { URL contentUrl = new URL(args[0]); vcfParser testParser = new vcfParser(); - byte[] content = httpc.singleGET(contentUrl, contentUrl.getHost(), 10000, null, null, null); + byte[] content = httpc.singleGET(contentUrl, contentUrl.getHost(), 10000, null, null, null, null); ByteArrayInputStream input = new ByteArrayInputStream(content); testParser.parse(contentUrl, "text/x-vcard", "UTF-8",input); } catch (Exception e) { diff --git a/source/de/anomic/server/serverByteBuffer.java b/source/de/anomic/server/serverByteBuffer.java index 988c2a410..a61408f2c 100644 --- a/source/de/anomic/server/serverByteBuffer.java +++ b/source/de/anomic/server/serverByteBuffer.java @@ -300,6 +300,7 @@ public final class serverByteBuffer extends OutputStream { // start is inclusive, end is exclusive if (len > length) throw new IndexOutOfBoundsException("getBytes: len > length"); if (start > length) throw new IndexOutOfBoundsException("getBytes: start > length"); + if ((start == 0) && (len == length) && (len == buffer.length)) return buffer; byte[] tmp = new byte[len]; System.arraycopy(buffer, offset + start, tmp, 0, len); return tmp; diff --git a/source/de/anomic/tools/loaderThreads.java b/source/de/anomic/tools/loaderThreads.java index 011818aa3..50690e377 100644 --- a/source/de/anomic/tools/loaderThreads.java +++ b/source/de/anomic/tools/loaderThreads.java @@ -146,7 +146,7 @@ public class loaderThreads { public void run() { try { - page = httpc.wget(url, url.getHost(), timeout, user, password, remoteProxyConfig); + page = httpc.wget(url, url.getHost(), timeout, user, password, remoteProxyConfig, null, null); loaded = true; process.feed(page); if (process.status() == loaderCore.STATUS_FAILED) { diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index f5e282731..dd4cadd21 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -281,6 +281,7 @@ public final class yacyClient { null, null, (useProxy)? plasmaSwitchboard.getSwitchboard().remoteProxyConfig:null, + null, null ) , "UTF-8"); @@ -318,7 +319,9 @@ public final class yacyClient { 10000, null, null, - (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null + (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null, + null, + null ) , "UTF-8"); @@ -360,7 +363,9 @@ public final class yacyClient { 6000, null, null, - (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null + (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null, + null, + null ) , "UTF-8"); @@ -945,7 +950,9 @@ public final class yacyClient { 60000, null, null, - (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null + (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null, + null, + null ) , "UTF-8"); } catch (Exception e) { @@ -1247,7 +1254,9 @@ public final class yacyClient { 5000, null, null, - (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null + (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null, + null, + null ) , "UTF-8"); System.out.println("Result=" + result.toString()); diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index e91484754..a591fd680 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -177,7 +177,7 @@ public class yacyPeerActions { yacyCore.log.logInfo("BOOTSTRAP: seed-list URL " + seedListFileURL + " too old (" + (header.age() / 86400000) + " days)"); } else { ssc++; - seedList = nxTools.strings(httpc.wget(url, url.getHost(), this.bootstrapLoadTimeout, null, null, this.sb.remoteProxyConfig,reqHeader), "UTF-8"); + seedList = nxTools.strings(httpc.wget(url, url.getHost(), this.bootstrapLoadTimeout, null, null, this.sb.remoteProxyConfig,reqHeader, null), "UTF-8"); enu = seedList.iterator(); lc = 0; while (enu.hasNext()) { diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index 47cbcd5d1..1d7dc8dc1 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -42,7 +42,6 @@ import de.anomic.http.httpc; import de.anomic.net.URL; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; -import de.anomic.server.serverFileUtils; import de.anomic.server.serverSystem; import de.anomic.server.logging.serverLog; @@ -230,17 +229,18 @@ public final class yacyVersion implements Comparator, Comparable { public static void downloadRelease(yacyVersion release) throws IOException { File storagePath = plasmaSwitchboard.getSwitchboard().releasePath; // load file - byte[] file = httpc.wget( + File download = new File(storagePath, release.url.getFileName()); + httpc.wget( release.url, release.url.getHost(), 1000, null, null, - plasmaSwitchboard.getSwitchboard().remoteProxyConfig + plasmaSwitchboard.getSwitchboard().remoteProxyConfig, + null, + download ); - if (file == null) throw new IOException("wget of url " + release.url + " failed"); - // save file - serverFileUtils.write(file, new File(storagePath, release.url.getFileName())); + if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + release.url + " failed"); } diff --git a/source/restarttest.java b/source/restarttest.java new file mode 100644 index 000000000..d6172a65d --- /dev/null +++ b/source/restarttest.java @@ -0,0 +1,111 @@ + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class restarttest { + + private static final int DEFAULT_BUFFER_SIZE = 4096; + public static final byte cr = 13; + public static final byte lf = 10; + public static final String lfstring = new String(new byte[]{lf}); + + public static void write(byte[] source, File dest) throws IOException { + copy(new ByteArrayInputStream(source), dest); + } + public static void copy(InputStream source, File dest) throws IOException { + FileOutputStream fos = null; + try { + fos = new FileOutputStream(dest); + copy(source, fos, -1); + } finally { + if (fos != null) try {fos.close();} catch (Exception e) {} + } + } + public static long copy(InputStream source, OutputStream dest, long count) throws IOException { + byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; + int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE); + + int c; long total = 0; + while ((c = source.read(buffer,0,chunkSize)) > 0) { + dest.write(buffer, 0, c); + dest.flush(); + total += c; + + if (count > 0) { + chunkSize = (int)Math.min(count-total,DEFAULT_BUFFER_SIZE); + if (chunkSize == 0) break; + } + + } + dest.flush(); + + return total; + } + + + public static void deployScript(File scriptFile, String theScript) throws IOException { + write(theScript.getBytes(), scriptFile); + try { + Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath()).waitFor(); + } catch (InterruptedException e) { + throw new IOException(e.getMessage()); + } + } + + public static void execAsynchronous(File scriptFile) throws IOException { + // runs a unix/linux script as separate thread + File starterFile = new File(scriptFile.getAbsolutePath() + ".starter.sh"); + //deployScript(starterFile, "touch restart.starter.startet1"); + deployScript(starterFile, scriptFile.getAbsolutePath() + " &" + lfstring); + try { + Runtime.getRuntime().exec(starterFile.getAbsolutePath()).waitFor(); + } catch (InterruptedException e) { + throw new IOException(e.getMessage()); + } + starterFile.delete(); + } + + public static void restart(File root) { + + try { + System.out.println("initiated"); + String script = "cd " + root.getAbsolutePath() + lfstring + "while [ -e restarttest.running ]; do" + lfstring + "sleep 1" + lfstring + "done" + lfstring + "java restarttest again"; + File scriptFile = new File(root, "restart.sh"); + deployScript(scriptFile, script); + System.out.println("wrote restart-script to " + scriptFile.getAbsolutePath()); + execAsynchronous(scriptFile); + System.out.println("script is running"); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + if (args.length > 0) { + File f = new File("restarttest.restartet"); + if (f.exists()) f.delete(); + try { + f.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + } + } else { + File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/')); + File f = new File("restarttest.running"); + if (f.exists()) f.delete(); + try { + f.createNewFile(); + f.deleteOnExit(); + System.out.println("start-up"); + restart(applicationRoot); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + +} diff --git a/source/yacy.java b/source/yacy.java index e0df66104..6846dfdcc 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -458,7 +458,7 @@ public final class yacy { server.interrupt(); if (server.isAlive()) try { URL u = new URL((server.withSSL()?"https":"http")+"://localhost:" + serverCore.getPortNr(port)); - httpc.wget(u, u.getHost(), 1000, null, null, null); // kick server + httpc.wget(u, u.getHost(), 1000, null, null, null, null, null); // kick server serverLog.logConfig("SHUTDOWN", "sent termination signal to server socket"); } catch (IOException ee) { serverLog.logConfig("SHUTDOWN", "termination signal to server socket missed (server shutdown, ok)");