diff --git a/source/de/anomic/http/server/HTTPDProxyHandler.java b/source/de/anomic/http/server/HTTPDProxyHandler.java index ddae705a0..c38ef1379 100644 --- a/source/de/anomic/http/server/HTTPDProxyHandler.java +++ b/source/de/anomic/http/server/HTTPDProxyHandler.java @@ -43,7 +43,7 @@ package de.anomic.http.server; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; +//import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -71,6 +71,7 @@ import java.util.logging.LogManager; import java.util.logging.Logger; import java.util.zip.GZIPOutputStream; +import net.yacy.cora.protocol.Client; import net.yacy.cora.protocol.ProxySettings; import net.yacy.document.TextParser; import net.yacy.document.parser.html.ContentTransformer; @@ -87,7 +88,7 @@ import de.anomic.crawler.retrieval.HTTPLoader; import de.anomic.crawler.retrieval.Request; import de.anomic.crawler.retrieval.Response; import de.anomic.http.client.MultiOutputStream; -import de.anomic.http.client.Client; +//import de.anomic.http.client.Client; import de.anomic.http.client.Cache; import de.anomic.search.Switchboard; import de.anomic.search.SwitchboardConstants; @@ -439,7 +440,7 @@ public final class HTTPDProxyHandler { final GZIPOutputStream gzippedOut = null; - ResponseContainer res = null; +// ResponseContainer res = null; try { final int reqID = requestHeader.hashCode(); @@ -478,17 +479,22 @@ public final class HTTPDProxyHandler { // send request try { - res = client.GET(getUrl); - if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); +// res = client.GET(getUrl); +// if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); + client.GET(getUrl); + if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine()); conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader); - final ResponseHeader responseHeader = res.getResponseHeader(); +// final ResponseHeader responseHeader = res.getResponseHeader(); + final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap()); // determine if it's an internal error of the httpc if (responseHeader.isEmpty()) { - throw new Exception(res.getStatusLine()); +// throw new Exception(res.getStatusLine()); + throw new Exception(client.getHttpResponse().getStatusLine().toString()); } - final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond); +// final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond); + final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), respond); // the cache does either not exist or is (supposed to be) stale long sizeBeforeDelete = -1; @@ -519,7 +525,8 @@ public final class HTTPDProxyHandler { // handle incoming cookies handleIncomingCookies(responseHeader, host, ip); - prepareResponseHeader(responseHeader, res.getHttpVer()); +// prepareResponseHeader(responseHeader, res.getHttpVer()); + prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString()); // sending the respond header back to the client if (chunkedOut != null) { @@ -527,22 +534,31 @@ public final class HTTPDProxyHandler { } if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader); +// HTTPDemon.sendRespondHeader( +// conProp, +// respond, +// httpVer, +// res.getStatusCode(), +// res.getStatusLine().substring(4), // status text +// responseHeader); HTTPDemon.sendRespondHeader( conProp, respond, httpVer, - res.getStatusCode(), - res.getStatusLine().substring(4), // status text + client.getHttpResponse().getStatusLine().getStatusCode(), + client.getHttpResponse().getStatusLine().toString(), // status text responseHeader); - if (hasBody(res.getStatusCode())) { +// if (hasBody(res.getStatusCode())) { + if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) { final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond); final Response response = new Response( request, requestHeader, responseHeader, - res.getStatusLine(), +// res.getStatusLine(), + Integer.toString(client.getHttpResponse().getStatusLine().getStatusCode()), sb.crawler.defaultProxyProfile ); final String storeError = response.shallStoreCacheForProxy(); @@ -561,11 +577,13 @@ public final class HTTPDProxyHandler { ((storeHTCache) || (supportError != null)) ) { // we don't write actually into a file, only to RAM, and schedule writing the file. - int l = res.getResponseHeader().size(); +// int l = res.getResponseHeader().size(); + int l = responseHeader.size(); final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l); final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream}); - FileUtils.copy(res.getDataAsStream(), toClientAndMemory); +// FileUtils.copy(res.getDataAsStream(), toClientAndMemory); + client.writeTo(toClientAndMemory); // cached bytes byte[] cacheArray; if (byteStream.size() > 0) { @@ -608,7 +626,8 @@ public final class HTTPDProxyHandler { " StoreHTCache=" + storeHTCache + " SupportError=" + supportError); - FileUtils.copy(res.getDataAsStream(), outStream); +// FileUtils.copy(res.getDataAsStream(), outStream); + client.writeTo(outStream); conProp.setProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); } @@ -623,17 +642,19 @@ public final class HTTPDProxyHandler { } // end hasBody } catch(SocketException se) { // if opened ... - if(res != null) { - // client cut proxy connection, abort download - res.abort(); - } +// if(res != null) { +// // client cut proxy connection, abort download +// res.abort(); +// } + client.finish(); handleProxyException(se,conProp,respond,url); } finally { // if opened ... - if(res != null) { - // ... close connection - res.closeStream(); - } +// if(res != null) { +// // ... close connection +// res.closeStream(); +// } + client.finish(); } } catch (final Exception e) { handleProxyException(e,conProp,respond,url); @@ -741,7 +762,7 @@ public final class HTTPDProxyHandler { public static void doHead(final Properties conProp, final RequestHeader requestHeader, OutputStream respond) { - ResponseContainer res = null; +// ResponseContainer res = null; DigestURI url = null; try { final int reqID = requestHeader.hashCode(); @@ -813,28 +834,42 @@ public final class HTTPDProxyHandler { final Client client = setupHttpClient(requestHeader, connectHost); // send request - try { - res = client.HEAD(getUrl); - if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); +// try { +// res = client.HEAD(getUrl); +// if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); + client.HEADResponse(getUrl); + if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine()); // determine if it's an internal error of the httpc - final ResponseHeader responseHeader = res.getResponseHeader(); +// final ResponseHeader responseHeader = res.getResponseHeader(); +// if (responseHeader.isEmpty()) { +// throw new Exception(res.getStatusLine()); +// } + final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap()); if (responseHeader.isEmpty()) { - throw new Exception(res.getStatusLine()); - } + throw new Exception(client.getHttpResponse().getStatusLine().toString()); + } - prepareResponseHeader(responseHeader, res.getHttpVer()); +// prepareResponseHeader(responseHeader, res.getHttpVer()); + prepareResponseHeader(responseHeader, client.getHttpResponse().getStatusLine().getProtocolVersion().toString()); // sending the server respond back to the client if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader); - HTTPDemon.sendRespondHeader(conProp,respond,httpVer,res.getStatusCode(),res.getStatusLine().substring(4),responseHeader); +// HTTPDemon.sendRespondHeader(conProp,respond,httpVer,res.getStatusCode(),res.getStatusLine().substring(4),responseHeader); + HTTPDemon.sendRespondHeader( + conProp, + respond, + httpVer, + client.getHttpResponse().getStatusLine().getStatusCode(), + client.getHttpResponse().getStatusLine().toString(), + responseHeader); respond.flush(); - } finally { - if(res != null) { - // ... close connection - res.closeStream(); - } - } +// } finally { +// if(res != null) { +// // ... close connection +// res.closeStream(); +// } +// } } catch (final Exception e) { handleProxyException(e,conProp,respond,url); } @@ -901,6 +936,10 @@ public final class HTTPDProxyHandler { final String getUrl = "http://"+ connectHost + remotePath; if (log.isFinest()) log.logFinest(reqID +" using url: "+ getUrl); + // the CONTENT_LENGTH will be added by entity and cause a ClientProtocolException if set + final int contentLength = requestHeader.getContentLength(); + requestHeader.remove(RequestHeader.CONTENT_LENGTH); + final Client client = setupHttpClient(requestHeader, connectHost); // check input @@ -911,79 +950,95 @@ public final class HTTPDProxyHandler { // "if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader" // it seems that it is a HTTP/1.1 connection which stays open (the inputStream) and endlessly waits for // input so we have to end it to do the request - final int contentLength = requestHeader.getContentLength(); - if (contentLength > -1) { - final byte[] bodyData; - if(contentLength == 0) { - // no body - bodyData = new byte[0]; - } else { - // read content-length bytes into memory - bodyData = new byte[contentLength]; - int bytes_read = 0; - while(bytes_read < contentLength) { - bytes_read += body.read(bodyData, bytes_read, contentLength-bytes_read); - } - } - body = new ByteArrayInputStream(bodyData); - } - ResponseContainer res = null; - try { - // sending the request - res = client.POST(getUrl, body); - if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); - - final ResponseHeader responseHeader = res.getResponseHeader(); - // determine if it's an internal error of the httpc - if (responseHeader.isEmpty()) { - throw new Exception(res.getStatusLine()); - } - - final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), countedRespond); - - prepareResponseHeader(responseHeader, res.getHttpVer()); - - // sending the respond header back to the client - if (chunked != null) { - responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked"); - } - - // sending response headers - if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader); - HTTPDemon.sendRespondHeader(conProp, - countedRespond, - httpVer, - res.getStatusCode(), - res.getStatusLine().substring(4), // status text - responseHeader); - - // respondHeader(respond, res.status, res.responseHeader); - // Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond)); - /* - // *** (Uebernommen aus Saver-Klasse: warum ist dies hier die einzige Methode, die einen OutputStream statt einen Writer benutzt?) + // this should not be needed anymore - see org.apache.http.entity.InputStreamEntity +// final int contentLength = requestHeader.getContentLength(); +// if (contentLength > -1) { +// final byte[] bodyData; +// if(contentLength == 0) { +// // no body +// bodyData = new byte[0]; +// } else { +// // read content-length bytes into memory +// bodyData = new byte[contentLength]; +// int bytes_read = 0; +// while(bytes_read < contentLength) { +// bytes_read += body.read(bodyData, bytes_read, contentLength-bytes_read); +// } +// } +// body = new ByteArrayInputStream(bodyData); +// } +// ResponseContainer res = null; try { - serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond)); - } finally { - res.closeStream(); - } - if (chunked != null) chunked.finish(); - */ - final OutputStream outStream = (chunked != null) ? chunked : countedRespond; - FileUtils.copy(res.getDataAsStream(), outStream); - - if (chunked != null) { - chunked.finish(); - } - outStream.flush(); + // sending the request +// res = client.POST(getUrl, body); +// if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); + client.POST(getUrl, body, contentLength); + if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine()); + +// final ResponseHeader responseHeader = res.getResponseHeader(); + final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap()); + // determine if it's an internal error of the httpc + if (responseHeader.isEmpty()) { +// throw new Exception(res.getStatusLine()); + throw new Exception(client.getHttpResponse().getStatusLine().toString()); + } + +// final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), countedRespond); + final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), countedRespond); + +// prepareResponseHeader(responseHeader, res.getHttpVer()); + prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString()); + + // sending the respond header back to the client + if (chunked != null) { + responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked"); + } + + // sending response headers + if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader); +// HTTPDemon.sendRespondHeader(conProp, +// countedRespond, +// httpVer, +// res.getStatusCode(), +// res.getStatusLine().substring(4), // status text +// responseHeader); + HTTPDemon.sendRespondHeader(conProp, + countedRespond, + httpVer, + client.getHttpResponse().getStatusLine().getStatusCode(), + client.getHttpResponse().getStatusLine().toString(), // status text + responseHeader); + + // respondHeader(respond, res.status, res.responseHeader); + // Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond)); + /* + // *** (Uebernommen aus Saver-Klasse: warum ist dies hier die einzige Methode, die einen OutputStream statt einen Writer benutzt?) + try { + serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond)); + } finally { + res.closeStream(); + } + if (chunked != null) chunked.finish(); + */ + final OutputStream outStream = (chunked != null) ? chunked : countedRespond; +// FileUtils.copy(res.getDataAsStream(), outStream); + client.writeTo(outStream); + + if (chunked != null) { + chunked.finish(); + } + outStream.flush(); } catch(SocketException se) { // connection closed by client, abort download - res.abort(); +// res.abort(); + client.finish(); } finally { // if opened ... - if(res != null) { - // ... close connection - res.closeStream(); - } +// if(res != null) { +// // ... close connection +// res.closeStream(); +// } + client.finish(); } } catch (final Exception e) { handleProxyException(e,conProp,countedRespond,url); @@ -1069,8 +1124,12 @@ public final class HTTPDProxyHandler { */ private static Client setupHttpClient(final RequestHeader requestHeader, final String connectHost) { // setup HTTP-client - final Client client = new Client(timeout, requestHeader); - client.setFollowRedirects(false); +// final Client client = new Client(timeout, requestHeader); +// client.setFollowRedirects(false); + final Client client = new Client(); + client.setTimout(timeout); + client.setHeader(requestHeader.entrySet()); + client.setRedirecting(false); return client; } @@ -1240,16 +1299,22 @@ public final class HTTPDProxyHandler { // possibly branch into PROXY-PROXY connection if (ProxySettings.use && ProxySettings.use4ssl) { - final Client remoteProxy = new Client(timeout, requestHeader); - remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe +// final Client remoteProxy = new Client(timeout, requestHeader); +// remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe + final Client remoteProxy = setupHttpClient(requestHeader, host); - ResponseContainer response = null; +// ResponseContainer response = null; try { - response = remoteProxy.CONNECT(host, port); +// response = remoteProxy.CONNECT(host, port); + remoteProxy.HEADResponse("http://" + host + ":" + port); + ResponseHeader header = new ResponseHeader(null, remoteProxy.getHeaderHashMap()); + // outputs a logline to the serverlog with the current status - log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString()); - // (response.getStatusLine().charAt(0) == '2') || (response.getStatusLine().charAt(0) == '3') - final boolean success = response.getStatusCode() >= 200 && response.getStatusCode() <= 399; +// log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString()); +// // (response.getStatusLine().charAt(0) == '2') || (response.getStatusLine().charAt(0) == '3') +// final boolean success = response.getStatusCode() >= 200 && response.getStatusCode() <= 399; + log.logInfo("CONNECT-RESPONSE: status=" + remoteProxy.getHttpResponse().getStatusLine() + ", header=" + header.toString()); + final boolean success = remoteProxy.getHttpResponse().getStatusLine().getStatusCode() >= 200 && remoteProxy.getHttpResponse().getStatusLine().getStatusCode() <= 399; if (success) { // replace connection details host = ProxySettings.host; @@ -1257,21 +1322,28 @@ public final class HTTPDProxyHandler { // go on (see below) } else { // pass error response back to client - HTTPDemon.sendRespondHeader(conProp,clientOut,httpVersion,response.getStatusCode(),response.getStatusLine().substring(4),response.getResponseHeader()); +// HTTPDemon.sendRespondHeader(conProp,clientOut,httpVersion,response.getStatusCode(),response.getStatusLine().substring(4),response.getResponseHeader()); + HTTPDemon.sendRespondHeader( + conProp, + clientOut, + httpVersion, + remoteProxy.getHttpResponse().getStatusLine().getStatusCode(), + remoteProxy.getHttpResponse().getStatusLine().toString(), + header); //respondHeader(clientOut, response.status, response.responseHeader); forceConnectionClose(conProp); return; } - } catch (SocketException se) { - // connection closed by client, abort download - response.abort(); +// } catch (SocketException se) { +// // connection closed by client, abort download +// response.abort(); } catch (final Exception e) { throw new IOException(e.getMessage()); - } finally { - if(response != null) { - // release connection - response.closeStream(); - } +// } finally { +// if(response != null) { +// // release connection +// response.closeStream(); +// } } } diff --git a/source/de/anomic/yacy/yacyRelease.java b/source/de/anomic/yacy/yacyRelease.java index c3043d1cc..848768193 100644 --- a/source/de/anomic/yacy/yacyRelease.java +++ b/source/de/anomic/yacy/yacyRelease.java @@ -27,7 +27,7 @@ package de.anomic.yacy; -import java.io.BufferedInputStream; +//import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; @@ -46,6 +46,7 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.protocol.Client; import net.yacy.document.parser.html.ContentScraper; import net.yacy.kelondro.io.CharBuffer; import net.yacy.kelondro.logging.Log; @@ -55,10 +56,11 @@ import net.yacy.kelondro.util.OS; import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.retrieval.HTTPLoader; -import de.anomic.http.client.Client; +//import de.anomic.http.client.Client; import de.anomic.http.server.HeaderFramework; import de.anomic.http.server.RequestHeader; -import de.anomic.http.server.ResponseContainer; +//import de.anomic.http.server.ResponseContainer; +import de.anomic.http.server.ResponseHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverCore; import de.anomic.tools.CryptoLib; @@ -285,14 +287,19 @@ public final class yacyRelease extends yacyVersion { final RequestHeader reqHeader = new RequestHeader(); reqHeader.put(HeaderFramework.USER_AGENT, HTTPLoader.yacyUserAgent); - ResponseContainer res = null; +// ResponseContainer res = null; final String name = this.getUrl().getFileName(); byte[] signatureBytes = null; + final Client client = new Client(); + client.setTimout(6000); + client.setHeader(reqHeader.entrySet()); + // download signature first, if public key is available try { if (this.publicKey != null) { - final byte[] signatureData = Client.wget(this.getUrl().toString() + ".sig", reqHeader, 6000); +// final byte[] signatureData = Client.wget(this.getUrl().toString() + ".sig", reqHeader, 6000); + final byte[] signatureData = client.GETbytes(this.getUrl().toString()); if (signatureData == null) { Log.logWarning("yacyVersion", "download of signature " + this.getUrl().toString() + " failed. ignoring signature file."); } else try { @@ -302,10 +309,14 @@ public final class yacyRelease extends yacyVersion { } // in case that the download of a signature file failed (can be caused by bad working http servers), then it is assumed that no signature exists } - final Client client = new Client(120000, reqHeader); - res = client.GET(this.getUrl().toString()); +// final Client client = new Client(120000, reqHeader); +// res = client.GET(this.getUrl().toString()); + client.setTimout(120000); + client.GET(this.getUrl().toString()); + final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap()); - final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file +// final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file + final boolean unzipped = header.gzip() && (header.mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file if ((unzipped) && (name.endsWith(".tar.gz"))) { download = new File(storagePath, name.substring(0, name.length() - 3)); } else { @@ -316,7 +327,8 @@ public final class yacyRelease extends yacyVersion { SignatureOutputStream verifyOutput = null; try { verifyOutput = new SignatureOutputStream(new FileOutputStream(download), CryptoLib.signAlgorithm, publicKey); - FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(verifyOutput)); +// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(verifyOutput)); + client.writeTo(new BufferedOutputStream(verifyOutput)); if (!verifyOutput.verify(signatureBytes)) throw new IOException("Bad Signature!"); } catch (NoSuchAlgorithmException e) { @@ -333,12 +345,13 @@ public final class yacyRelease extends yacyVersion { if ((!signatureFile.exists()) || (signatureFile.length() == 0)) throw new IOException("create signature file failed"); } else { // just copy into file - FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download))); +// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download))); + client.writeTo(new BufferedOutputStream(new FileOutputStream(download))); } if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + this.getUrl() + " failed"); } catch (final IOException e) { // Saving file failed, abort download - if (res != null) res.abort(); +// if (res != null) res.abort(); Log.logSevere("yacyVersion", "download of " + this.getName() + " failed: " + e.getMessage()); if (download != null && download.exists()) { FileUtils.deletedelete(download); @@ -346,10 +359,15 @@ public final class yacyRelease extends yacyVersion { } download = null; } finally { - if (res != null) { - // release connection - res.closeStream(); - } +// if (res != null) { +// // release connection +// res.closeStream(); +// } + try { + client.finish(); + } catch (IOException e) { + Log.logSevere("yacyVersion", "finish of " + this.getName() + " failed: " + e.getMessage()); + } } this.releaseFile = download; Switchboard.getSwitchboard().setConfig("update.time.download", System.currentTimeMillis()); diff --git a/source/net/yacy/cora/protocol/Client.java b/source/net/yacy/cora/protocol/Client.java index 2d66f4dab..64d6b2398 100644 --- a/source/net/yacy/cora/protocol/Client.java +++ b/source/net/yacy/cora/protocol/Client.java @@ -1,6 +1,8 @@ package net.yacy.cora.protocol; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.util.HashMap; import java.util.LinkedHashMap; @@ -13,11 +15,13 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpVersion; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.params.HttpClientParams; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.params.ConnManagerParams; import org.apache.http.conn.params.ConnPerRouteBean; @@ -27,6 +31,7 @@ import org.apache.http.conn.scheme.PlainSocketFactory; import org.apache.http.conn.scheme.Scheme; import org.apache.http.conn.scheme.SchemeRegistry; import org.apache.http.conn.ssl.SSLSocketFactory; +import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; @@ -57,10 +62,12 @@ public class Client { private static HttpClient httpClient = null; private Header[] headers = null; private HttpResponse httpResponse = null; + private HttpUriRequest currentRequest = null; private long upbytes = 0L; private int timeout = 10000; private String userAgent = null; private String host = null; + private boolean redirecting = true; public Client() { super(); @@ -181,6 +188,14 @@ public class Client { this.host = host; } + /** + * + * @param redirecting + */ + public void setRedirecting(final boolean redirecting) { + this.redirecting = redirecting; + } + /** * This method GETs a page from the server. * @@ -205,6 +220,21 @@ public class Client { return getContentBytes(httpGet, maxBytes); } + /** + * This method GETs a page from the server. + * to be used for streaming out + * Please take care to call finish()! + * + * @param uri the url to get + * @throws IOException + */ + public void GET(final String uri) throws IOException { + if (currentRequest != null) throw new IOException("Client is in use!"); + final HttpGet httpGet = new HttpGet(uri); + currentRequest = httpGet; + execute(httpGet); + } + /** * This method gets HEAD response * @@ -214,18 +244,42 @@ public class Client { */ public HttpResponse HEADResponse(final String uri) throws IOException { final HttpHead httpHead = new HttpHead(uri); - getContentBytes(httpHead, Long.MAX_VALUE); + execute(httpHead); + finish(); + ConnectionInfo.removeConnection(httpHead.hashCode()); return httpResponse; } /** + * This method POSTs a page from the server. + * to be used for streaming out + * Please take care to call finish()! + * + * @param uri the url to post + * @param instream the input to post + * @param length the contentlength + * @throws IOException + */ + public void POST(final String uri, final InputStream instream, long length) throws IOException { + if (currentRequest != null) throw new IOException("Client is in use!"); + final HttpPost httpPost = new HttpPost(uri); + final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length); + // statistics + upbytes = length; + httpPost.setEntity(inputStreamEntity); + currentRequest = httpPost; + execute(httpPost); + } + + /** + * This method POSTs a page from the server. * * @param uri the url to post * @param parts to post * @return content bytes * @throws IOException */ - public byte[] POSTbytes(final String uri, LinkedHashMap parts) throws IOException { + public byte[] POSTbytes(final String uri, final LinkedHashMap parts) throws IOException { final HttpPost httpPost = new HttpPost(uri); final MultipartEntity multipartEntity = new MultipartEntity(); @@ -255,18 +309,48 @@ public class Client { } return hmap; } + + public void writeTo(final OutputStream outputStream) throws IOException { + if (httpResponse != null && currentRequest != null) { + final HttpEntity httpEntity = httpResponse.getEntity(); + if (httpEntity != null) try { + httpEntity.writeTo(outputStream); + outputStream.flush(); + // TODO: The name of this method is misnomer. + // It will be renamed to #finish() in the next major release of httpcore + httpEntity.consumeContent(); + ConnectionInfo.removeConnection(currentRequest.hashCode()); + currentRequest = null; + } catch (final IOException e) { + currentRequest.abort(); + ConnectionInfo.removeConnection(currentRequest.hashCode()); + currentRequest = null; + throw e; + } + } + } + + public void finish() throws IOException { + if (httpResponse != null) { + final HttpEntity httpEntity = httpResponse.getEntity(); + if (httpEntity != null && httpEntity.isStreaming()) { + // TODO: The name of this method is misnomer. + // It will be renamed to #finish() in the next major release of httpcore + httpEntity.consumeContent(); + } + } + if (currentRequest != null) { + currentRequest.abort(); + ConnectionInfo.removeConnection(currentRequest.hashCode()); + currentRequest = null; + } + } - private byte[] getContentBytes(HttpUriRequest httpUriRequest, long maxBytes) throws IOException { + private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final long maxBytes) throws IOException { byte[] content = null; - final HttpContext httpContext = new BasicHttpContext(); - setHeaders(httpUriRequest); - setParams(httpUriRequest.getParams()); - setProxy(httpUriRequest.getParams()); - // statistics - storeConnectionInfo(httpUriRequest); try { - // execute the method - httpResponse = httpClient.execute(httpUriRequest, httpContext); + execute(httpUriRequest); + if (httpResponse == null) return null; // get the response body final HttpEntity httpEntity = httpResponse.getEntity(); if (httpEntity != null) { @@ -286,7 +370,24 @@ public class Client { return content; } - private void setHeaders(HttpUriRequest httpUriRequest) { + private void execute(final HttpUriRequest httpUriRequest) throws IOException { + final HttpContext httpContext = new BasicHttpContext(); + setHeaders(httpUriRequest); + setParams(httpUriRequest.getParams()); + setProxy(httpUriRequest.getParams()); + // statistics + storeConnectionInfo(httpUriRequest); + try { + // execute the method + httpResponse = httpClient.execute(httpUriRequest, httpContext); + } catch (ClientProtocolException e) { + httpUriRequest.abort(); + ConnectionInfo.removeConnection(httpUriRequest.hashCode()); + throw new IOException("Client can't execute: " + e.getMessage()); + } + } + + private void setHeaders(final HttpUriRequest httpUriRequest) { if (headers != null) { for (Header header : headers) { httpUriRequest.addHeader(header); @@ -294,7 +395,8 @@ public class Client { } } - private void setParams(HttpParams httpParams) { + private void setParams(final HttpParams httpParams) { + HttpClientParams.setRedirecting(httpParams, redirecting); HttpConnectionParams.setConnectionTimeout(httpParams, timeout); HttpConnectionParams.setSoTimeout(httpParams, timeout); if (userAgent != null) @@ -303,14 +405,14 @@ public class Client { httpParams.setParameter(HTTP.TARGET_HOST, host); } - private void setProxy(HttpParams httpParams) { + private void setProxy(final HttpParams httpParams) { if (ProxySettings.use) ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost()); // TODO find a better way for this ProxySettings.setProxyCreds((AbstractHttpClient) httpClient); } - private void storeConnectionInfo(HttpUriRequest httpUriRequest) { + private void storeConnectionInfo(final HttpUriRequest httpUriRequest) { final int port = httpUriRequest.getURI().getPort(); final String thost = httpUriRequest.getURI().getHost(); ConnectionInfo.addConnection(new ConnectionInfo( @@ -368,7 +470,7 @@ public class Client { } Client client = new Client(); client.setUserAgent("foobar"); - client.setHost("sixcooler"); + client.setRedirecting(false); // Get some for (int i = 0; i < args.length; i++) { url = args[i]; diff --git a/source/net/yacy/cora/protocol/GzipDecompressingEntity.java b/source/net/yacy/cora/protocol/GzipDecompressingEntity.java index 4b2f3d128..4202a011b 100644 --- a/source/net/yacy/cora/protocol/GzipDecompressingEntity.java +++ b/source/net/yacy/cora/protocol/GzipDecompressingEntity.java @@ -2,12 +2,15 @@ package net.yacy.cora.protocol; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.util.zip.GZIPInputStream; import org.apache.http.HttpEntity; import org.apache.http.entity.HttpEntityWrapper; public class GzipDecompressingEntity extends HttpEntityWrapper { + + private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size public GzipDecompressingEntity(final HttpEntity entity) { super(entity); @@ -20,6 +23,22 @@ public class GzipDecompressingEntity extends HttpEntityWrapper { return new GZIPInputStream(wrappedin); } + + public void writeTo(OutputStream outstream) throws IOException { + if (outstream == null) { + throw new IllegalArgumentException("Output stream may not be null"); + } + InputStream instream = this.getContent(); + int l; + byte[] tmp = new byte[DEFAULT_BUFFER_SIZE]; + while ((l = instream.read(tmp)) != -1) { + outstream.write(tmp, 0, l); + } + } + + public boolean isChunked() { + return true; + } public long getContentLength() { // length of ungzipped content not known in advance