boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

... migrating to HttpComponents-Client-4.x ...

Browse Source 
- Proxy
- Release-download

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7011 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 15 years ago
parent ced07970c1
commit c29f24a519
4 changed files with 367 additions and 156 deletions
Show Stats Download Patch File Download Diff File

246
source/de/anomic/http/server/HTTPDProxyHandler.java
Unescape View File

@ -43,7 +43,7 @@
package de.anomic.http.server; package de.anomic.http.server;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.ByteArrayInputStream; //import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@ -71,6 +71,7 @@ import java.util.logging.LogManager;
import java.util.logging.Logger; import java.util.logging.Logger;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
import net.yacy.cora.protocol.Client;
import net.yacy.cora.protocol.ProxySettings; import net.yacy.cora.protocol.ProxySettings;
import net.yacy.document.TextParser; import net.yacy.document.TextParser;
import net.yacy.document.parser.html.ContentTransformer; import net.yacy.document.parser.html.ContentTransformer;
@ -87,7 +88,7 @@ import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.crawler.retrieval.Request; import de.anomic.crawler.retrieval.Request;
import de.anomic.crawler.retrieval.Response; import de.anomic.crawler.retrieval.Response;
import de.anomic.http.client.MultiOutputStream; import de.anomic.http.client.MultiOutputStream;
import de.anomic.http.client.Client; //import de.anomic.http.client.Client;
import de.anomic.http.client.Cache; import de.anomic.http.client.Cache;
import de.anomic.search.Switchboard; import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants; import de.anomic.search.SwitchboardConstants;
@ -439,7 +440,7 @@ public final class HTTPDProxyHandler {
final GZIPOutputStream gzippedOut = null; final GZIPOutputStream gzippedOut = null;
ResponseContainer res = null; // ResponseContainer res = null;
try { try {
final int reqID = requestHeader.hashCode(); final int reqID = requestHeader.hashCode();
@ -478,17 +479,22 @@ public final class HTTPDProxyHandler {
// send request // send request
try { try {
res = client.GET(getUrl); // res = client.GET(getUrl);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); // if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
client.GET(getUrl);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader); conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader);
final ResponseHeader responseHeader = res.getResponseHeader(); // final ResponseHeader responseHeader = res.getResponseHeader();
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
// determine if it's an internal error of the httpc // determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) { if (responseHeader.isEmpty()) {
throw new Exception(res.getStatusLine()); // throw new Exception(res.getStatusLine());
throw new Exception(client.getHttpResponse().getStatusLine().toString());
} }
final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond); // final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond);
final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), respond);
// the cache does either not exist or is (supposed to be) stale // the cache does either not exist or is (supposed to be) stale
long sizeBeforeDelete = -1; long sizeBeforeDelete = -1;
@ -519,7 +525,8 @@ public final class HTTPDProxyHandler {
// handle incoming cookies // handle incoming cookies
handleIncomingCookies(responseHeader, host, ip); handleIncomingCookies(responseHeader, host, ip);
prepareResponseHeader(responseHeader, res.getHttpVer()); // prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
// sending the respond header back to the client // sending the respond header back to the client
if (chunkedOut != null) { if (chunkedOut != null) {
@ -527,22 +534,31 @@ public final class HTTPDProxyHandler {
} }
if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader); if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader);
// HTTPDemon.sendRespondHeader(
// conProp,
// respond,
// httpVer,
// res.getStatusCode(),
// res.getStatusLine().substring(4), // status text
// responseHeader);
HTTPDemon.sendRespondHeader( HTTPDemon.sendRespondHeader(
conProp, conProp,
respond, respond,
httpVer, httpVer,
res.getStatusCode(), client.getHttpResponse().getStatusLine().getStatusCode(),
res.getStatusLine().substring(4), // status text client.getHttpResponse().getStatusLine().toString(), // status text
responseHeader); responseHeader);
if (hasBody(res.getStatusCode())) { // if (hasBody(res.getStatusCode())) {
if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) {
final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond); final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
final Response response = new Response( final Response response = new Response(
request, request,
requestHeader, requestHeader,
responseHeader, responseHeader,
res.getStatusLine(), // res.getStatusLine(),
Integer.toString(client.getHttpResponse().getStatusLine().getStatusCode()),
sb.crawler.defaultProxyProfile sb.crawler.defaultProxyProfile
); );
final String storeError = response.shallStoreCacheForProxy(); final String storeError = response.shallStoreCacheForProxy();
@ -561,11 +577,13 @@ public final class HTTPDProxyHandler {
((storeHTCache) || (supportError != null)) ((storeHTCache) || (supportError != null))
) { ) {
// we don't write actually into a file, only to RAM, and schedule writing the file. // we don't write actually into a file, only to RAM, and schedule writing the file.
int l = res.getResponseHeader().size(); // int l = res.getResponseHeader().size();
int l = responseHeader.size();
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l); final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);
final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream}); final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});
FileUtils.copy(res.getDataAsStream(), toClientAndMemory); // FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
client.writeTo(toClientAndMemory);
// cached bytes // cached bytes
byte[] cacheArray; byte[] cacheArray;
if (byteStream.size() > 0) { if (byteStream.size() > 0) {
@ -608,7 +626,8 @@ public final class HTTPDProxyHandler {
" StoreHTCache=" + storeHTCache + " StoreHTCache=" + storeHTCache +
" SupportError=" + supportError); " SupportError=" + supportError);
FileUtils.copy(res.getDataAsStream(), outStream); // FileUtils.copy(res.getDataAsStream(), outStream);
client.writeTo(outStream);
conProp.setProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); conProp.setProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
} }
@ -623,17 +642,19 @@ public final class HTTPDProxyHandler {
} // end hasBody } // end hasBody
} catch(SocketException se) { } catch(SocketException se) {
// if opened ... // if opened ...
if(res != null) { // if(res != null) {
// client cut proxy connection, abort download // // client cut proxy connection, abort download
res.abort(); // res.abort();
} // }
client.finish();
handleProxyException(se,conProp,respond,url); handleProxyException(se,conProp,respond,url);
} finally { } finally {
// if opened ... // if opened ...
if(res != null) { // if(res != null) {
// ... close connection // // ... close connection
res.closeStream(); // res.closeStream();
} // }
client.finish();
} }
} catch (final Exception e) { } catch (final Exception e) {
handleProxyException(e,conProp,respond,url); handleProxyException(e,conProp,respond,url);
@ -741,7 +762,7 @@ public final class HTTPDProxyHandler {
public static void doHead(final Properties conProp, final RequestHeader requestHeader, OutputStream respond) { public static void doHead(final Properties conProp, final RequestHeader requestHeader, OutputStream respond) {
ResponseContainer res = null; // ResponseContainer res = null;
DigestURI url = null; DigestURI url = null;
try { try {
final int reqID = requestHeader.hashCode(); final int reqID = requestHeader.hashCode();
@ -813,28 +834,42 @@ public final class HTTPDProxyHandler {
final Client client = setupHttpClient(requestHeader, connectHost); final Client client = setupHttpClient(requestHeader, connectHost);
// send request // send request
try { // try {
res = client.HEAD(getUrl); // res = client.HEAD(getUrl);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); // if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
client.HEADResponse(getUrl);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
// determine if it's an internal error of the httpc // determine if it's an internal error of the httpc
final ResponseHeader responseHeader = res.getResponseHeader(); // final ResponseHeader responseHeader = res.getResponseHeader();
// if (responseHeader.isEmpty()) {
// throw new Exception(res.getStatusLine());
// }
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
if (responseHeader.isEmpty()) { if (responseHeader.isEmpty()) {
throw new Exception(res.getStatusLine()); throw new Exception(client.getHttpResponse().getStatusLine().toString());
} }
prepareResponseHeader(responseHeader, res.getHttpVer()); // prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getStatusLine().getProtocolVersion().toString());
// sending the server respond back to the client // sending the server respond back to the client
if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader); if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader);
HTTPDemon.sendRespondHeader(conProp,respond,httpVer,res.getStatusCode(),res.getStatusLine().substring(4),responseHeader); // HTTPDemon.sendRespondHeader(conProp,respond,httpVer,res.getStatusCode(),res.getStatusLine().substring(4),responseHeader);
HTTPDemon.sendRespondHeader(
conProp,
respond,
httpVer,
client.getHttpResponse().getStatusLine().getStatusCode(),
client.getHttpResponse().getStatusLine().toString(),
responseHeader);
respond.flush(); respond.flush();
} finally { // } finally {
if(res != null) { // if(res != null) {
// ... close connection // // ... close connection
res.closeStream(); // res.closeStream();
} // }
} // }
} catch (final Exception e) { } catch (final Exception e) {
handleProxyException(e,conProp,respond,url); handleProxyException(e,conProp,respond,url);
} }
@ -901,6 +936,10 @@ public final class HTTPDProxyHandler {
final String getUrl = "http://"+ connectHost + remotePath; final String getUrl = "http://"+ connectHost + remotePath;
if (log.isFinest()) log.logFinest(reqID +" using url: "+ getUrl); if (log.isFinest()) log.logFinest(reqID +" using url: "+ getUrl);
// the CONTENT_LENGTH will be added by entity and cause a ClientProtocolException if set
final int contentLength = requestHeader.getContentLength();
requestHeader.remove(RequestHeader.CONTENT_LENGTH);
final Client client = setupHttpClient(requestHeader, connectHost); final Client client = setupHttpClient(requestHeader, connectHost);
// check input // check input
@ -911,37 +950,44 @@ public final class HTTPDProxyHandler {
// "if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader" // "if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader"
// it seems that it is a HTTP/1.1 connection which stays open (the inputStream) and endlessly waits for // it seems that it is a HTTP/1.1 connection which stays open (the inputStream) and endlessly waits for
// input so we have to end it to do the request // input so we have to end it to do the request
final int contentLength = requestHeader.getContentLength(); // this should not be needed anymore - see org.apache.http.entity.InputStreamEntity
if (contentLength > -1) { // final int contentLength = requestHeader.getContentLength();
final byte[] bodyData; // if (contentLength > -1) {
if(contentLength == 0) { // final byte[] bodyData;
// no body // if(contentLength == 0) {
bodyData = new byte[0]; // // no body
} else { // bodyData = new byte[0];
// read content-length bytes into memory // } else {
bodyData = new byte[contentLength]; // // read content-length bytes into memory
int bytes_read = 0; // bodyData = new byte[contentLength];
while(bytes_read < contentLength) { // int bytes_read = 0;
bytes_read += body.read(bodyData, bytes_read, contentLength-bytes_read); // while(bytes_read < contentLength) {
} // bytes_read += body.read(bodyData, bytes_read, contentLength-bytes_read);
} // }
body = new ByteArrayInputStream(bodyData); // }
} // body = new ByteArrayInputStream(bodyData);
ResponseContainer res = null; // }
// ResponseContainer res = null;
try { try {
// sending the request // sending the request
res = client.POST(getUrl, body); // res = client.POST(getUrl, body);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine()); // if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
client.POST(getUrl, body, contentLength);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
final ResponseHeader responseHeader = res.getResponseHeader(); // final ResponseHeader responseHeader = res.getResponseHeader();
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
// determine if it's an internal error of the httpc // determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) { if (responseHeader.isEmpty()) {
throw new Exception(res.getStatusLine()); // throw new Exception(res.getStatusLine());
throw new Exception(client.getHttpResponse().getStatusLine().toString());
} }
final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), countedRespond); // final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), countedRespond);
final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), countedRespond);
prepareResponseHeader(responseHeader, res.getHttpVer()); // prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
// sending the respond header back to the client // sending the respond header back to the client
if (chunked != null) { if (chunked != null) {
@ -950,11 +996,17 @@ public final class HTTPDProxyHandler {
// sending response headers // sending response headers
if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader); if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader);
// HTTPDemon.sendRespondHeader(conProp,
// countedRespond,
// httpVer,
// res.getStatusCode(),
// res.getStatusLine().substring(4), // status text
// responseHeader);
HTTPDemon.sendRespondHeader(conProp, HTTPDemon.sendRespondHeader(conProp,
countedRespond, countedRespond,
httpVer, httpVer,
res.getStatusCode(), client.getHttpResponse().getStatusLine().getStatusCode(),
res.getStatusLine().substring(4), // status text client.getHttpResponse().getStatusLine().toString(), // status text
responseHeader); responseHeader);
// respondHeader(respond, res.status, res.responseHeader); // respondHeader(respond, res.status, res.responseHeader);
@ -969,7 +1021,8 @@ public final class HTTPDProxyHandler {
if (chunked != null) chunked.finish(); if (chunked != null) chunked.finish();
*/ */
final OutputStream outStream = (chunked != null) ? chunked : countedRespond; final OutputStream outStream = (chunked != null) ? chunked : countedRespond;
FileUtils.copy(res.getDataAsStream(), outStream); // FileUtils.copy(res.getDataAsStream(), outStream);
client.writeTo(outStream);
if (chunked != null) { if (chunked != null) {
chunked.finish(); chunked.finish();
@ -977,13 +1030,15 @@ public final class HTTPDProxyHandler {
outStream.flush(); outStream.flush();
} catch(SocketException se) { } catch(SocketException se) {
// connection closed by client, abort download // connection closed by client, abort download
res.abort(); // res.abort();
client.finish();
} finally { } finally {
// if opened ... // if opened ...
if(res != null) { // if(res != null) {
// ... close connection // // ... close connection
res.closeStream(); // res.closeStream();
} // }
client.finish();
} }
} catch (final Exception e) { } catch (final Exception e) {
handleProxyException(e,conProp,countedRespond,url); handleProxyException(e,conProp,countedRespond,url);
@ -1069,8 +1124,12 @@ public final class HTTPDProxyHandler {
*/ */
private static Client setupHttpClient(final RequestHeader requestHeader, final String connectHost) { private static Client setupHttpClient(final RequestHeader requestHeader, final String connectHost) {
// setup HTTP-client // setup HTTP-client
final Client client = new Client(timeout, requestHeader); // final Client client = new Client(timeout, requestHeader);
client.setFollowRedirects(false); // client.setFollowRedirects(false);
final Client client = new Client();
client.setTimout(timeout);
client.setHeader(requestHeader.entrySet());
client.setRedirecting(false);
return client; return client;
} }
@ -1240,16 +1299,22 @@ public final class HTTPDProxyHandler {
// possibly branch into PROXY-PROXY connection // possibly branch into PROXY-PROXY connection
if (ProxySettings.use && ProxySettings.use4ssl) { if (ProxySettings.use && ProxySettings.use4ssl) {
final Client remoteProxy = new Client(timeout, requestHeader); // final Client remoteProxy = new Client(timeout, requestHeader);
remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe // remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe
final Client remoteProxy = setupHttpClient(requestHeader, host);
ResponseContainer response = null; // ResponseContainer response = null;
try { try {
response = remoteProxy.CONNECT(host, port); // response = remoteProxy.CONNECT(host, port);
remoteProxy.HEADResponse("http://" + host + ":" + port);
ResponseHeader header = new ResponseHeader(null, remoteProxy.getHeaderHashMap());
// outputs a logline to the serverlog with the current status // outputs a logline to the serverlog with the current status
log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString()); // log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString());
// (response.getStatusLine().charAt(0) == '2') || (response.getStatusLine().charAt(0) == '3') // // (response.getStatusLine().charAt(0) == '2') || (response.getStatusLine().charAt(0) == '3')
final boolean success = response.getStatusCode() >= 200 && response.getStatusCode() <= 399; // final boolean success = response.getStatusCode() >= 200 && response.getStatusCode() <= 399;
log.logInfo("CONNECT-RESPONSE: status=" + remoteProxy.getHttpResponse().getStatusLine() + ", header=" + header.toString());
final boolean success = remoteProxy.getHttpResponse().getStatusLine().getStatusCode() >= 200 && remoteProxy.getHttpResponse().getStatusLine().getStatusCode() <= 399;
if (success) { if (success) {
// replace connection details // replace connection details
host = ProxySettings.host; host = ProxySettings.host;
@ -1257,21 +1322,28 @@ public final class HTTPDProxyHandler {
// go on (see below) // go on (see below)
} else { } else {
// pass error response back to client // pass error response back to client
HTTPDemon.sendRespondHeader(conProp,clientOut,httpVersion,response.getStatusCode(),response.getStatusLine().substring(4),response.getResponseHeader()); // HTTPDemon.sendRespondHeader(conProp,clientOut,httpVersion,response.getStatusCode(),response.getStatusLine().substring(4),response.getResponseHeader());
HTTPDemon.sendRespondHeader(
conProp,
clientOut,
httpVersion,
remoteProxy.getHttpResponse().getStatusLine().getStatusCode(),
remoteProxy.getHttpResponse().getStatusLine().toString(),
header);
//respondHeader(clientOut, response.status, response.responseHeader); //respondHeader(clientOut, response.status, response.responseHeader);
forceConnectionClose(conProp); forceConnectionClose(conProp);
return; return;
} }
} catch (SocketException se) { // } catch (SocketException se) {
// connection closed by client, abort download // // connection closed by client, abort download
response.abort(); // response.abort();
} catch (final Exception e) { } catch (final Exception e) {
throw new IOException(e.getMessage()); throw new IOException(e.getMessage());
} finally { // } finally {
if(response != null) { // if(response != null) {
// release connection // // release connection
response.closeStream(); // response.closeStream();
} // }
} }
} }

46
source/de/anomic/yacy/yacyRelease.java
Unescape View File

@ -27,7 +27,7 @@
package de.anomic.yacy; package de.anomic.yacy;
import java.io.BufferedInputStream; //import java.io.BufferedInputStream;
import java.io.BufferedOutputStream; import java.io.BufferedOutputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
@ -46,6 +46,7 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.Client;
import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.ContentScraper;
import net.yacy.kelondro.io.CharBuffer; import net.yacy.kelondro.io.CharBuffer;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -55,10 +56,11 @@ import net.yacy.kelondro.util.OS;
import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.retrieval.HTTPLoader; import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.http.client.Client; //import de.anomic.http.client.Client;
import de.anomic.http.server.HeaderFramework; import de.anomic.http.server.HeaderFramework;
import de.anomic.http.server.RequestHeader; import de.anomic.http.server.RequestHeader;
import de.anomic.http.server.ResponseContainer; //import de.anomic.http.server.ResponseContainer;
import de.anomic.http.server.ResponseHeader;
import de.anomic.search.Switchboard; import de.anomic.search.Switchboard;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.tools.CryptoLib; import de.anomic.tools.CryptoLib;
@ -285,14 +287,19 @@ public final class yacyRelease extends yacyVersion {
final RequestHeader reqHeader = new RequestHeader(); final RequestHeader reqHeader = new RequestHeader();
reqHeader.put(HeaderFramework.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(HeaderFramework.USER_AGENT, HTTPLoader.yacyUserAgent);
ResponseContainer res = null; // ResponseContainer res = null;
final String name = this.getUrl().getFileName(); final String name = this.getUrl().getFileName();
byte[] signatureBytes = null; byte[] signatureBytes = null;
final Client client = new Client();
client.setTimout(6000);
client.setHeader(reqHeader.entrySet());
// download signature first, if public key is available // download signature first, if public key is available
try { try {
if (this.publicKey != null) { if (this.publicKey != null) {
final byte[] signatureData = Client.wget(this.getUrl().toString() + ".sig", reqHeader, 6000); // final byte[] signatureData = Client.wget(this.getUrl().toString() + ".sig", reqHeader, 6000);
final byte[] signatureData = client.GETbytes(this.getUrl().toString());
if (signatureData == null) { if (signatureData == null) {
Log.logWarning("yacyVersion", "download of signature " + this.getUrl().toString() + " failed. ignoring signature file."); Log.logWarning("yacyVersion", "download of signature " + this.getUrl().toString() + " failed. ignoring signature file.");
} else try { } else try {
@ -302,10 +309,14 @@ public final class yacyRelease extends yacyVersion {
} }
// in case that the download of a signature file failed (can be caused by bad working http servers), then it is assumed that no signature exists // in case that the download of a signature file failed (can be caused by bad working http servers), then it is assumed that no signature exists
} }
final Client client = new Client(120000, reqHeader); // final Client client = new Client(120000, reqHeader);
res = client.GET(this.getUrl().toString()); // res = client.GET(this.getUrl().toString());
client.setTimout(120000);
client.GET(this.getUrl().toString());
final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file // final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
final boolean unzipped = header.gzip() && (header.mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
if ((unzipped) && (name.endsWith(".tar.gz"))) { if ((unzipped) && (name.endsWith(".tar.gz"))) {
download = new File(storagePath, name.substring(0, name.length() - 3)); download = new File(storagePath, name.substring(0, name.length() - 3));
} else { } else {
@ -316,7 +327,8 @@ public final class yacyRelease extends yacyVersion {
SignatureOutputStream verifyOutput = null; SignatureOutputStream verifyOutput = null;
try { try {
verifyOutput = new SignatureOutputStream(new FileOutputStream(download), CryptoLib.signAlgorithm, publicKey); verifyOutput = new SignatureOutputStream(new FileOutputStream(download), CryptoLib.signAlgorithm, publicKey);
FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(verifyOutput)); // FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(verifyOutput));
client.writeTo(new BufferedOutputStream(verifyOutput));
if (!verifyOutput.verify(signatureBytes)) throw new IOException("Bad Signature!"); if (!verifyOutput.verify(signatureBytes)) throw new IOException("Bad Signature!");
} catch (NoSuchAlgorithmException e) { } catch (NoSuchAlgorithmException e) {
@ -333,12 +345,13 @@ public final class yacyRelease extends yacyVersion {
if ((!signatureFile.exists()) || (signatureFile.length() == 0)) throw new IOException("create signature file failed"); if ((!signatureFile.exists()) || (signatureFile.length() == 0)) throw new IOException("create signature file failed");
} else { } else {
// just copy into file // just copy into file
FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download))); // FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download)));
client.writeTo(new BufferedOutputStream(new FileOutputStream(download)));
} }
if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + this.getUrl() + " failed"); if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + this.getUrl() + " failed");
} catch (final IOException e) { } catch (final IOException e) {
// Saving file failed, abort download // Saving file failed, abort download
if (res != null) res.abort(); // if (res != null) res.abort();
Log.logSevere("yacyVersion", "download of " + this.getName() + " failed: " + e.getMessage()); Log.logSevere("yacyVersion", "download of " + this.getName() + " failed: " + e.getMessage());
if (download != null && download.exists()) { if (download != null && download.exists()) {
FileUtils.deletedelete(download); FileUtils.deletedelete(download);
@ -346,9 +359,14 @@ public final class yacyRelease extends yacyVersion {
} }
download = null; download = null;
} finally { } finally {
if (res != null) { // if (res != null) {
// release connection // // release connection
res.closeStream(); // res.closeStream();
// }
try {
client.finish();
} catch (IOException e) {
Log.logSevere("yacyVersion", "finish of " + this.getName() + " failed: " + e.getMessage());
} }
} }
this.releaseFile = download; this.releaseFile = download;

134
source/net/yacy/cora/protocol/Client.java
Unescape View File

@ -1,6 +1,8 @@
package net.yacy.cora.protocol; package net.yacy.cora.protocol;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@ -13,11 +15,13 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion; import org.apache.http.HttpVersion;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams; import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean; import org.apache.http.conn.params.ConnPerRouteBean;
@ -27,6 +31,7 @@ import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme; import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry; import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory; import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody; import org.apache.http.entity.mime.content.StringBody;
@ -57,10 +62,12 @@ public class Client {
private static HttpClient httpClient = null; private static HttpClient httpClient = null;
private Header[] headers = null; private Header[] headers = null;
private HttpResponse httpResponse = null; private HttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null;
private long upbytes = 0L; private long upbytes = 0L;
private int timeout = 10000; private int timeout = 10000;
private String userAgent = null; private String userAgent = null;
private String host = null; private String host = null;
private boolean redirecting = true;
public Client() { public Client() {
super(); super();
@ -181,6 +188,14 @@ public class Client {
this.host = host; this.host = host;
} }
/**
*
* @param redirecting
*/
public void setRedirecting(final boolean redirecting) {
this.redirecting = redirecting;
}
/** /**
* This method GETs a page from the server. * This method GETs a page from the server.
* *
@ -205,6 +220,21 @@ public class Client {
return getContentBytes(httpGet, maxBytes); return getContentBytes(httpGet, maxBytes);
} }
/**
* This method GETs a page from the server.
* to be used for streaming out
* Please take care to call finish()!
*
* @param uri the url to get
* @throws IOException
*/
public void GET(final String uri) throws IOException {
if (currentRequest != null) throw new IOException("Client is in use!");
final HttpGet httpGet = new HttpGet(uri);
currentRequest = httpGet;
execute(httpGet);
}
/** /**
* This method gets HEAD response * This method gets HEAD response
* *
@ -214,18 +244,42 @@ public class Client {
*/ */
public HttpResponse HEADResponse(final String uri) throws IOException { public HttpResponse HEADResponse(final String uri) throws IOException {
final HttpHead httpHead = new HttpHead(uri); final HttpHead httpHead = new HttpHead(uri);
getContentBytes(httpHead, Long.MAX_VALUE); execute(httpHead);
finish();
ConnectionInfo.removeConnection(httpHead.hashCode());
return httpResponse; return httpResponse;
} }
/** /**
* This method POSTs a page from the server.
* to be used for streaming out
* Please take care to call finish()!
*
* @param uri the url to post
* @param instream the input to post
* @param length the contentlength
* @throws IOException
*/
public void POST(final String uri, final InputStream instream, long length) throws IOException {
if (currentRequest != null) throw new IOException("Client is in use!");
final HttpPost httpPost = new HttpPost(uri);
final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length);
// statistics
upbytes = length;
httpPost.setEntity(inputStreamEntity);
currentRequest = httpPost;
execute(httpPost);
}
/**
* This method POSTs a page from the server.
* *
* @param uri the url to post * @param uri the url to post
* @param parts to post * @param parts to post
* @return content bytes * @return content bytes
* @throws IOException * @throws IOException
*/ */
public byte[] POSTbytes(final String uri, LinkedHashMap<String,ContentBody> parts) throws IOException { public byte[] POSTbytes(final String uri, final LinkedHashMap<String,ContentBody> parts) throws IOException {
final HttpPost httpPost = new HttpPost(uri); final HttpPost httpPost = new HttpPost(uri);
final MultipartEntity multipartEntity = new MultipartEntity(); final MultipartEntity multipartEntity = new MultipartEntity();
@ -256,17 +310,47 @@ public class Client {
return hmap; return hmap;
} }
private byte[] getContentBytes(HttpUriRequest httpUriRequest, long maxBytes) throws IOException { public void writeTo(final OutputStream outputStream) throws IOException {
if (httpResponse != null && currentRequest != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) try {
httpEntity.writeTo(outputStream);
outputStream.flush();
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
} catch (final IOException e) {
currentRequest.abort();
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
throw e;
}
}
}
public void finish() throws IOException {
if (httpResponse != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null && httpEntity.isStreaming()) {
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
}
}
if (currentRequest != null) {
currentRequest.abort();
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
}
}
private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final long maxBytes) throws IOException {
byte[] content = null; byte[] content = null;
final HttpContext httpContext = new BasicHttpContext();
setHeaders(httpUriRequest);
setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams());
// statistics
storeConnectionInfo(httpUriRequest);
try { try {
// execute the method execute(httpUriRequest);
httpResponse = httpClient.execute(httpUriRequest, httpContext); if (httpResponse == null) return null;
// get the response body // get the response body
final HttpEntity httpEntity = httpResponse.getEntity(); final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) { if (httpEntity != null) {
@ -286,7 +370,24 @@ public class Client {
return content; return content;
} }
private void setHeaders(HttpUriRequest httpUriRequest) { private void execute(final HttpUriRequest httpUriRequest) throws IOException {
final HttpContext httpContext = new BasicHttpContext();
setHeaders(httpUriRequest);
setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams());
// statistics
storeConnectionInfo(httpUriRequest);
try {
// execute the method
httpResponse = httpClient.execute(httpUriRequest, httpContext);
} catch (ClientProtocolException e) {
httpUriRequest.abort();
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
throw new IOException("Client can't execute: " + e.getMessage());
}
}
private void setHeaders(final HttpUriRequest httpUriRequest) {
if (headers != null) { if (headers != null) {
for (Header header : headers) { for (Header header : headers) {
httpUriRequest.addHeader(header); httpUriRequest.addHeader(header);
@ -294,7 +395,8 @@ public class Client {
} }
} }
private void setParams(HttpParams httpParams) { private void setParams(final HttpParams httpParams) {
HttpClientParams.setRedirecting(httpParams, redirecting);
HttpConnectionParams.setConnectionTimeout(httpParams, timeout); HttpConnectionParams.setConnectionTimeout(httpParams, timeout);
HttpConnectionParams.setSoTimeout(httpParams, timeout); HttpConnectionParams.setSoTimeout(httpParams, timeout);
if (userAgent != null) if (userAgent != null)
@ -303,14 +405,14 @@ public class Client {
httpParams.setParameter(HTTP.TARGET_HOST, host); httpParams.setParameter(HTTP.TARGET_HOST, host);
} }
private void setProxy(HttpParams httpParams) { private void setProxy(final HttpParams httpParams) {
if (ProxySettings.use) if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost()); ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this // TODO find a better way for this
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient); ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
} }
private void storeConnectionInfo(HttpUriRequest httpUriRequest) { private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
final int port = httpUriRequest.getURI().getPort(); final int port = httpUriRequest.getURI().getPort();
final String thost = httpUriRequest.getURI().getHost(); final String thost = httpUriRequest.getURI().getHost();
ConnectionInfo.addConnection(new ConnectionInfo( ConnectionInfo.addConnection(new ConnectionInfo(
@ -368,7 +470,7 @@ public class Client {
} }
Client client = new Client(); Client client = new Client();
client.setUserAgent("foobar"); client.setUserAgent("foobar");
client.setHost("sixcooler"); client.setRedirecting(false);
// Get some // Get some
for (int i = 0; i < args.length; i++) { for (int i = 0; i < args.length; i++) {
url = args[i]; url = args[i];

19
source/net/yacy/cora/protocol/GzipDecompressingEntity.java
Unescape View File

@ -2,6 +2,7 @@ package net.yacy.cora.protocol;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
@ -9,6 +10,8 @@ import org.apache.http.entity.HttpEntityWrapper;
public class GzipDecompressingEntity extends HttpEntityWrapper { public class GzipDecompressingEntity extends HttpEntityWrapper {
private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size
public GzipDecompressingEntity(final HttpEntity entity) { public GzipDecompressingEntity(final HttpEntity entity) {
super(entity); super(entity);
} }
@ -21,6 +24,22 @@ public class GzipDecompressingEntity extends HttpEntityWrapper {
return new GZIPInputStream(wrappedin); return new GZIPInputStream(wrappedin);
} }
public void writeTo(OutputStream outstream) throws IOException {
if (outstream == null) {
throw new IllegalArgumentException("Output stream may not be null");
}
InputStream instream = this.getContent();
int l;
byte[] tmp = new byte[DEFAULT_BUFFER_SIZE];
while ((l = instream.read(tmp)) != -1) {
outstream.write(tmp, 0, l);
}
}
public boolean isChunked() {
return true;
}
public long getContentLength() { public long getContentLength() {
// length of ungzipped content not known in advance // length of ungzipped content not known in advance
return -1; return -1;

Write Preview
Loading…
Cancel
Save