boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

... migrating to HttpComponents-Client-4.x ...

Browse Source 
- Proxy
- Release-download

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7011 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 15 years ago
parent ced07970c1
commit c29f24a519
4 changed files with 367 additions and 156 deletions
Show Stats Download Patch File Download Diff File

322
source/de/anomic/http/server/HTTPDProxyHandler.java
Unescape View File

@ -43,7 +43,7 @@
package de.anomic.http.server;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
//import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
@ -71,6 +71,7 @@ import java.util.logging.LogManager;
import java.util.logging.Logger;
import java.util.zip.GZIPOutputStream;
import net.yacy.cora.protocol.Client;
import net.yacy.cora.protocol.ProxySettings;
import net.yacy.document.TextParser;
import net.yacy.document.parser.html.ContentTransformer;
@ -87,7 +88,7 @@ import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.crawler.retrieval.Request;
import de.anomic.crawler.retrieval.Response;
import de.anomic.http.client.MultiOutputStream;
import de.anomic.http.client.Client;
//import de.anomic.http.client.Client;
import de.anomic.http.client.Cache;
import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
@ -439,7 +440,7 @@ public final class HTTPDProxyHandler {
final GZIPOutputStream gzippedOut = null;
ResponseContainer res = null;
// ResponseContainer res = null;
try {
final int reqID = requestHeader.hashCode();
@ -478,17 +479,22 @@ public final class HTTPDProxyHandler {
// send request
try {
res = client.GET(getUrl);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
// res = client.GET(getUrl);
// if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
client.GET(getUrl);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader);
final ResponseHeader responseHeader = res.getResponseHeader();
// final ResponseHeader responseHeader = res.getResponseHeader();
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
// determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) {
throw new Exception(res.getStatusLine());
// throw new Exception(res.getStatusLine());
throw new Exception(client.getHttpResponse().getStatusLine().toString());
}
final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond);
// final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond);
final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), respond);
// the cache does either not exist or is (supposed to be) stale
long sizeBeforeDelete = -1;
@ -519,7 +525,8 @@ public final class HTTPDProxyHandler {
// handle incoming cookies
handleIncomingCookies(responseHeader, host, ip);
prepareResponseHeader(responseHeader, res.getHttpVer());
// prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
// sending the respond header back to the client
if (chunkedOut != null) {
@ -527,22 +534,31 @@ public final class HTTPDProxyHandler {
}
if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader);
// HTTPDemon.sendRespondHeader(
// conProp,
// respond,
// httpVer,
// res.getStatusCode(),
// res.getStatusLine().substring(4), // status text
// responseHeader);
HTTPDemon.sendRespondHeader(
conProp,
respond,
httpVer,
res.getStatusCode(),
res.getStatusLine().substring(4), // status text
client.getHttpResponse().getStatusLine().getStatusCode(),
client.getHttpResponse().getStatusLine().toString(), // status text
responseHeader);
if (hasBody(res.getStatusCode())) {
// if (hasBody(res.getStatusCode())) {
if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) {
final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
final Response response = new Response(
request,
requestHeader,
responseHeader,
res.getStatusLine(),
// res.getStatusLine(),
Integer.toString(client.getHttpResponse().getStatusLine().getStatusCode()),
sb.crawler.defaultProxyProfile
);
final String storeError = response.shallStoreCacheForProxy();
@ -561,11 +577,13 @@ public final class HTTPDProxyHandler {
((storeHTCache) || (supportError != null))
) {
// we don't write actually into a file, only to RAM, and schedule writing the file.
int l = res.getResponseHeader().size();
// int l = res.getResponseHeader().size();
int l = responseHeader.size();
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);
final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});
FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
// FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
client.writeTo(toClientAndMemory);
// cached bytes
byte[] cacheArray;
if (byteStream.size() > 0) {
@ -608,7 +626,8 @@ public final class HTTPDProxyHandler {
" StoreHTCache=" + storeHTCache +
" SupportError=" + supportError);
FileUtils.copy(res.getDataAsStream(), outStream);
// FileUtils.copy(res.getDataAsStream(), outStream);
client.writeTo(outStream);
conProp.setProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
}
@ -623,17 +642,19 @@ public final class HTTPDProxyHandler {
} // end hasBody
} catch(SocketException se) {
// if opened ...
if(res != null) {
// client cut proxy connection, abort download
res.abort();
}
// if(res != null) {
// // client cut proxy connection, abort download
// res.abort();
// }
client.finish();
handleProxyException(se,conProp,respond,url);
} finally {
// if opened ...
if(res != null) {
// ... close connection
res.closeStream();
}
// if(res != null) {
// // ... close connection
// res.closeStream();
// }
client.finish();
}
} catch (final Exception e) {
handleProxyException(e,conProp,respond,url);
@ -741,7 +762,7 @@ public final class HTTPDProxyHandler {
public static void doHead(final Properties conProp, final RequestHeader requestHeader, OutputStream respond) {
ResponseContainer res = null;
// ResponseContainer res = null;
DigestURI url = null;
try {
final int reqID = requestHeader.hashCode();
@ -813,28 +834,42 @@ public final class HTTPDProxyHandler {
final Client client = setupHttpClient(requestHeader, connectHost);
// send request
try {
res = client.HEAD(getUrl);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
// try {
// res = client.HEAD(getUrl);
// if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
client.HEADResponse(getUrl);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
// determine if it's an internal error of the httpc
final ResponseHeader responseHeader = res.getResponseHeader();
// final ResponseHeader responseHeader = res.getResponseHeader();
// if (responseHeader.isEmpty()) {
// throw new Exception(res.getStatusLine());
// }
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
if (responseHeader.isEmpty()) {
throw new Exception(res.getStatusLine());
}
throw new Exception(client.getHttpResponse().getStatusLine().toString());
}
prepareResponseHeader(responseHeader, res.getHttpVer());
// prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getStatusLine().getProtocolVersion().toString());
// sending the server respond back to the client
if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader);
HTTPDemon.sendRespondHeader(conProp,respond,httpVer,res.getStatusCode(),res.getStatusLine().substring(4),responseHeader);
// HTTPDemon.sendRespondHeader(conProp,respond,httpVer,res.getStatusCode(),res.getStatusLine().substring(4),responseHeader);
HTTPDemon.sendRespondHeader(
conProp,
respond,
httpVer,
client.getHttpResponse().getStatusLine().getStatusCode(),
client.getHttpResponse().getStatusLine().toString(),
responseHeader);
respond.flush();
} finally {
if(res != null) {
// ... close connection
res.closeStream();
}
}
// } finally {
// if(res != null) {
// // ... close connection
// res.closeStream();
// }
// }
} catch (final Exception e) {
handleProxyException(e,conProp,respond,url);
}
@ -901,6 +936,10 @@ public final class HTTPDProxyHandler {
final String getUrl = "http://"+ connectHost + remotePath;
if (log.isFinest()) log.logFinest(reqID +" using url: "+ getUrl);
// the CONTENT_LENGTH will be added by entity and cause a ClientProtocolException if set
final int contentLength = requestHeader.getContentLength();
requestHeader.remove(RequestHeader.CONTENT_LENGTH);
final Client client = setupHttpClient(requestHeader, connectHost);
// check input
@ -911,79 +950,95 @@ public final class HTTPDProxyHandler {
// "if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader"
// it seems that it is a HTTP/1.1 connection which stays open (the inputStream) and endlessly waits for
// input so we have to end it to do the request
final int contentLength = requestHeader.getContentLength();
if (contentLength > -1) {
final byte[] bodyData;
if(contentLength == 0) {
// no body
bodyData = new byte[0];
} else {
// read content-length bytes into memory
bodyData = new byte[contentLength];
int bytes_read = 0;
while(bytes_read < contentLength) {
bytes_read += body.read(bodyData, bytes_read, contentLength-bytes_read);
}
}
body = new ByteArrayInputStream(bodyData);
}
ResponseContainer res = null;
try {
// sending the request
res = client.POST(getUrl, body);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
final ResponseHeader responseHeader = res.getResponseHeader();
// determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) {
throw new Exception(res.getStatusLine());
}
final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), countedRespond);
prepareResponseHeader(responseHeader, res.getHttpVer());
// sending the respond header back to the client
if (chunked != null) {
responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked");
}
// sending response headers
if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader);
HTTPDemon.sendRespondHeader(conProp,
countedRespond,
httpVer,
res.getStatusCode(),
res.getStatusLine().substring(4), // status text
responseHeader);
// respondHeader(respond, res.status, res.responseHeader);
// Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond));
/*
// *** (Uebernommen aus Saver-Klasse: warum ist dies hier die einzige Methode, die einen OutputStream statt einen Writer benutzt?)
// this should not be needed anymore - see org.apache.http.entity.InputStreamEntity
// final int contentLength = requestHeader.getContentLength();
// if (contentLength > -1) {
// final byte[] bodyData;
// if(contentLength == 0) {
// // no body
// bodyData = new byte[0];
// } else {
// // read content-length bytes into memory
// bodyData = new byte[contentLength];
// int bytes_read = 0;
// while(bytes_read < contentLength) {
// bytes_read += body.read(bodyData, bytes_read, contentLength-bytes_read);
// }
// }
// body = new ByteArrayInputStream(bodyData);
// }
// ResponseContainer res = null;
try {
serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond));
} finally {
res.closeStream();
}
if (chunked != null) chunked.finish();
*/
final OutputStream outStream = (chunked != null) ? chunked : countedRespond;
FileUtils.copy(res.getDataAsStream(), outStream);
if (chunked != null) {
chunked.finish();
}
outStream.flush();
// sending the request
// res = client.POST(getUrl, body);
// if (log.isFinest()) log.logFinest(reqID +" response status: "+ res.getStatusLine());
client.POST(getUrl, body, contentLength);
if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
// final ResponseHeader responseHeader = res.getResponseHeader();
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
// determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) {
// throw new Exception(res.getStatusLine());
throw new Exception(client.getHttpResponse().getStatusLine().toString());
}
// final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), countedRespond);
final ChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), countedRespond);
// prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
// sending the respond header back to the client
if (chunked != null) {
responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked");
}
// sending response headers
if (log.isFinest()) log.logFinest(reqID +" sending response header: "+ responseHeader);
// HTTPDemon.sendRespondHeader(conProp,
// countedRespond,
// httpVer,
// res.getStatusCode(),
// res.getStatusLine().substring(4), // status text
// responseHeader);
HTTPDemon.sendRespondHeader(conProp,
countedRespond,
httpVer,
client.getHttpResponse().getStatusLine().getStatusCode(),
client.getHttpResponse().getStatusLine().toString(), // status text
responseHeader);
// respondHeader(respond, res.status, res.responseHeader);
// Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond));
/*
// *** (Uebernommen aus Saver-Klasse: warum ist dies hier die einzige Methode, die einen OutputStream statt einen Writer benutzt?)
try {
serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond));
} finally {
res.closeStream();
}
if (chunked != null) chunked.finish();
*/
final OutputStream outStream = (chunked != null) ? chunked : countedRespond;
// FileUtils.copy(res.getDataAsStream(), outStream);
client.writeTo(outStream);
if (chunked != null) {
chunked.finish();
}
outStream.flush();
} catch(SocketException se) {
// connection closed by client, abort download
res.abort();
// res.abort();
client.finish();
} finally {
// if opened ...
if(res != null) {
// ... close connection
res.closeStream();
}
// if(res != null) {
// // ... close connection
// res.closeStream();
// }
client.finish();
}
} catch (final Exception e) {
handleProxyException(e,conProp,countedRespond,url);
@ -1069,8 +1124,12 @@ public final class HTTPDProxyHandler {
*/
private static Client setupHttpClient(final RequestHeader requestHeader, final String connectHost) {
// setup HTTP-client
final Client client = new Client(timeout, requestHeader);
client.setFollowRedirects(false);
// final Client client = new Client(timeout, requestHeader);
// client.setFollowRedirects(false);
final Client client = new Client();
client.setTimout(timeout);
client.setHeader(requestHeader.entrySet());
client.setRedirecting(false);
return client;
}
@ -1240,16 +1299,22 @@ public final class HTTPDProxyHandler {
// possibly branch into PROXY-PROXY connection
if (ProxySettings.use && ProxySettings.use4ssl) {
final Client remoteProxy = new Client(timeout, requestHeader);
remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe
// final Client remoteProxy = new Client(timeout, requestHeader);
// remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe
final Client remoteProxy = setupHttpClient(requestHeader, host);
ResponseContainer response = null;
// ResponseContainer response = null;
try {
response = remoteProxy.CONNECT(host, port);
// response = remoteProxy.CONNECT(host, port);
remoteProxy.HEADResponse("http://" + host + ":" + port);
ResponseHeader header = new ResponseHeader(null, remoteProxy.getHeaderHashMap());
// outputs a logline to the serverlog with the current status
log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString());
// (response.getStatusLine().charAt(0) == '2') || (response.getStatusLine().charAt(0) == '3')
final boolean success = response.getStatusCode() >= 200 && response.getStatusCode() <= 399;
// log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString());
// // (response.getStatusLine().charAt(0) == '2') || (response.getStatusLine().charAt(0) == '3')
// final boolean success = response.getStatusCode() >= 200 && response.getStatusCode() <= 399;
log.logInfo("CONNECT-RESPONSE: status=" + remoteProxy.getHttpResponse().getStatusLine() + ", header=" + header.toString());
final boolean success = remoteProxy.getHttpResponse().getStatusLine().getStatusCode() >= 200 && remoteProxy.getHttpResponse().getStatusLine().getStatusCode() <= 399;
if (success) {
// replace connection details
host = ProxySettings.host;
@ -1257,21 +1322,28 @@ public final class HTTPDProxyHandler {
// go on (see below)
} else {
// pass error response back to client
HTTPDemon.sendRespondHeader(conProp,clientOut,httpVersion,response.getStatusCode(),response.getStatusLine().substring(4),response.getResponseHeader());
// HTTPDemon.sendRespondHeader(conProp,clientOut,httpVersion,response.getStatusCode(),response.getStatusLine().substring(4),response.getResponseHeader());
HTTPDemon.sendRespondHeader(
conProp,
clientOut,
httpVersion,
remoteProxy.getHttpResponse().getStatusLine().getStatusCode(),
remoteProxy.getHttpResponse().getStatusLine().toString(),
header);
//respondHeader(clientOut, response.status, response.responseHeader);
forceConnectionClose(conProp);
return;
}
} catch (SocketException se) {
// connection closed by client, abort download
response.abort();
// } catch (SocketException se) {
// // connection closed by client, abort download
// response.abort();
} catch (final Exception e) {
throw new IOException(e.getMessage());
} finally {
if(response != null) {
// release connection
response.closeStream();
}
// } finally {
// if(response != null) {
// // release connection
// response.closeStream();
// }
}
}

48
source/de/anomic/yacy/yacyRelease.java
Unescape View File

@ -27,7 +27,7 @@
package de.anomic.yacy;
import java.io.BufferedInputStream;
//import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
@ -46,6 +46,7 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.Client;
import net.yacy.document.parser.html.ContentScraper;
import net.yacy.kelondro.io.CharBuffer;
import net.yacy.kelondro.logging.Log;
@ -55,10 +56,11 @@ import net.yacy.kelondro.util.OS;
import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.http.client.Client;
//import de.anomic.http.client.Client;
import de.anomic.http.server.HeaderFramework;
import de.anomic.http.server.RequestHeader;
import de.anomic.http.server.ResponseContainer;
//import de.anomic.http.server.ResponseContainer;
import de.anomic.http.server.ResponseHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverCore;
import de.anomic.tools.CryptoLib;
@ -285,14 +287,19 @@ public final class yacyRelease extends yacyVersion {
final RequestHeader reqHeader = new RequestHeader();
reqHeader.put(HeaderFramework.USER_AGENT, HTTPLoader.yacyUserAgent);
ResponseContainer res = null;
// ResponseContainer res = null;
final String name = this.getUrl().getFileName();
byte[] signatureBytes = null;
final Client client = new Client();
client.setTimout(6000);
client.setHeader(reqHeader.entrySet());
// download signature first, if public key is available
try {
if (this.publicKey != null) {
final byte[] signatureData = Client.wget(this.getUrl().toString() + ".sig", reqHeader, 6000);
// final byte[] signatureData = Client.wget(this.getUrl().toString() + ".sig", reqHeader, 6000);
final byte[] signatureData = client.GETbytes(this.getUrl().toString());
if (signatureData == null) {
Log.logWarning("yacyVersion", "download of signature " + this.getUrl().toString() + " failed. ignoring signature file.");
} else try {
@ -302,10 +309,14 @@ public final class yacyRelease extends yacyVersion {
}
// in case that the download of a signature file failed (can be caused by bad working http servers), then it is assumed that no signature exists
}
final Client client = new Client(120000, reqHeader);
res = client.GET(this.getUrl().toString());
// final Client client = new Client(120000, reqHeader);
// res = client.GET(this.getUrl().toString());
client.setTimout(120000);
client.GET(this.getUrl().toString());
final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
// final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
final boolean unzipped = header.gzip() && (header.mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
if ((unzipped) && (name.endsWith(".tar.gz"))) {
download = new File(storagePath, name.substring(0, name.length() - 3));
} else {
@ -316,7 +327,8 @@ public final class yacyRelease extends yacyVersion {
SignatureOutputStream verifyOutput = null;
try {
verifyOutput = new SignatureOutputStream(new FileOutputStream(download), CryptoLib.signAlgorithm, publicKey);
FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(verifyOutput));
// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(verifyOutput));
client.writeTo(new BufferedOutputStream(verifyOutput));
if (!verifyOutput.verify(signatureBytes)) throw new IOException("Bad Signature!");
} catch (NoSuchAlgorithmException e) {
@ -333,12 +345,13 @@ public final class yacyRelease extends yacyVersion {
if ((!signatureFile.exists()) || (signatureFile.length() == 0)) throw new IOException("create signature file failed");
} else {
// just copy into file
FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download)));
// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download)));
client.writeTo(new BufferedOutputStream(new FileOutputStream(download)));
}
if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + this.getUrl() + " failed");
} catch (final IOException e) {
// Saving file failed, abort download
if (res != null) res.abort();
// if (res != null) res.abort();
Log.logSevere("yacyVersion", "download of " + this.getName() + " failed: " + e.getMessage());
if (download != null && download.exists()) {
FileUtils.deletedelete(download);
@ -346,10 +359,15 @@ public final class yacyRelease extends yacyVersion {
}
download = null;
} finally {
if (res != null) {
// release connection
res.closeStream();
}
// if (res != null) {
// // release connection
// res.closeStream();
// }
try {
client.finish();
} catch (IOException e) {
Log.logSevere("yacyVersion", "finish of " + this.getName() + " failed: " + e.getMessage());
}
}
this.releaseFile = download;
Switchboard.getSwitchboard().setConfig("update.time.download", System.currentTimeMillis());

134
source/net/yacy/cora/protocol/Client.java
Unescape View File

@ -1,6 +1,8 @@
package net.yacy.cora.protocol;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.LinkedHashMap;
@ -13,11 +15,13 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
@ -27,6 +31,7 @@ import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
@ -57,10 +62,12 @@ public class Client {
private static HttpClient httpClient = null;
private Header[] headers = null;
private HttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null;
private long upbytes = 0L;
private int timeout = 10000;
private String userAgent = null;
private String host = null;
private boolean redirecting = true;
public Client() {
super();
@ -181,6 +188,14 @@ public class Client {
this.host = host;
}
/**
*
* @param redirecting
*/
public void setRedirecting(final boolean redirecting) {
this.redirecting = redirecting;
}
/**
* This method GETs a page from the server.
*
@ -205,6 +220,21 @@ public class Client {
return getContentBytes(httpGet, maxBytes);
}
/**
* This method GETs a page from the server.
* to be used for streaming out
* Please take care to call finish()!
*
* @param uri the url to get
* @throws IOException
*/
public void GET(final String uri) throws IOException {
if (currentRequest != null) throw new IOException("Client is in use!");
final HttpGet httpGet = new HttpGet(uri);
currentRequest = httpGet;
execute(httpGet);
}
/**
* This method gets HEAD response
*
@ -214,18 +244,42 @@ public class Client {
*/
public HttpResponse HEADResponse(final String uri) throws IOException {
final HttpHead httpHead = new HttpHead(uri);
getContentBytes(httpHead, Long.MAX_VALUE);
execute(httpHead);
finish();
ConnectionInfo.removeConnection(httpHead.hashCode());
return httpResponse;
}
/**
* This method POSTs a page from the server.
* to be used for streaming out
* Please take care to call finish()!
*
* @param uri the url to post
* @param instream the input to post
* @param length the contentlength
* @throws IOException
*/
public void POST(final String uri, final InputStream instream, long length) throws IOException {
if (currentRequest != null) throw new IOException("Client is in use!");
final HttpPost httpPost = new HttpPost(uri);
final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length);
// statistics
upbytes = length;
httpPost.setEntity(inputStreamEntity);
currentRequest = httpPost;
execute(httpPost);
}
/**
* This method POSTs a page from the server.
*
* @param uri the url to post
* @param parts to post
* @return content bytes
* @throws IOException
*/
public byte[] POSTbytes(final String uri, LinkedHashMap<String,ContentBody> parts) throws IOException {
public byte[] POSTbytes(final String uri, final LinkedHashMap<String,ContentBody> parts) throws IOException {
final HttpPost httpPost = new HttpPost(uri);
final MultipartEntity multipartEntity = new MultipartEntity();
@ -255,18 +309,48 @@ public class Client {
}
return hmap;
}
public void writeTo(final OutputStream outputStream) throws IOException {
if (httpResponse != null && currentRequest != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) try {
httpEntity.writeTo(outputStream);
outputStream.flush();
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
} catch (final IOException e) {
currentRequest.abort();
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
throw e;
}
}
}
public void finish() throws IOException {
if (httpResponse != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null && httpEntity.isStreaming()) {
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
}
}
if (currentRequest != null) {
currentRequest.abort();
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
}
}
private byte[] getContentBytes(HttpUriRequest httpUriRequest, long maxBytes) throws IOException {
private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final long maxBytes) throws IOException {
byte[] content = null;
final HttpContext httpContext = new BasicHttpContext();
setHeaders(httpUriRequest);
setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams());
// statistics
storeConnectionInfo(httpUriRequest);
try {
// execute the method
httpResponse = httpClient.execute(httpUriRequest, httpContext);
execute(httpUriRequest);
if (httpResponse == null) return null;
// get the response body
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) {
@ -286,7 +370,24 @@ public class Client {
return content;
}
private void setHeaders(HttpUriRequest httpUriRequest) {
private void execute(final HttpUriRequest httpUriRequest) throws IOException {
final HttpContext httpContext = new BasicHttpContext();
setHeaders(httpUriRequest);
setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams());
// statistics
storeConnectionInfo(httpUriRequest);
try {
// execute the method
httpResponse = httpClient.execute(httpUriRequest, httpContext);
} catch (ClientProtocolException e) {
httpUriRequest.abort();
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
throw new IOException("Client can't execute: " + e.getMessage());
}
}
private void setHeaders(final HttpUriRequest httpUriRequest) {
if (headers != null) {
for (Header header : headers) {
httpUriRequest.addHeader(header);
@ -294,7 +395,8 @@ public class Client {
}
}
private void setParams(HttpParams httpParams) {
private void setParams(final HttpParams httpParams) {
HttpClientParams.setRedirecting(httpParams, redirecting);
HttpConnectionParams.setConnectionTimeout(httpParams, timeout);
HttpConnectionParams.setSoTimeout(httpParams, timeout);
if (userAgent != null)
@ -303,14 +405,14 @@ public class Client {
httpParams.setParameter(HTTP.TARGET_HOST, host);
}
private void setProxy(HttpParams httpParams) {
private void setProxy(final HttpParams httpParams) {
if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
}
private void storeConnectionInfo(HttpUriRequest httpUriRequest) {
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
final int port = httpUriRequest.getURI().getPort();
final String thost = httpUriRequest.getURI().getHost();
ConnectionInfo.addConnection(new ConnectionInfo(
@ -368,7 +470,7 @@ public class Client {
}
Client client = new Client();
client.setUserAgent("foobar");
client.setHost("sixcooler");
client.setRedirecting(false);
// Get some
for (int i = 0; i < args.length; i++) {
url = args[i];

19
source/net/yacy/cora/protocol/GzipDecompressingEntity.java
Unescape View File

@ -2,12 +2,15 @@ package net.yacy.cora.protocol;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.GZIPInputStream;
import org.apache.http.HttpEntity;
import org.apache.http.entity.HttpEntityWrapper;
public class GzipDecompressingEntity extends HttpEntityWrapper {
private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size
public GzipDecompressingEntity(final HttpEntity entity) {
super(entity);
@ -20,6 +23,22 @@ public class GzipDecompressingEntity extends HttpEntityWrapper {
return new GZIPInputStream(wrappedin);
}
public void writeTo(OutputStream outstream) throws IOException {
if (outstream == null) {
throw new IllegalArgumentException("Output stream may not be null");
}
InputStream instream = this.getContent();
int l;
byte[] tmp = new byte[DEFAULT_BUFFER_SIZE];
while ((l = instream.read(tmp)) != -1) {
outstream.write(tmp, 0, l);
}
}
public boolean isChunked() {
return true;
}
public long getContentLength() {
// length of ungzipped content not known in advance

Write Preview
Loading…
Cancel
Save