boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Not storing external header in an Header-Array and reduce a loop for its conversion.

Browse Source 
Ensure connection close if a OOM is thrown.
Ensure setting resolved host is set at the request.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7542 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 14 years ago
parent 42d90664f3
commit e3d75d6cd5
1 changed files with 15 additions and 19 deletions
Show Stats Download Patch File Download Diff File

32
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -72,7 +72,6 @@ import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.AbstractHttpClient; import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.params.BasicHttpParams; import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams; import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams; import org.apache.http.params.HttpParams;
@ -94,7 +93,7 @@ public class HTTPClient {
private final static int maxcon = 200; private final static int maxcon = 200;
private static IdledConnectionEvictor idledConnectionEvictor = null; private static IdledConnectionEvictor idledConnectionEvictor = null;
private static HttpClient httpClient = initConnectionManager(); private static HttpClient httpClient = initConnectionManager();
private Header[] headers = null; private Set<Entry<String, String>> headers = null;
private HttpResponse httpResponse = null; private HttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null; private HttpUriRequest currentRequest = null;
private long upbytes = 0L; private long upbytes = 0L;
@ -196,13 +195,7 @@ public class HTTPClient {
* @param entrys to be set as request header * @param entrys to be set as request header
*/ */
public void setHeader(final Set<Entry<String, String>> entrys) { public void setHeader(final Set<Entry<String, String>> entrys) {
if (entrys != null) { this.headers = entrys;
int i = 0;
headers = new Header[entrys.size()];
for (final Entry<String, String> entry : entrys) {
headers[i++] = new BasicHeader(entry.getKey(),entry.getValue());
}
}
} }
/** /**
@ -272,7 +265,7 @@ public class HTTPClient {
public byte[] GETbytes(final String uri, long maxBytes) throws IOException { public byte[] GETbytes(final String uri, long maxBytes) throws IOException {
final MultiProtocolURI url = new MultiProtocolURI(uri); final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false)); final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false));
httpGet.setHeader("Host", url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
return getContentBytes(httpGet, maxBytes); return getContentBytes(httpGet, maxBytes);
} }
@ -288,7 +281,7 @@ public class HTTPClient {
if (currentRequest != null) throw new IOException("Client is in use!"); if (currentRequest != null) throw new IOException("Client is in use!");
final MultiProtocolURI url = new MultiProtocolURI(uri); final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false)); final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false));
httpGet.setHeader("Host", url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
currentRequest = httpGet; currentRequest = httpGet;
execute(httpGet); execute(httpGet);
} }
@ -303,7 +296,7 @@ public class HTTPClient {
public HttpResponse HEADResponse(final String uri) throws IOException { public HttpResponse HEADResponse(final String uri) throws IOException {
final MultiProtocolURI url = new MultiProtocolURI(uri); final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpHead httpHead = new HttpHead(url.toNormalform(true, false, true, false)); final HttpHead httpHead = new HttpHead(url.toNormalform(true, false, true, false));
httpHead.setHeader("Host", url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
execute(httpHead); execute(httpHead);
finish(); finish();
ConnectionInfo.removeConnection(httpHead.hashCode()); ConnectionInfo.removeConnection(httpHead.hashCode());
@ -324,7 +317,7 @@ public class HTTPClient {
if (currentRequest != null) throw new IOException("Client is in use!"); if (currentRequest != null) throw new IOException("Client is in use!");
final MultiProtocolURI url = new MultiProtocolURI(uri); final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false)); final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
httpPost.setHeader("Host", url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length); final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length);
// statistics // statistics
upbytes = length; upbytes = length;
@ -344,7 +337,7 @@ public class HTTPClient {
public byte[] POSTbytes(final String uri, final Map<String, ContentBody> parts, final boolean usegzip) throws IOException { public byte[] POSTbytes(final String uri, final Map<String, ContentBody> parts, final boolean usegzip) throws IOException {
final MultiProtocolURI url = new MultiProtocolURI(uri); final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false)); final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
httpPost.setHeader("Host", url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
final MultipartEntity multipartEntity = new MultipartEntity(); final MultipartEntity multipartEntity = new MultipartEntity();
for (final Entry<String,ContentBody> part : parts.entrySet()) for (final Entry<String,ContentBody> part : parts.entrySet())
@ -460,11 +453,12 @@ public class HTTPClient {
content = EntityUtils.toByteArray(httpEntity); content = EntityUtils.toByteArray(httpEntity);
} catch (OutOfMemoryError e) { } catch (OutOfMemoryError e) {
throw new IOException(e.toString()); throw new IOException(e.toString());
} } finally {
}
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed. // Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
EntityUtils.consume(httpEntity); EntityUtils.consume(httpEntity);
} }
}
}
} catch (final IOException e) { } catch (final IOException e) {
ConnectionInfo.removeConnection(httpUriRequest.hashCode()); ConnectionInfo.removeConnection(httpUriRequest.hashCode());
httpUriRequest.abort(); httpUriRequest.abort();
@ -503,10 +497,12 @@ public class HTTPClient {
private void setHeaders(final HttpUriRequest httpUriRequest) { private void setHeaders(final HttpUriRequest httpUriRequest) {
if (headers != null) { if (headers != null) {
for (final Header header : headers) { for (final Entry<String, String> entry : headers) {
httpUriRequest.setHeader(header); httpUriRequest.setHeader(entry.getKey(),entry.getValue());
} }
} }
if (host != null)
httpUriRequest.setHeader(HTTP.TARGET_HOST, host);
if (realm != null) if (realm != null)
httpUriRequest.setHeader("Authorization", "realm=" + realm); httpUriRequest.setHeader("Authorization", "realm=" + realm);
} }

Write Preview
Loading…
Cancel
Save