boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

... migrating to HttpComponents-Client-4.x ...

Browse Source 
make the occurrence of multiple header-keys possible

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7031 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 15 years ago
parent 171f2bd84e
commit a6ed6e8cb9
7 changed files with 22 additions and 27 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -355,7 +355,7 @@ public class RobotsTxt {
// res = client.GET(robotsURL.toString());
robotsTxt = client.GETbytes(robotsURL.toString());
final int code = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
// check the response status
// if (res.getStatusLine().startsWith("2")) {

4
source/de/anomic/crawler/retrieval/HTTPLoader.java
Unescape View File

@ -127,7 +127,7 @@ public final class HTTPLoader {
// send request
// res = client.GET(request.url().toString(), maxFileSize);
final byte[] responseBody = client.GETbytes(request.url().toString(), maxFileSize);
final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
final int code = client.getHttpResponse().getStatusLine().getStatusCode();
// FIXME: 30*-handling (bottom) is never reached
// we always get the final content because httpClient.followRedirects = true
@ -258,7 +258,7 @@ public final class HTTPLoader {
// send request
// res = client.GET(request.url().toString(), Long.MAX_VALUE);
final byte[] responseBody = client.GETbytes(request.url().toString(), Long.MAX_VALUE);
final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
final int code = client.getHttpResponse().getStatusLine().getStatusCode();
// FIXME: 30*-handling (bottom) is never reached
// we always get the final content because httpClient.followRedirects = true

8
source/de/anomic/http/server/HTTPDProxyHandler.java
Unescape View File

@ -486,7 +486,7 @@ public final class HTTPDProxyHandler {
conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader);
// final ResponseHeader responseHeader = res.getResponseHeader();
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders());
// determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) {
// throw new Exception(res.getStatusLine());
@ -845,7 +845,7 @@ public final class HTTPDProxyHandler {
// if (responseHeader.isEmpty()) {
// throw new Exception(res.getStatusLine());
// }
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders());
if (responseHeader.isEmpty()) {
throw new Exception(client.getHttpResponse().getStatusLine().toString());
}
@ -976,7 +976,7 @@ public final class HTTPDProxyHandler {
if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
// final ResponseHeader responseHeader = res.getResponseHeader();
final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders());
// determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) {
// throw new Exception(res.getStatusLine());
@ -1307,7 +1307,7 @@ public final class HTTPDProxyHandler {
try {
// response = remoteProxy.CONNECT(host, port);
remoteProxy.HEADResponse("http://" + host + ":" + port);
ResponseHeader header = new ResponseHeader(null, remoteProxy.getHeaderHashMap());
ResponseHeader header = new ResponseHeader(remoteProxy.getHttpResponse().getAllHeaders());
// outputs a logline to the serverlog with the current status
// log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString());

9
source/de/anomic/http/server/ResponseHeader.java
Unescape View File

@ -30,6 +30,8 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import org.apache.http.Header;
import net.yacy.kelondro.logging.Log;
@ -42,6 +44,13 @@ public class ResponseHeader extends HeaderFramework {
public ResponseHeader() {
super();
}
public ResponseHeader(Header[] headers) {
super();
for (final Header h : headers) {
this.add(h.getName(), h.getValue());
}
}
public ResponseHeader(final HashMap<String, String> reverseMappingCache) {
super(reverseMappingCache);

10
source/de/anomic/search/Switchboard.java
Unescape View File

@ -2133,8 +2133,8 @@ public final class Switchboard extends serverSwitch {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount());
// close unused connections
// Client.cleanup();
} else if (kbytesUp > 512) {
log.logInfo("dhtTransferJob: too much upload, currently uploading: " + kbytesUp + " Kb");
} else if (kbytesUp > 128) {
log.logInfo("dhtTransferJob: too much upload(1), currently uploading: " + kbytesUp + " Kb");
} else {
byte[] startHash = null, limitHash = null;
int tries = 10;
@ -2167,8 +2167,8 @@ public final class Switchboard extends serverSwitch {
log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount());
// close unused connections
// Client.cleanup();
} else if (kbytesUp > 512) {
log.logInfo("dhtTransferJob: too much upload, currently uploading: " + kbytesUp + " Kb");
} else if (kbytesUp > 256) {
log.logInfo("dhtTransferJob: too much upload(2), currently uploading: " + kbytesUp + " Kb");
} else {
boolean dequeued = this.dhtDispatcher.dequeueContainer();
hasDoneSomething = hasDoneSomething | dequeued;
@ -2363,7 +2363,7 @@ public final class Switchboard extends serverSwitch {
final long start = System.currentTimeMillis();
// header = Client.whead(url.toString(), reqHeader);
client.HEADResponse(url.toString());
header = new ResponseHeader(null, client.getHeaderHashMap());
header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
final long loadtime = System.currentTimeMillis() - start;
// if (header == null) {
if (header == null) {

2
source/de/anomic/yacy/yacyRelease.java
Unescape View File

@ -313,7 +313,7 @@ public final class yacyRelease extends yacyVersion {
// res = client.GET(this.getUrl().toString());
client.setTimout(120000);
client.GET(this.getUrl().toString());
final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
// final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
final boolean unzipped = header.gzip() && (header.mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file

14
source/net/yacy/cora/protocol/Client.java
Unescape View File

@ -29,7 +29,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Set;
import java.util.Map.Entry;
@ -46,7 +45,6 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
@ -148,9 +146,6 @@ public class Client {
final SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443));
// cookie policy
HttpClientParams.setCookiePolicy(httpParams, CookiePolicy.RFC_2965);
ClientConnectionManager clientConnectionManager = new ThreadSafeClientConnManager(httpParams, schemeRegistry);
@ -331,15 +326,6 @@ public class Client {
return httpResponse;
}
public HashMap<String, String> getHeaderHashMap() {
if (httpResponse == null) return null;
final HashMap<String, String> hmap = new HashMap<String, String>();
for (Header h : httpResponse.getAllHeaders()) {
hmap.put(h.getName(), h.getValue());
}
return hmap;
}
public void writeTo(final OutputStream outputStream) throws IOException {
if (httpResponse != null && currentRequest != null) {
final HttpEntity httpEntity = httpResponse.getEntity();

Write Preview
Loading…
Cancel
Save