|
|
|
|
@ -43,7 +43,6 @@
|
|
|
|
|
package de.anomic.http.server;
|
|
|
|
|
|
|
|
|
|
import java.io.BufferedReader;
|
|
|
|
|
//import java.io.ByteArrayInputStream;
|
|
|
|
|
import java.io.ByteArrayOutputStream;
|
|
|
|
|
import java.io.File;
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
@ -88,10 +87,8 @@ import net.yacy.kelondro.util.FileUtils;
|
|
|
|
|
import net.yacy.repository.Blacklist;
|
|
|
|
|
import net.yacy.search.Switchboard;
|
|
|
|
|
import net.yacy.search.SwitchboardConstants;
|
|
|
|
|
|
|
|
|
|
import de.anomic.crawler.retrieval.Request;
|
|
|
|
|
import de.anomic.crawler.retrieval.Response;
|
|
|
|
|
//import de.anomic.http.client.Client;
|
|
|
|
|
import de.anomic.http.client.Cache;
|
|
|
|
|
import de.anomic.server.serverCore;
|
|
|
|
|
import de.anomic.server.serverObjects;
|
|
|
|
|
@ -234,7 +231,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
*/
|
|
|
|
|
private static final StringBuilder userAgentStr = new StringBuilder();
|
|
|
|
|
|
|
|
|
|
public static void handleOutgoingCookies(final RequestHeader requestHeader, final String targethost, final String clienthost) {
|
|
|
|
|
private static void handleOutgoingCookies(final RequestHeader requestHeader, final String targethost, final String clienthost) {
|
|
|
|
|
/*
|
|
|
|
|
The syntax for the header is:
|
|
|
|
|
|
|
|
|
|
@ -257,7 +254,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static void handleIncomingCookies(final ResponseHeader respondHeader, final String serverhost, final String targetclient) {
|
|
|
|
|
private static void handleIncomingCookies(final ResponseHeader respondHeader, final String serverhost, final String targetclient) {
|
|
|
|
|
/*
|
|
|
|
|
The syntax for the Set-Cookie response header is
|
|
|
|
|
|
|
|
|
|
@ -363,7 +360,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
// handle outgoing cookies
|
|
|
|
|
handleOutgoingCookies(requestHeader, host, ip);
|
|
|
|
|
prepareRequestHeader(conProp, requestHeader, hostlow);
|
|
|
|
|
ResponseHeader cachedResponseHeader = Cache.getResponseHeader(url.hash());
|
|
|
|
|
final ResponseHeader cachedResponseHeader = Cache.getResponseHeader(url.hash());
|
|
|
|
|
|
|
|
|
|
// why are files unzipped upon arrival? why not zip all files in cache?
|
|
|
|
|
// This follows from the following premises
|
|
|
|
|
@ -409,7 +406,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
"200 OK",
|
|
|
|
|
sb.crawler.defaultProxyProfile
|
|
|
|
|
);
|
|
|
|
|
byte[] cacheContent = Cache.getContent(url.hash());
|
|
|
|
|
final byte[] cacheContent = Cache.getContent(url.hash());
|
|
|
|
|
if (cacheContent != null && response.isFreshForProxy()) {
|
|
|
|
|
if (log.isFinest()) log.logFinest(reqID + " fulfill request from cache");
|
|
|
|
|
fulfillRequestFromCache(conProp, url, requestHeader, cachedResponseHeader, cacheContent, countedRespond);
|
|
|
|
|
@ -467,7 +464,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
String yAddress = resolveYacyDomains(host);
|
|
|
|
|
|
|
|
|
|
// re-calc the url path
|
|
|
|
|
String remotePath = (args == null) ? path : (path + "?" + args); // with leading '/'
|
|
|
|
|
final String remotePath = (args == null) ? path : (path + "?" + args); // with leading '/'
|
|
|
|
|
|
|
|
|
|
// remove yacy-subdomain-path, when accessing /env
|
|
|
|
|
if ( (yAddress != null)
|
|
|
|
|
@ -500,9 +497,9 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
long sizeBeforeDelete = -1;
|
|
|
|
|
if (cachedResponseHeader != null) {
|
|
|
|
|
// delete the cache
|
|
|
|
|
ResponseHeader rh = Cache.getResponseHeader(url.hash());
|
|
|
|
|
final ResponseHeader rh = Cache.getResponseHeader(url.hash());
|
|
|
|
|
if (rh != null && (sizeBeforeDelete = rh.getContentLength()) == 0) {
|
|
|
|
|
byte[] b = Cache.getContent(url.hash());
|
|
|
|
|
final byte[] b = Cache.getContent(url.hash());
|
|
|
|
|
if (b != null) sizeBeforeDelete = b.length;
|
|
|
|
|
}
|
|
|
|
|
Cache.delete(url);
|
|
|
|
|
@ -570,7 +567,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
) {
|
|
|
|
|
// we don't write actually into a file, only to RAM, and schedule writing the file.
|
|
|
|
|
// int l = res.getResponseHeader().size();
|
|
|
|
|
int l = responseHeader.size();
|
|
|
|
|
final int l = responseHeader.size();
|
|
|
|
|
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);
|
|
|
|
|
|
|
|
|
|
final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});
|
|
|
|
|
@ -591,7 +588,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
try {
|
|
|
|
|
Cache.store(response.url(), response.getResponseHeader(), cacheArray);
|
|
|
|
|
sb.toIndexer(response);
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
|
} catch (final IOException e) {
|
|
|
|
|
log.logWarning("cannot write " + response.url() + " to Cache (1): " + e.getMessage(), e);
|
|
|
|
|
}
|
|
|
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE, "TCP_MISS");
|
|
|
|
|
@ -606,7 +603,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
try {
|
|
|
|
|
Cache.store(response.url(), response.getResponseHeader(), cacheArray);
|
|
|
|
|
sb.toIndexer(response);
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
|
} catch (final IOException e) {
|
|
|
|
|
log.logWarning("cannot write " + response.url() + " to Cache (2): " + e.getMessage(), e);
|
|
|
|
|
}
|
|
|
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE, "TCP_REFRESH_MISS");
|
|
|
|
|
@ -629,7 +626,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
chunkedOut.flush();
|
|
|
|
|
}
|
|
|
|
|
} // end hasBody
|
|
|
|
|
} catch(SocketException se) {
|
|
|
|
|
} catch(final SocketException se) {
|
|
|
|
|
// if opened ...
|
|
|
|
|
// if(res != null) {
|
|
|
|
|
// // client cut proxy connection, abort download
|
|
|
|
|
@ -774,7 +771,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
final String hostlow = host.toLowerCase();
|
|
|
|
|
|
|
|
|
|
// re-calc the url path
|
|
|
|
|
String remotePath = (args == null) ? path : (path + "?" + args);
|
|
|
|
|
final String remotePath = (args == null) ? path : (path + "?" + args);
|
|
|
|
|
|
|
|
|
|
if (Switchboard.urlBlacklist.isListed(Blacklist.BLACKLIST_PROXY, hostlow, remotePath)) {
|
|
|
|
|
HTTPDemon.sendRespondError(conProp,respond,4,403,null,
|
|
|
|
|
@ -845,7 +842,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static void doPost(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream respond, InputStream body) throws IOException {
|
|
|
|
|
public static void doPost(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream respond, final InputStream body) throws IOException {
|
|
|
|
|
assert conProp != null : "precondition violated: conProp != null";
|
|
|
|
|
assert requestHeader != null : "precondition violated: requestHeader != null";
|
|
|
|
|
assert body != null : "precondition violated: body != null";
|
|
|
|
|
@ -892,7 +889,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
String yAddress = resolveYacyDomains(host);
|
|
|
|
|
|
|
|
|
|
// re-calc the url path
|
|
|
|
|
String remotePath = (args == null) ? path : (path + "?" + args);
|
|
|
|
|
final String remotePath = (args == null) ? path : (path + "?" + args);
|
|
|
|
|
|
|
|
|
|
// remove yacy-subdomain-path, when accessing /env
|
|
|
|
|
if ( (yAddress != null)
|
|
|
|
|
@ -952,7 +949,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
chunked.finish();
|
|
|
|
|
}
|
|
|
|
|
outStream.flush();
|
|
|
|
|
} catch(SocketException se) {
|
|
|
|
|
} catch(final SocketException se) {
|
|
|
|
|
// connection closed by client, abort download
|
|
|
|
|
client.finish();
|
|
|
|
|
} finally {
|
|
|
|
|
@ -1219,7 +1216,7 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
remoteProxy.HEADResponse("http://" + host + ":" + port);
|
|
|
|
|
ResponseHeader header = new ResponseHeader(remoteProxy.getHttpResponse().getAllHeaders());
|
|
|
|
|
final ResponseHeader header = new ResponseHeader(remoteProxy.getHttpResponse().getAllHeaders());
|
|
|
|
|
|
|
|
|
|
// outputs a logline to the serverlog with the current status
|
|
|
|
|
log.logInfo("CONNECT-RESPONSE: status=" + remoteProxy.getHttpResponse().getStatusLine() + ", header=" + header.toString());
|
|
|
|
|
@ -1302,26 +1299,26 @@ public final class HTTPDProxyHandler {
|
|
|
|
|
final byte[] buffer = new byte[512];
|
|
|
|
|
int len;
|
|
|
|
|
try {
|
|
|
|
|
while ((socket != null) &&
|
|
|
|
|
(socket.isBound()) &&
|
|
|
|
|
(!(socket.isClosed())) &&
|
|
|
|
|
(socket.isConnected()) &&
|
|
|
|
|
(!(terminate)) &&
|
|
|
|
|
(in != null) &&
|
|
|
|
|
(out != null) &&
|
|
|
|
|
((len = in.read(buffer)) >= 0)
|
|
|
|
|
while ((this.socket != null) &&
|
|
|
|
|
(this.socket.isBound()) &&
|
|
|
|
|
(!(this.socket.isClosed())) &&
|
|
|
|
|
(this.socket.isConnected()) &&
|
|
|
|
|
(!(this.terminate)) &&
|
|
|
|
|
(this.in != null) &&
|
|
|
|
|
(this.out != null) &&
|
|
|
|
|
((len = this.in.read(buffer)) >= 0)
|
|
|
|
|
) {
|
|
|
|
|
out.write(buffer, 0, len);
|
|
|
|
|
this.out.write(buffer, 0, len);
|
|
|
|
|
}
|
|
|
|
|
} catch (final IOException e) {
|
|
|
|
|
// do nothing
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
} catch (final Exception e) {
|
|
|
|
|
Log.logException(e);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void pleaseTerminate() {
|
|
|
|
|
terminate = true;
|
|
|
|
|
this.terminate = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
sixcooler
13 years ago