boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Speed up the ProxyHandler:

Browse Source 
simplified cache-storing and make it concurrent in order to free the
clientconnection asap
let other prozesses wait on proxy-access like it was bevore
pull/1/head
sixcooler 11 years ago
parent 8cf4d04ae0
commit 0b2101c59c
1 changed files with 85 additions and 90 deletions
Show Stats Download Patch File Download Diff File

103
source/net/yacy/http/ProxyHandler.java
Unescape View File

@ -84,7 +84,7 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
return result;
}
static void convertHeaderToJetty(HttpResponse in, HttpServletResponse out) {
private void convertHeaderToJetty(HttpResponse in, HttpServletResponse out) {
for(Header h: in.getAllHeaders()) {
out.addHeader(h.getName(), h.getValue());
}
@ -95,10 +95,56 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
headers.removeHeaders(HeaderFramework.CONTENT_LENGTH);
}
private void deleteFromCache(final byte[] hash) {
// long size = -1;
ResponseHeader rh = Cache.getResponseHeader(hash);
if (rh != null) {
// delete the cache
// if ((size = rh.getContentLength()) == 0) {
// byte[] b = Cache.getContent(hash);
// if (b != null) size = b.length;
// }
try {
Cache.delete(hash);
} catch (final IOException e) {
// log refresh miss
HTTPDProxyHandler.proxyLog.fine(e.getMessage());
}
}
}
private void storeToCache(final Response response, final byte[] array) {
final Thread t = new Thread() {
final Response yacyResponse = response;
final byte[] cacheArray = array;
public void run() {
this.setName("ProxyHandler.storeToCache(" + yacyResponse.url() + ")");
if (yacyResponse == null) return;
// the cache does either not exist or is (supposed to be) stale
deleteFromCache(yacyResponse.url().hash());
if (cacheArray == null || cacheArray.length <= 0) return;
yacyResponse.setContent(cacheArray);
try {
Cache.store(yacyResponse.url(), yacyResponse.getResponseHeader(), cacheArray);
sb.toIndexer(yacyResponse);
} catch (IOException e) {
//log.logWarning("cannot write " + response.url() + " to Cache (1): " + e.getMessage(), e);
}
}
};
t.setPriority(Thread.MIN_PRIORITY);
t.start();
}
@Override
public void handleRemote(String target, Request baseRequest, HttpServletRequest request,
HttpServletResponse response) throws IOException, ServletException {
sb.proxyLastAccess = System.currentTimeMillis();
if (request.getMethod().equalsIgnoreCase(HeaderFramework.METHOD_CONNECT)) {
handleConnect(request, response);
return;
@ -133,22 +179,6 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
}
cleanResponseHeader(clientresponse);
// TODO: is this fast, if not, use value from ProxyCacheHandler
ResponseHeader cachedResponseHeader = Cache.getResponseHeader(digestURI.hash());
// the cache does either not exist or is (supposed to be) stale
long sizeBeforeDelete = -1;
if (cachedResponseHeader != null) {
// delete the cache
ResponseHeader rh = Cache.getResponseHeader(digestURI.hash());
if (rh != null && (sizeBeforeDelete = rh.getContentLength()) == 0) {
byte[] b = Cache.getContent(digestURI.hash());
if (b != null) sizeBeforeDelete = b.length;
}
Cache.delete(digestURI.hash());
// log refresh miss
}
// reserver cache entry
final net.yacy.crawler.retrieval.Request yacyRequest = new net.yacy.crawler.retrieval.Request(
null,
@ -194,42 +224,7 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
client.writeTo(toClientAndMemory);
// cached bytes
byte[] cacheArray;
if (byteStream.size() > 0) {
cacheArray = byteStream.toByteArray();
} else {
cacheArray = null;
}
//if (log.isFine()) log.logFine(reqID +" writeContent of " + url + " produced cacheArray = " + ((cacheArray == null) ? "null" : ("size=" + cacheArray.length)));
if (sizeBeforeDelete == -1) {
// totally fresh file
yacyResponse.setContent(cacheArray);
try {
Cache.store(yacyResponse.url(), yacyResponse.getResponseHeader(), cacheArray);
sb.toIndexer(yacyResponse);
} catch (IOException e) {
//log.logWarning("cannot write " + response.url() + " to Cache (1): " + e.getMessage(), e);
}
// log cache miss
} else if (cacheArray != null && sizeBeforeDelete == cacheArray.length) {
// TODO: what should happen here?
// before we came here we deleted a cache entry
cacheArray = null;
//cacheManager.push(cacheEntry); // unnecessary update
// log cache refresh fail miss
} else {
// before we came here we deleted a cache entry
yacyResponse.setContent(cacheArray);
try {
Cache.store(yacyResponse.url(), yacyResponse.getResponseHeader(), cacheArray);
sb.toIndexer(yacyResponse);
} catch (IOException e) {
//log.logWarning("cannot write " + response.url() + " to Cache (2): " + e.getMessage(), e);
}
// log refresh cache miss
}
storeToCache(yacyResponse, byteStream.toByteArray());
} else {
// no caching
/*if (log.isFine()) log.logFine(reqID +" "+ url.toString() + " not cached." +
@ -241,7 +236,7 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
client.writeTo(response.getOutputStream());
}
} catch(SocketException se) {
} catch(final SocketException se) {
throw new ServletException("Socket Exception: " + se.getMessage());
} finally {
client.finish();

Write Preview
Loading…
Cancel
Save