diff --git a/source/de/anomic/http/server/HTTPDProxyHandler.java b/source/de/anomic/http/server/HTTPDProxyHandler.java index 58aead8aa..d0c076a1b 100644 --- a/source/de/anomic/http/server/HTTPDProxyHandler.java +++ b/source/de/anomic/http/server/HTTPDProxyHandler.java @@ -70,6 +70,7 @@ import java.util.logging.Level; import java.util.logging.LogManager; import java.util.logging.Logger; +import net.yacy.cider.AugmentedHtmlStream; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -479,6 +480,8 @@ public final class HTTPDProxyHandler { final String connectHost = hostPart(host, port, yAddress); final String getUrl = "http://"+ connectHost + remotePath; + requestHeader.remove(HeaderFramework.HOST); + final HTTPClient client = setupHttpClient(requestHeader, connectHost); // send request @@ -493,7 +496,14 @@ public final class HTTPDProxyHandler { throw new Exception(client.getHttpResponse().getStatusLine().toString()); } - final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), respond); + if(AugmentedHtmlStream.supportsMime(responseHeader.mime())) { + // enable chunk encoding, because we don't know the length after annotating + responseHeader.remove(HeaderFramework.CONTENT_LENGTH); + responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked"); + + } + + ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), respond); // the cache does either not exist or is (supposed to be) stale long sizeBeforeDelete = -1; @@ -527,9 +537,9 @@ public final class HTTPDProxyHandler { // prepareResponseHeader(responseHeader, res.getHttpVer()); prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString()); - - // sending the respond header back to the client - if (chunkedOut != null) { + + if(AugmentedHtmlStream.supportsMime(responseHeader.mime())) { + // chunked encoding disables somewhere, add it again responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked"); } @@ -544,7 +554,7 @@ public final class HTTPDProxyHandler { if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) { - final OutputStream outStream = chunkedOut != null ? chunkedOut : respond; + OutputStream outStream = chunkedOut != null ? chunkedOut : respond; final Response response = new Response( request, requestHeader, @@ -555,6 +565,10 @@ public final class HTTPDProxyHandler { final String storeError = response.shallStoreCacheForProxy(); final boolean storeHTCache = response.profile().storeHTCache(); final String supportError = TextParser.supports(response.url(), response.getMimeType()); + + if(AugmentedHtmlStream.supportsMime(responseHeader.mime())) { + outStream = new AugmentedHtmlStream(outStream, responseHeader.getCharSet()); + } if ( /* * Now we store the response into the htcache directory if @@ -623,6 +637,7 @@ public final class HTTPDProxyHandler { conProp.setProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); } + outStream.close(); if (chunkedOut != null) { chunkedOut.finish(); chunkedOut.flush(); @@ -672,7 +687,7 @@ public final class HTTPDProxyHandler { final RequestHeader requestHeader, final ResponseHeader cachedResponseHeader, final byte[] cacheEntry, - final OutputStream respond + OutputStream respond ) throws IOException { final String httpVer = conProp.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER); @@ -708,6 +723,10 @@ public final class HTTPDProxyHandler { // send cached header with replaced date and added length HTTPDemon.sendRespondHeader(conProp,respond,httpVer,203,cachedResponseHeader); //respondHeader(respond, "203 OK", cachedResponseHeader); // respond with 'non-authoritative' + + if(AugmentedHtmlStream.supportsMime(cachedResponseHeader.mime())) { + respond = new AugmentedHtmlStream(respond, cachedResponseHeader.getCharSet()); + } // send also the complete body now from the cache // simply read the file and transfer to out socket @@ -1085,6 +1104,7 @@ public final class HTTPDProxyHandler { forceConnectionClose(conProp); } else { chunkedOut = new ChunkedOutputStream(respond); + responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked"); } responseHeader.remove(HeaderFramework.CONTENT_LENGTH); } @@ -1154,7 +1174,7 @@ public final class HTTPDProxyHandler { headers.remove(RequestHeader.X_CACHE_LOOKUP); // remove transfer encoding header - headers.remove(HeaderFramework.TRANSFER_ENCODING); + //headers.remove(HeaderFramework.TRANSFER_ENCODING); //removing yacy status headers headers.remove(HeaderFramework.X_YACY_KEEP_ALIVE_REQUEST_COUNT); diff --git a/source/net/yacy/cider/AugmentedHtmlStream.java b/source/net/yacy/cider/AugmentedHtmlStream.java new file mode 100644 index 000000000..755aa8f88 --- /dev/null +++ b/source/net/yacy/cider/AugmentedHtmlStream.java @@ -0,0 +1,51 @@ +package net.yacy.cider; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.Charset; + +public class AugmentedHtmlStream extends FilterOutputStream { + private Writer out; + private ByteArrayOutputStream buffer; + private Charset charset; + + public AugmentedHtmlStream(OutputStream out, Charset charset) { + super(out); + this.out = new BufferedWriter(new OutputStreamWriter(out, charset)); + this.buffer = new ByteArrayOutputStream(); + this.charset = charset; + } + + public void write(int b) throws IOException { + this.buffer.write(b); + } + + public void write(byte[] b, int off, int len) throws IOException { + this.buffer.write(b, off, len); + } + + public void close() throws IOException { + StringBuffer b = new StringBuffer(this.buffer.toString(charset.name())); + b = process(b); + out.write(b.toString()); + System.out.println(b); + out.close(); + } + + public StringBuffer process(StringBuffer data) { + System.out.println("got something!"); + data.append("It works!"); + return data; + } + + public static boolean supportsMime(String mime) { + System.out.println("mime" +mime); + return mime.split(";")[0].equals("text/html"); + } + +}