Merge branch 'augmentedProxy' of github.com:f1ori/yacy

source/de/anomic/http/server/HTTPDProxyHandler.java

@@ -70,6 +70,7 @@ import java.util.logging.Level;
 import java.util.logging.LogManager;
 import java.util.logging.Logger;
 
+import net.yacy.cider.AugmentedHtmlStream;
 import net.yacy.cora.document.MultiProtocolURI;
 import net.yacy.cora.protocol.Domains;
 import net.yacy.cora.protocol.HeaderFramework;
@@ -479,6 +480,8 @@ public final class HTTPDProxyHandler {
             final String connectHost = hostPart(host, port, yAddress);
             final String getUrl = "http://"+ connectHost + remotePath;
             
+            requestHeader.remove(HeaderFramework.HOST);
+            
             final HTTPClient client = setupHttpClient(requestHeader, connectHost);
             
             // send request
@@ -493,7 +496,14 @@ public final class HTTPDProxyHandler {
                 	throw new Exception(client.getHttpResponse().getStatusLine().toString());
                 }
 
-                final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), respond);
+                if(AugmentedHtmlStream.supportsMime(responseHeader.mime())) {
+                    // enable chunk encoding, because we don't know the length after annotating
+                    responseHeader.remove(HeaderFramework.CONTENT_LENGTH);
+                    responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked");
+
+                } 
+
+                ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, client.getHttpResponse().getStatusLine().getStatusCode(), respond);
 
                 // the cache does either not exist or is (supposed to be) stale
                 long sizeBeforeDelete = -1;
@@ -527,9 +537,9 @@ public final class HTTPDProxyHandler {
 
 //                prepareResponseHeader(responseHeader, res.getHttpVer());
                 prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
-
-                // sending the respond header back to the client
-                if (chunkedOut != null) {
+                
+                if(AugmentedHtmlStream.supportsMime(responseHeader.mime())) {
+                	// chunked encoding disables somewhere, add it again
                     responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked");
                 }
 
@@ -544,7 +554,7 @@ public final class HTTPDProxyHandler {
 
                 if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) {
 
-                    final OutputStream outStream = chunkedOut != null ? chunkedOut : respond;
+                    OutputStream outStream = chunkedOut != null ? chunkedOut : respond;
                     final Response response = new Response(
                             request,
                             requestHeader,
@@ -555,6 +565,10 @@ public final class HTTPDProxyHandler {
                     final String storeError = response.shallStoreCacheForProxy();
                     final boolean storeHTCache = response.profile().storeHTCache();
                     final String supportError = TextParser.supports(response.url(), response.getMimeType());
+
+                    if(AugmentedHtmlStream.supportsMime(responseHeader.mime())) {
+                        outStream = new AugmentedHtmlStream(outStream, responseHeader.getCharSet());
+                    } 
                     if (
                             /*
                              * Now we store the response into the htcache directory if
@@ -623,6 +637,7 @@ public final class HTTPDProxyHandler {
                         conProp.setProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
                     }
 
+                    outStream.close();
                     if (chunkedOut != null) {
                         chunkedOut.finish();
                         chunkedOut.flush();
@@ -672,7 +687,7 @@ public final class HTTPDProxyHandler {
             final RequestHeader requestHeader, 
             final ResponseHeader cachedResponseHeader,
             final byte[] cacheEntry,
-            final OutputStream respond
+            OutputStream respond
     ) throws IOException {
         
         final String httpVer = conProp.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER);
@@ -708,6 +723,10 @@ public final class HTTPDProxyHandler {
                 // send cached header with replaced date and added length 
                 HTTPDemon.sendRespondHeader(conProp,respond,httpVer,203,cachedResponseHeader);
                 //respondHeader(respond, "203 OK", cachedResponseHeader); // respond with 'non-authoritative'
+
+                if(AugmentedHtmlStream.supportsMime(cachedResponseHeader.mime())) {
+                    respond = new AugmentedHtmlStream(respond, cachedResponseHeader.getCharSet());
+                } 
                 
                 // send also the complete body now from the cache
                 // simply read the file and transfer to out socket
@@ -1085,6 +1104,7 @@ public final class HTTPDProxyHandler {
                     forceConnectionClose(conProp);
                 } else {
                     chunkedOut = new ChunkedOutputStream(respond);
+                    responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked");
                 }
                 responseHeader.remove(HeaderFramework.CONTENT_LENGTH);
             }
@@ -1154,7 +1174,7 @@ public final class HTTPDProxyHandler {
         headers.remove(RequestHeader.X_CACHE_LOOKUP);     
         
         // remove transfer encoding header
-        headers.remove(HeaderFramework.TRANSFER_ENCODING);
+        //headers.remove(HeaderFramework.TRANSFER_ENCODING);
         
         //removing yacy status headers
         headers.remove(HeaderFramework.X_YACY_KEEP_ALIVE_REQUEST_COUNT);

source/net/yacy/cider/AugmentedHtmlStream.java

@@ -0,0 +1,51 @@
+package net.yacy.cider;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.Charset;
+
+public class AugmentedHtmlStream extends FilterOutputStream {
+	private Writer out;
+	private ByteArrayOutputStream buffer;
+	private Charset charset;
+
+	public AugmentedHtmlStream(OutputStream out, Charset charset) {
+		super(out);
+		this.out = new BufferedWriter(new OutputStreamWriter(out, charset));
+		this.buffer = new ByteArrayOutputStream();
+		this.charset = charset;
+	}
+	
+	public void write(int b) throws IOException {
+		this.buffer.write(b);
+	}
+	
+	public void write(byte[] b, int off, int len) throws IOException {
+		this.buffer.write(b, off, len);
+	}
+	
+	public void close() throws IOException {
+		StringBuffer b = new StringBuffer(this.buffer.toString(charset.name()));
+		b = process(b);
+		out.write(b.toString());
+		System.out.println(b);
+		out.close();
+	}
+	
+	public StringBuffer process(StringBuffer data) {
+		System.out.println("got something!");
+		data.append("It works!");
+		return data;
+	}
+	
+	public static boolean supportsMime(String mime) {
+		System.out.println("mime" +mime);
+		return mime.split(";")[0].equals("text/html");
+	}
+
+}