augmentedProxy, which forwards every proxy request to a

rewrite engine to customize existing webpages. originally implemented by
Florian Richter.

Conflicts:
	source/de/anomic/http/server/HTTPDProxyHandler.java

htroot/robots.java

@@ -10,18 +10,17 @@ import java.util.ArrayList;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.search.Switchboard;
 import net.yacy.search.SwitchboardConstants;
-
 import de.anomic.http.server.RobotsTxtConfig;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.server.servletProperties;
 
 public class robots {
-    
+
     public static servletProperties respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
         final servletProperties prop = new servletProperties();
         final RobotsTxtConfig rbc = ((Switchboard)env).robotstxtConfig;
-        
+
         if (rbc.isAllDisallowed()) {
             prop.put(RobotsTxtConfig.ALL, 1);
         } else {
@@ -35,7 +34,7 @@ public class robots {
             if (rbc.isSurftipsDisallowed()) prop.put(RobotsTxtConfig.ALL + "_" + RobotsTxtConfig.SURFTIPS, "1");
             if (rbc.isWikiDisallowed()) prop.put(RobotsTxtConfig.ALL + "_" + RobotsTxtConfig.WIKI, "1");
             if (rbc.isProfileDisallowed()) prop.put(RobotsTxtConfig.ALL + "_" + RobotsTxtConfig.PROFILE, "1");
-            
+
             if (rbc.isLockedDisallowed() || rbc.isDirsDisallowed()) {
                 final ArrayList<String>[] p = getFiles(env.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
                 if (rbc.isLockedDisallowed()) {
@@ -50,10 +49,10 @@ public class robots {
                 }
             }
         }
-        
+
         return prop;
     }
-    
+
     @SuppressWarnings("unchecked")
     private static ArrayList<String>[] getFiles(final String htrootPath) {
         final File htroot = new File(htrootPath);
@@ -75,6 +74,6 @@ public class robots {
                 htrootFiles.add(htroots[i]);
             }
         }
-        return new ArrayList[] { htrootFiles, htrootDirs };
+        return (ArrayList<String>[]) new Object[] { htrootFiles, htrootDirs };
     }
 }

source/de/anomic/http/server/AugmentedHtmlStream.java

@@ -0,0 +1,72 @@
+package de.anomic.http.server;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.Charset;
+
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.interaction.AugmentHtmlStream;
+import net.yacy.kelondro.data.meta.DigestURI;
+import net.yacy.search.Switchboard;
+
+public class AugmentedHtmlStream extends FilterOutputStream {
+	private Writer out;
+	private ByteArrayOutputStream buffer;
+	private Charset charset;
+	private DigestURI url;
+	private byte[] urlhash;
+	private RequestHeader requestHeader;
+
+	public AugmentedHtmlStream(OutputStream out, Charset charset, DigestURI url, byte[] urlhash, RequestHeader requestHeader) {
+		super(out);
+		this.out = new BufferedWriter(new OutputStreamWriter(out, charset));
+		this.buffer = new ByteArrayOutputStream();
+		this.charset = charset;
+		this.url = url;
+		this.urlhash = urlhash;
+		this.requestHeader = requestHeader;
+	}
+	
+	public void write(int b) throws IOException {
+		this.buffer.write(b);
+	}
+	
+	public void write(byte[] b, int off, int len) throws IOException {
+		this.buffer.write(b, off, len);
+	}
+	
+	public void close() throws IOException {
+		StringBuffer b = new StringBuffer(this.buffer.toString(charset.name()));
+		b = process(b);
+		out.write(b.toString());
+		out.close();
+	}
+	
+	public StringBuffer process(StringBuffer data) {
+		
+		if (Switchboard.getSwitchboard().getConfigBool("proxyAugmentation", false) == true) {
+			
+			if (!this.url.toNormalform(false, true).contains("currentyacypeer/")) {
+								
+				return AugmentHtmlStream.process (data, charset, url, requestHeader);
+				
+			} else {
+				return data;
+			}
+			
+		} else {		
+			return data;
+		}
+	}
+	
+	public static boolean supportsMime(String mime) {
+//		System.out.println("mime" +mime);
+		return mime.split(";")[0].equals("text/html");
+	}
+
+}

source/net/yacy/document/parser/images/genericImageParser.java

@@ -87,6 +87,7 @@ public class genericImageParser extends AbstractParser implements Parser {
         super("Generic Image Parser");
     }
 
+    @Override
     public Document[] parse(
             final MultiProtocolURI location,
             final String mimeType,
@@ -211,10 +212,12 @@ public class genericImageParser extends AbstractParser implements Parser {
              false)}; // images
     }
 
+    @Override
     public Set<String> supportedMimeTypes() {
         return SUPPORTED_MIME_TYPES;
     }
 
+    @Override
     public Set<String> supportedExtensions() {
         return SUPPORTED_EXTENSIONS;
     }

source/net/yacy/interaction/AugmentHtmlStream.java

@@ -0,0 +1,25 @@
+package net.yacy.interaction;
+
+import java.nio.charset.Charset;
+
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.kelondro.data.meta.DigestURI;
+
+
+public class AugmentHtmlStream {
+	
+	public static StringBuffer process (StringBuffer data, Charset charset, DigestURI url, RequestHeader requestHeader) {
+		
+		boolean augmented = false;
+		
+		String Doc = data.toString();
+		
+		if (augmented) {
+			
+			return (new StringBuffer (Doc));
+		} else {
+			return (data);
+		}
+	}
+
+}

source/net/yacy/kelondro/blob/ArrayStack.java

@@ -628,7 +628,7 @@ public class ArrayStack implements BLOB {
      */
     @Override
     public byte[] get(final byte[] key) throws IOException, RowSpaceExceededException {
-        if (this.blobs.size() == 0) return null;
+        if (this.blobs == null || this.blobs.size() == 0) return null;
         if (this.blobs.size() == 1) {
             final blobItem bi = this.blobs.get(0);
             return bi.blob.get(key);