harmonization of http post of files for one and several files - this had

been differently - and wrong for several files. also: base64-encoding
for gzipped push files because our data structures currently only
supports ASCII POST pushes..

htroot/api/push_p.java

@@ -74,7 +74,7 @@ public class push_p {
                 String collection = post.get("collection-" + i, "");
                 String lastModified = post.get("lastModified-" + i, ""); // must be in RFC1123 format
                 String contentType = post.get("contentType-" + i, "");
-                String data64 = post.get("data-" + i, ""); // file uploads are base64encoded in YaCyDefaultServlet.parseMultipart
+                String data64 = post.get("data-" + i + "$file", ""); // multi-file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
                 byte[] data = Base64Order.standardCoder.decode(data64);
                 if ((data == null || data.length == 0) && data64.length() > 0) data = UTF8.getBytes(data64); // for test cases
     

source/net/yacy/http/servlets/YaCyDefaultServlet.java

@@ -88,7 +88,6 @@ import org.apache.commons.fileupload.FileItem;
 import org.apache.commons.fileupload.FileItemFactory;
 import org.apache.commons.fileupload.disk.DiskFileItemFactory;
 import org.apache.commons.fileupload.servlet.ServletFileUpload;
-
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.MimeTypes;
@@ -148,7 +147,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
     protected ConcurrentHashMap<File, SoftReference<Method>> templateMethodCache = null;
     // settings for multipart/form-data
     protected static final File TMPDIR = new File(System.getProperty("java.io.tmpdir"));
-    protected static final int SIZE_FILE_THRESHOLD = 100 * 1024 * 1024; // 100 MB is a lot but appropriate for multi-document pushed using the push_p.json servlet
+    protected static final int SIZE_FILE_THRESHOLD = 1024 * 1024 * 1024; // 1GB is a lot but appropriate for multi-document pushed using the push_p.json servlet
     protected static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(SIZE_FILE_THRESHOLD, TMPDIR);
     private final static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newCachedThreadPool());
     /* ------------------------------------------------------------ */
@@ -1114,8 +1113,19 @@ public class YaCyDefaultServlet extends HttpServlet  {
                 }
             }
             if (files.size() <= 1) { // TODO: should include additonal checks to limit parameter.size below rel. large SIZE_FILE_THRESHOLD
-                for (Map.Entry<String, byte[]> fe: files) { // add the file content to parameter fieldname$file
-                    args.put(fe.getKey()+"$file", (fe.getValue()));
+                for (Map.Entry<String, byte[]> job: files) { // add the file content to parameter fieldname$file
+                    String n = job.getKey();
+                    byte[] v = job.getValue();
+                    String filename = args.get(n);
+                    if (filename != null && filename.endsWith(".gz")) {
+                        // transform this value into base64
+                        String b64 = Base64Order.standardCoder.encode(v);
+                        args.put(n + "$file", b64);
+                        args.remove(n);
+                        args.put(n, filename + ".base64");
+                    } else {
+                        args.put(n + "$file", v); // the byte[] is transformed into UTF8. You cannot push binaries here
+                    }
                 }
             } else {
                 // do this concurrently (this would all be superfluous if serverObjects could store byte[] instead only String)
@@ -1129,8 +1139,15 @@ public class YaCyDefaultServlet extends HttpServlet  {
                         public void run() {
                             Map.Entry<String, byte[]> job;
                             try {while ((job = files.take()) != POISON) {
-                                String b64 = Base64Order.standardCoder.encode(job.getValue());
-                                synchronized (args) {args.put(job.getKey(), b64);}
+                                String n = job.getKey();
+                                byte[] v = job.getValue();
+                                String filename = args.get(n);
+                                String b64 = Base64Order.standardCoder.encode(v);
+                                synchronized (args) {
+                                    args.put(n + "$file", b64);
+                                    args.remove(n);
+                                    args.put(n, filename + ".base64");
+                                }
                             }} catch (InterruptedException e) {}
                         }
                     };