From cf32a9262970600f23ce419dab1a7cb8573993ff Mon Sep 17 00:00:00 2001 From: reger Date: Sun, 13 Oct 2013 20:56:03 +0200 Subject: [PATCH] - add size check to multipart form data handling of YaCyDefaultServlet (same as in HTTPDemon.parseMultipart) - reduce Jetty logging - give build.run a bit more memory (set to YaCy.default 600m from 512m) --- build.xml | 2 +- defaults/yacy.logging | 3 +- .../yacy/http/Jetty8YaCyDefaultServlet.java | 1 - source/net/yacy/http/YaCyDefaultServlet.java | 40 +++++++++++++------ 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/build.xml b/build.xml index a78f2725a..8ed311960 100644 --- a/build.xml +++ b/build.xml @@ -652,7 +652,7 @@ - + diff --git a/defaults/yacy.logging b/defaults/yacy.logging index abbb105f6..818361f7a 100644 --- a/defaults/yacy.logging +++ b/defaults/yacy.logging @@ -72,8 +72,7 @@ net.yacy.kelondro.logging.LogalizerHandler.debug = false net.yacy.kelondro.logging.LogalizerHandler.parserPackage = net.yacy.kelondro.logging # Poperties for the HttpServer (Jetty) -org.eclipse.jetty.io.level = INFO -org.eclipse.jetty.http.HttpParser.level = INFO +org.eclipse.jetty.level = INFO org.apache.http.level = OFF org.apache.commons.httpclient.level = OFF diff --git a/source/net/yacy/http/Jetty8YaCyDefaultServlet.java b/source/net/yacy/http/Jetty8YaCyDefaultServlet.java index dd2fff882..2153ce657 100644 --- a/source/net/yacy/http/Jetty8YaCyDefaultServlet.java +++ b/source/net/yacy/http/Jetty8YaCyDefaultServlet.java @@ -21,7 +21,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URL; import java.util.Enumeration; import java.util.List; diff --git a/source/net/yacy/http/YaCyDefaultServlet.java b/source/net/yacy/http/YaCyDefaultServlet.java index fa9a665cb..e698c6fbb 100644 --- a/source/net/yacy/http/YaCyDefaultServlet.java +++ b/source/net/yacy/http/YaCyDefaultServlet.java @@ -34,8 +34,10 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.util.Enumeration; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.servlet.RequestDispatcher; @@ -65,6 +67,7 @@ import net.yacy.server.serverSwitch; import net.yacy.server.servletProperties; import net.yacy.visualization.RasterPlotter; import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.FileItemFactory; import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.servlet.ServletFileUpload; @@ -137,7 +140,10 @@ public abstract class YaCyDefaultServlet extends HttpServlet implements Resource protected File _htDocsPath; protected static final serverClassLoader provider = new serverClassLoader(/*this.getClass().getClassLoader()*/); protected ConcurrentHashMap> templateMethodCache = null; - + // settings for multipart/form-data + protected static final File TMPDIR = new File(System.getProperty("java.io.tmpdir")); + protected static final int SIZE_FILE_THRESHOLD = 20 * 1024 * 1024; + protected static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(SIZE_FILE_THRESHOLD, TMPDIR); /* ------------------------------------------------------------ */ @Override public void init() throws UnavailableException { @@ -762,15 +768,16 @@ public abstract class YaCyDefaultServlet extends HttpServlet implements Resource * @param request * @param args found fields/values are added to the map */ - protected void parseMultipart(HttpServletRequest request, serverObjects args) { - DiskFileItemFactory factory = new DiskFileItemFactory(); - // maximum size that will be stored in memory - factory.setSizeThreshold(4096 * 16); - // Location to save data that is larger than maxMemSize. - // factory.setRepository(new File(".")); - // Create a new file upload handler - ServletFileUpload upload = new ServletFileUpload(factory); - upload.setSizeMax(4096 * 16); + protected void parseMultipart(HttpServletRequest request, serverObjects args) throws IOException { + + // reject too large uploads + if (request.getContentLength() > SIZE_FILE_THRESHOLD) throw new IOException("FileUploadException: uploaded file too large = " + request.getContentLength()); + + // check if we have enough memory + if (!MemoryControl.request(request.getContentLength() * 3, false)) { + throw new IOException("not enough memory available for request. request.getContentLength() = " + request.getContentLength() + ", MemoryControl.available() = " + MemoryControl.available()); + } + ServletFileUpload upload = new ServletFileUpload(DISK_FILE_ITEM_FACTORY); try { // Parse the request to get form field items @SuppressWarnings("unchecked") @@ -778,9 +785,16 @@ public abstract class YaCyDefaultServlet extends HttpServlet implements Resource // Process the uploaded file items Iterator i = fileItems.iterator(); while (i.hasNext()) { - FileItem fi = i.next(); - if (fi.isFormField()) { - args.put(fi.getFieldName(), fi.getString()); + FileItem item = i.next(); + if (item.isFormField()) { + // simple text + if (item.getContentType() == null || !item.getContentType().contains("charset")) { + // old yacy clients use their local default charset, on most systems UTF-8 (I hope ;) + args.add(item.getFieldName(), item.getString("UTF-8")); + } else { + // use default encoding (given as header or ISO-8859-1) + args.add(item.getFieldName(), item.getString()); + } } } } catch (Exception ex) {