diff --git a/source/net/yacy/http/servlets/YaCyDefaultServlet.java b/source/net/yacy/http/servlets/YaCyDefaultServlet.java index cd981f9b2..ad90db7c9 100644 --- a/source/net/yacy/http/servlets/YaCyDefaultServlet.java +++ b/source/net/yacy/http/servlets/YaCyDefaultServlet.java @@ -37,13 +37,16 @@ import java.util.Enumeration; import java.util.Iterator; import java.util.List; import java.util.concurrent.ConcurrentHashMap; +import java.util.zip.GZIPInputStream; import javax.servlet.RequestDispatcher; import javax.servlet.ServletContext; import javax.servlet.ServletException; +import javax.servlet.ServletInputStream; import javax.servlet.UnavailableException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; import javax.servlet.http.HttpServletResponse; import net.yacy.cora.date.GenericFormatter; @@ -63,7 +66,6 @@ import net.yacy.peers.operation.yacyBuildProperties; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.server.http.HTTPDFileHandler; -import net.yacy.server.http.HTTPDemon; import net.yacy.server.http.TemplateEngine; import net.yacy.server.serverClassLoader; import net.yacy.server.serverCore; @@ -777,11 +779,9 @@ public class YaCyDefaultServlet extends HttpServlet { RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt); // add multipart-form fields to parameter if (ServletFileUpload.isMultipartContent(request)) { - //TODO: added quickfix to support gzip encoded content - // using existing HTTPDemon.parseMultipart() final String bodyEncoding = request.getHeader(HeaderFramework.CONTENT_ENCODING); if (HeaderFramework.CONTENT_ENCODING_GZIP.equalsIgnoreCase(bodyEncoding)) { - HTTPDemon.parseMultipart(legacyRequestHeader, args, request.getInputStream()); + parseMultipart(new GZIPRequestWrapper(request),args); } else { parseMultipart(request, args); } @@ -986,6 +986,7 @@ public class YaCyDefaultServlet extends HttpServlet { throw new IOException("not enough memory available for request. request.getContentLength() = " + request.getContentLength() + ", MemoryControl.available() = " + MemoryControl.available()); } ServletFileUpload upload = new ServletFileUpload(DISK_FILE_ITEM_FACTORY); + upload.setFileSizeMax(SIZE_FILE_THRESHOLD); try { // Parse the request to get form field items @SuppressWarnings("unchecked") @@ -1009,4 +1010,52 @@ public class YaCyDefaultServlet extends HttpServlet { ConcurrentLog.logException(ex); } } + + /** + * wraps request to uncompress gzip'ed input stream + */ + private class GZIPRequestWrapper extends HttpServletRequestWrapper { + + private final ServletInputStream is; + + public GZIPRequestWrapper(HttpServletRequest request) throws IOException { + super(request); + this.is = new GZIPRequestStream(request); + } + + @Override + public ServletInputStream getInputStream() throws IOException { + return is; + } + + } + + private class GZIPRequestStream extends ServletInputStream { + + private final GZIPInputStream in; + + public GZIPRequestStream(HttpServletRequest request) throws IOException { + this.in = new GZIPInputStream(request.getInputStream()); + } + + @Override + public int read() throws IOException { + return in.read(); + } + + @Override + public int read(byte[] b) throws IOException { + return in.read(b); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return in.read(b, off, len); + } + + @Override + public void close() throws IOException { + in.close(); + } + } } diff --git a/source/net/yacy/server/http/HTTPDemon.java b/source/net/yacy/server/http/HTTPDemon.java index fc456438e..46799a13f 100644 --- a/source/net/yacy/server/http/HTTPDemon.java +++ b/source/net/yacy/server/http/HTTPDemon.java @@ -28,7 +28,6 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.net.InetAddress; @@ -36,12 +35,7 @@ import java.net.MalformedURLException; import java.util.Date; import java.util.HashMap; import java.util.Iterator; -import java.util.List; -import java.util.Map; import java.util.Map.Entry; -import java.util.zip.GZIPInputStream; - -import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.Domains; @@ -52,18 +46,9 @@ import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.NumberTools; import net.yacy.kelondro.util.FileUtils; -import net.yacy.kelondro.util.MemoryControl; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.FileItemFactory; -import org.apache.commons.fileupload.FileUpload; -import org.apache.commons.fileupload.FileUploadBase; -import org.apache.commons.fileupload.FileUploadException; -import org.apache.commons.fileupload.RequestContext; -import org.apache.commons.fileupload.disk.DiskFileItemFactory; - /** @@ -78,152 +63,12 @@ public final class HTTPDemon { private static final int ERRORCASE_MESSAGE = 4; private static final int ERRORCASE_FILE = 5; - private static final File TMPDIR = new File(System.getProperty("java.io.tmpdir")); - private static final int SIZE_FILE_THRESHOLD = 20 * 1024 * 1024; - private static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(SIZE_FILE_THRESHOLD, TMPDIR); // static objects private static volatile Switchboard switchboard = Switchboard.getSwitchboard(); public static boolean keepAliveSupport = false; - /** - * parses the message accordingly to RFC 1867 using "Commons FileUpload" (http://commons.apache.org/fileupload/) - * - * @author danielr - * @since 07.08.2008 - * @param header - * hier muss ARGC gesetzt werden! - * @param args - * @param in the raw body - * @return - * @throws IOException - */ - @SuppressWarnings("unchecked") - public - static Map parseMultipart(final RequestHeader header, final serverObjects args, final InputStream in) throws IOException { - - final InputStream body = prepareBody(header, in); - final RequestContext request = new yacyContextRequest(header, body); - - // check information - if (!FileUploadBase.isMultipartContent(request)) { - body.close(); - throw new IOException("the request is not a multipart-message!"); - } - - // reject too large uploads - if (request.getContentLength() > SIZE_FILE_THRESHOLD) { - body.close(); - throw new IOException("FileUploadException: uploaded file too large = " + request.getContentLength()); - } - - // check if we have enough memory - if (!MemoryControl.request(request.getContentLength() * 3, false)) { - body.close(); - throw new IOException("not enough memory available for request. request.getContentLength() = " + request.getContentLength() + ", MemoryControl.available() = " + MemoryControl.available()); - } - - // parse data in memory - final List items; - - try { - final FileUpload upload = new FileUpload(DISK_FILE_ITEM_FACTORY); - items = upload.parseRequest(request); - } catch (final FileUploadException e) { - body.close(); - throw new IOException("FileUploadException " + e.getMessage()); - } - - // format information for further usage - final Map files = new HashMap(); - byte[] fileContent; - for (final FileItem item : items) { - if (item.isFormField()) { - // simple text - if (item.getContentType() == null || !item.getContentType().contains("charset")) { - // old yacy clients use their local default charset, on most systems UTF-8 (I hope ;) - args.add(item.getFieldName(), item.getString("UTF-8")); - } else { - // use default encoding (given as header or ISO-8859-1) - args.add(item.getFieldName(), item.getString()); - } - } else { - // file - args.add(item.getFieldName(), item.getName()); - fileContent = FileUtils.read(item.getInputStream(), (int) item.getSize()); - item.getInputStream().close(); - files.put(item.getFieldName(), fileContent); - } - } - header.put("ARGC", String.valueOf(items.size())); // store argument count - body.close(); - return files; - } - - /** - * prepares the body so that it can be read as whole plain text - * (uncompress if necessary and ensure correct ending) - * - * @param header - * @param in - * @return - * @throws IOException - */ - private static InputStream prepareBody(final RequestHeader header, final InputStream in) throws IOException { - InputStream body = in; - // data may be compressed - final String bodyEncoding = header.get(HeaderFramework.CONTENT_ENCODING); - if(HeaderFramework.CONTENT_ENCODING_GZIP.equalsIgnoreCase(bodyEncoding) && !(body instanceof GZIPInputStream)) { - body = new GZIPInputStream(body); - // length of uncompressed data is unknown - header.remove(HeaderFramework.CONTENT_LENGTH); - } else { - // ensure the end of data (if client keeps alive the connection) - final long clength = header.getContentLength(); - if (clength > 0) { - body = new ContentLengthInputStream(body, clength); - } - } - return body; - } - - /** - * wraps the request into a org.apache.commons.fileupload.RequestContext - * - * @author danielr - * @since 07.08.2008 - */ - private static class yacyContextRequest extends RequestHeader implements RequestContext { - - private static final long serialVersionUID = -8936741958551376593L; - - private final InputStream inStream; - - /** - * creates a new yacyContextRequest - * - * @param header - * @param in - */ - public yacyContextRequest(final Map requestHeader, final InputStream in) { - super(null, requestHeader); - this.inStream = in; - } - - /* - * (non-Javadoc) - * - * @see org.apache.commons.fileupload.RequestContext#getInputStream() - */ - // @Override - @Override - public InputStream getInputStream() throws IOException { - return this.inStream; - } - - } - static final void sendRespondError( final HashMap conProp, final OutputStream respond,