boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

implement gzip input handling directly in defaultservlet

Browse Source 
(making reference to legacy httpdemon obsolete)
pull/1/head
reger 11 years ago
parent 8470dfe3f8
commit 710054bb37
2 changed files with 53 additions and 159 deletions
Show Stats Download Patch File Download Diff File

57
source/net/yacy/http/servlets/YaCyDefaultServlet.java
Unescape View File

@ -37,13 +37,16 @@ import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.GZIPInputStream;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.UnavailableException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
import net.yacy.cora.date.GenericFormatter;
@ -63,7 +66,6 @@ import net.yacy.peers.operation.yacyBuildProperties;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.server.http.HTTPDemon;
import net.yacy.server.http.TemplateEngine;
import net.yacy.server.serverClassLoader;
import net.yacy.server.serverCore;
@ -777,11 +779,9 @@ public class YaCyDefaultServlet extends HttpServlet {
RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
// add multipart-form fields to parameter
if (ServletFileUpload.isMultipartContent(request)) {
//TODO: added quickfix to support gzip encoded content
// using existing HTTPDemon.parseMultipart()
final String bodyEncoding = request.getHeader(HeaderFramework.CONTENT_ENCODING);
if (HeaderFramework.CONTENT_ENCODING_GZIP.equalsIgnoreCase(bodyEncoding)) {
HTTPDemon.parseMultipart(legacyRequestHeader, args, request.getInputStream());
parseMultipart(new GZIPRequestWrapper(request),args);
} else {
parseMultipart(request, args);
}
@ -986,6 +986,7 @@ public class YaCyDefaultServlet extends HttpServlet {
throw new IOException("not enough memory available for request. request.getContentLength() = " + request.getContentLength() + ", MemoryControl.available() = " + MemoryControl.available());
}
ServletFileUpload upload = new ServletFileUpload(DISK_FILE_ITEM_FACTORY);
upload.setFileSizeMax(SIZE_FILE_THRESHOLD);
try {
// Parse the request to get form field items
@SuppressWarnings("unchecked")
@ -1009,4 +1010,52 @@ public class YaCyDefaultServlet extends HttpServlet {
ConcurrentLog.logException(ex);
}
}
/**
* wraps request to uncompress gzip'ed input stream
*/
private class GZIPRequestWrapper extends HttpServletRequestWrapper {
private final ServletInputStream is;
public GZIPRequestWrapper(HttpServletRequest request) throws IOException {
super(request);
this.is = new GZIPRequestStream(request);
}
@Override
public ServletInputStream getInputStream() throws IOException {
return is;
}
}
private class GZIPRequestStream extends ServletInputStream {
private final GZIPInputStream in;
public GZIPRequestStream(HttpServletRequest request) throws IOException {
this.in = new GZIPInputStream(request.getInputStream());
}
@Override
public int read() throws IOException {
return in.read();
}
@Override
public int read(byte[] b) throws IOException {
return in.read(b);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
return in.read(b, off, len);
}
@Override
public void close() throws IOException {
in.close();
}
}
}

155
source/net/yacy/server/http/HTTPDemon.java
Unescape View File

@ -28,7 +28,6 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.InetAddress;
@ -36,12 +35,7 @@ import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.zip.GZIPInputStream;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.Domains;
@ -52,18 +46,9 @@ import net.yacy.cora.util.ByteBuffer;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.NumberTools;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.FileUpload;
import org.apache.commons.fileupload.FileUploadBase;
import org.apache.commons.fileupload.FileUploadException;
import org.apache.commons.fileupload.RequestContext;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
/**
@ -78,152 +63,12 @@ public final class HTTPDemon {
private static final int ERRORCASE_MESSAGE = 4;
private static final int ERRORCASE_FILE = 5;
private static final File TMPDIR = new File(System.getProperty("java.io.tmpdir"));
private static final int SIZE_FILE_THRESHOLD = 20 * 1024 * 1024;
private static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(SIZE_FILE_THRESHOLD, TMPDIR);
// static objects
private static volatile Switchboard switchboard = Switchboard.getSwitchboard();
public static boolean keepAliveSupport = false;
/**
* parses the message accordingly to RFC 1867 using "Commons FileUpload" (http://commons.apache.org/fileupload/)
*
* @author danielr
* @since 07.08.2008
* @param header
* hier muss ARGC gesetzt werden!
* @param args
* @param in the raw body
* @return
* @throws IOException
*/
@SuppressWarnings("unchecked")
public
static Map<String, byte[]> parseMultipart(final RequestHeader header, final serverObjects args, final InputStream in) throws IOException {
final InputStream body = prepareBody(header, in);
final RequestContext request = new yacyContextRequest(header, body);
// check information
if (!FileUploadBase.isMultipartContent(request)) {
body.close();
throw new IOException("the request is not a multipart-message!");
}
// reject too large uploads
if (request.getContentLength() > SIZE_FILE_THRESHOLD) {
body.close();
throw new IOException("FileUploadException: uploaded file too large = " + request.getContentLength());
}
// check if we have enough memory
if (!MemoryControl.request(request.getContentLength() * 3, false)) {
body.close();
throw new IOException("not enough memory available for request. request.getContentLength() = " + request.getContentLength() + ", MemoryControl.available() = " + MemoryControl.available());
}
// parse data in memory
final List<FileItem> items;
try {
final FileUpload upload = new FileUpload(DISK_FILE_ITEM_FACTORY);
items = upload.parseRequest(request);
} catch (final FileUploadException e) {
body.close();
throw new IOException("FileUploadException " + e.getMessage());
}
// format information for further usage
final Map<String, byte[]> files = new HashMap<String, byte[]>();
byte[] fileContent;
for (final FileItem item : items) {
if (item.isFormField()) {
// simple text
if (item.getContentType() == null || !item.getContentType().contains("charset")) {
// old yacy clients use their local default charset, on most systems UTF-8 (I hope ;)
args.add(item.getFieldName(), item.getString("UTF-8"));
} else {
// use default encoding (given as header or ISO-8859-1)
args.add(item.getFieldName(), item.getString());
}
} else {
// file
args.add(item.getFieldName(), item.getName());
fileContent = FileUtils.read(item.getInputStream(), (int) item.getSize());
item.getInputStream().close();
files.put(item.getFieldName(), fileContent);
}
}
header.put("ARGC", String.valueOf(items.size())); // store argument count
body.close();
return files;
}
/**
* prepares the body so that it can be read as whole plain text
* (uncompress if necessary and ensure correct ending)
*
* @param header
* @param in
* @return
* @throws IOException
*/
private static InputStream prepareBody(final RequestHeader header, final InputStream in) throws IOException {
InputStream body = in;
// data may be compressed
final String bodyEncoding = header.get(HeaderFramework.CONTENT_ENCODING);
if(HeaderFramework.CONTENT_ENCODING_GZIP.equalsIgnoreCase(bodyEncoding) && !(body instanceof GZIPInputStream)) {
body = new GZIPInputStream(body);
// length of uncompressed data is unknown
header.remove(HeaderFramework.CONTENT_LENGTH);
} else {
// ensure the end of data (if client keeps alive the connection)
final long clength = header.getContentLength();
if (clength > 0) {
body = new ContentLengthInputStream(body, clength);
}
}
return body;
}
/**
* wraps the request into a org.apache.commons.fileupload.RequestContext
*
* @author danielr
* @since 07.08.2008
*/
private static class yacyContextRequest extends RequestHeader implements RequestContext {
private static final long serialVersionUID = -8936741958551376593L;
private final InputStream inStream;
/**
* creates a new yacyContextRequest
*
* @param header
* @param in
*/
public yacyContextRequest(final Map<String, String> requestHeader, final InputStream in) {
super(null, requestHeader);
this.inStream = in;
}
/*
* (non-Javadoc)
*
* @see org.apache.commons.fileupload.RequestContext#getInputStream()
*/
// @Override
@Override
public InputStream getInputStream() throws IOException {
return this.inStream;
}
}
static final void sendRespondError(
final HashMap<String, Object> conProp,
final OutputStream respond,

Write Preview
Loading…
Cancel
Save