boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Fixed gzip decompression regression on index transfer APIs

Browse Source 
Processing of gzip encoded incoming requests (on /yacy/transferRWI.html
and /yacy/transferURL.html) was no more working since upgrade to Jetty
9.4.12 (see commit 51f4be1).

To prevent any conflicting behavior with Jetty internals, use now the
GzipHandler provided by Jetty to decompress incoming gzip encoded
requests rather than the previously used custom GZIPRequestWrapper.

Fixes issue #249
pull/258/head
luccioman 6 years ago
parent e85f231bdf
commit a997133260
2 changed files with 31 additions and 164 deletions
Show Stats Download Patch File Download Diff File

21
source/net/yacy/http/Jetty9HttpServerImpl.java
Unescape View File

@ -34,6 +34,7 @@ import java.util.StringTokenizer;
import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext; import javax.net.ssl.SSLContext;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Handler;
@ -144,18 +145,20 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
//sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html
htrootContext.addServlet(sholder, "/*"); htrootContext.addServlet(sholder, "/*");
/* Handle gzip compression of responses to user agents accepting it */ final GzipHandler gzipHandler = new GzipHandler();
final GzipHandler gzipHandler;
if (sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) {
gzipHandler = new GzipHandler();
/* /*
* Ensure decompression of requests body is disabled : it is already handled by * Decompression of incoming requests body is required for index distribution
* the GZIPRequestWrapper in the YaCyDefaultServlet * APIs /yacy/transferRWI.html and /yacy/transferURL.html This was previously
* handled by a GZIPRequestWrapper in the YaCyDefaultServlet.
*/ */
gzipHandler.setInflateBufferSize(0); gzipHandler.setInflateBufferSize(4096);
htrootContext.setGzipHandler(gzipHandler);
if (!sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) {
/* Gzip compression of responses can be disabled by user configuration */
gzipHandler.setExcludedMethods(HttpMethod.GET.asString(), HttpMethod.POST.asString());
} }
htrootContext.setGzipHandler(gzipHandler);
// ----------------------------------------------------------------------------- // -----------------------------------------------------------------------------
// here we set and map the mandatory servlets, needed for typical YaCy operation // here we set and map the mandatory servlets, needed for typical YaCy operation

170
source/net/yacy/http/servlets/YaCyDefaultServlet.java
Unescape View File

@ -42,25 +42,33 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
import javax.servlet.ReadListener;
import javax.servlet.RequestDispatcher; import javax.servlet.RequestDispatcher;
import javax.servlet.ServletContext; import javax.servlet.ServletContext;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.UnavailableException; import javax.servlet.UnavailableException;
import javax.servlet.http.Cookie; import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.io.WriterOutputStream;
import org.eclipse.jetty.server.InclusiveByteRange;
import org.eclipse.jetty.util.MultiPartOutputStream;
import org.eclipse.jetty.util.URIUtil;
import org.eclipse.jetty.util.resource.Resource;
import com.google.common.net.HttpHeaders;
import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Base64Order;
@ -75,38 +83,19 @@ import net.yacy.data.InvalidURLLicenceException;
import net.yacy.data.TransactionManager; import net.yacy.data.TransactionManager;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.NamePrefixThreadFactory;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.peers.graphics.EncodedImage; import net.yacy.peers.graphics.EncodedImage;
import net.yacy.peers.operation.yacyBuildProperties; import net.yacy.peers.operation.yacyBuildProperties;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.server.http.TemplateEngine;
import net.yacy.server.serverClassLoader; import net.yacy.server.serverClassLoader;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
import net.yacy.server.servletProperties; import net.yacy.server.servletProperties;
import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.server.http.TemplateEngine;
import net.yacy.visualization.RasterPlotter; import net.yacy.visualization.RasterPlotter;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.io.WriterOutputStream;
import org.eclipse.jetty.server.InclusiveByteRange;
import org.eclipse.jetty.util.MultiPartOutputStream;
import org.eclipse.jetty.util.URIUtil;
import org.eclipse.jetty.util.resource.Resource;
import com.google.common.net.HttpHeaders;
import com.google.common.util.concurrent.SimpleTimeLimiter;
import com.google.common.util.concurrent.TimeLimiter;
import com.google.common.util.concurrent.UncheckedTimeoutException;
/** /**
* YaCyDefaultServlet based on Jetty DefaultServlet.java * YaCyDefaultServlet based on Jetty DefaultServlet.java
* handles static files and the YaCy servlets. * handles static files and the YaCy servlets.
@ -152,8 +141,6 @@ public class YaCyDefaultServlet extends HttpServlet {
protected static final File TMPDIR = new File(System.getProperty("java.io.tmpdir")); protected static final File TMPDIR = new File(System.getProperty("java.io.tmpdir"));
protected static final int SIZE_FILE_THRESHOLD = 1024 * 1024 * 1024; // 1GB is a lot but appropriate for multi-document pushed using the push_p.json servlet protected static final int SIZE_FILE_THRESHOLD = 1024 * 1024 * 1024; // 1GB is a lot but appropriate for multi-document pushed using the push_p.json servlet
protected static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(SIZE_FILE_THRESHOLD, TMPDIR); protected static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(SIZE_FILE_THRESHOLD, TMPDIR);
private final static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newCachedThreadPool(
new NamePrefixThreadFactory(YaCyDefaultServlet.class.getSimpleName() + ".timeLimiter")));
/* ------------------------------------------------------------ */ /* ------------------------------------------------------------ */
@Override @Override
public void init() throws UnavailableException { public void init() throws UnavailableException {
@ -866,13 +853,8 @@ public class YaCyDefaultServlet extends HttpServlet {
RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt); RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
// add multipart-form fields to parameter // add multipart-form fields to parameter
if (ServletFileUpload.isMultipartContent(request)) { if (ServletFileUpload.isMultipartContent(request)) {
final String bodyEncoding = request.getHeader(HeaderFramework.CONTENT_ENCODING);
if (HeaderFramework.CONTENT_ENCODING_GZIP.equalsIgnoreCase(bodyEncoding)) {
parseMultipart(new GZIPRequestWrapper(request),args);
} else {
parseMultipart(request, args); parseMultipart(request, args);
} }
}
// eof modification to read attribute // eof modification to read attribute
Object tmp; Object tmp;
try { try {
@ -1336,122 +1318,4 @@ public class YaCyDefaultServlet extends HttpServlet {
ConcurrentLog.info("FILEHANDLER", ex.getMessage()); ConcurrentLog.info("FILEHANDLER", ex.getMessage());
} }
} }
/**
* wraps request to uncompress gzip'ed input stream
*/
private class GZIPRequestWrapper extends HttpServletRequestWrapper {
private final ServletInputStream is;
public GZIPRequestWrapper(HttpServletRequest request) throws IOException {
super(request);
this.is = new GZIPRequestStream(request);
}
@Override
public ServletInputStream getInputStream() throws IOException {
return is;
}
}
private class GZIPRequestStream extends ServletInputStream {
private final GZIPInputStream in;
private final ServletInputStream sin;
public GZIPRequestStream(HttpServletRequest request) throws IOException {
sin = request.getInputStream();
in = new GZIPInputStream(sin);
}
@Override
public int read() throws IOException {
return in.read();
}
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
try {
return timeLimiter.callWithTimeout(new CallableReader(in, b, off, len), len + 600, TimeUnit.MILLISECONDS, false);
} catch (final UncheckedTimeoutException e) {
return -1;
} catch (Exception e) {
throw new IOException(e);
}
}
@Override
public void close() throws IOException {
in.close();
}
@Override
public int available() throws IOException {
return in.available();
}
@Override
public synchronized void mark(int readlimit) {
in.mark(readlimit);
}
@Override
public boolean markSupported() {
return in.markSupported();
}
@Override
public synchronized void reset() throws IOException {
in.reset();
}
@Override
public long skip(long n) throws IOException {
return in.skip(n);
}
@Override
public boolean isFinished() {
try {
return available() < 1;
} catch (final IOException ex) {
return true;
}
}
@Override
public boolean isReady() {
return sin.isReady() && !isFinished();
}
@Override
public void setReadListener(ReadListener rl) {
sin.setReadListener(rl);
}
}
private class CallableReader implements Callable<Integer> {
private int off, len;
private byte[] b;
private GZIPInputStream in;
public CallableReader(final GZIPInputStream in, byte[] b, int off, int len) {
this.in = in;
this.b = b;
this.off = off;
this.len = len;
}
@Override
public Integer call() throws Exception {
return in.read(b, off, len);
}
}
} }

Write Preview
Loading…
Cancel
Save