diff --git a/build.xml b/build.xml index deaf9241f..56518830c 100644 --- a/build.xml +++ b/build.xml @@ -161,13 +161,15 @@ - - - - - - - + + + + + + + + + diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java index efa74a8be..db0394c11 100644 --- a/htroot/ConfigBasic.java +++ b/htroot/ConfigBasic.java @@ -38,12 +38,11 @@ import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.data.Translator; import net.yacy.data.WorkTables; -import net.yacy.http.HttpServer; +import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.workflow.InstantBusyThread; import net.yacy.peers.Seed; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; -import net.yacy.server.serverCore; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.server.http.HTTPDFileHandler; @@ -132,7 +131,7 @@ public class ConfigBasic { final boolean reconnect; if (!(env.getConfigLong("port", port) == port) || env.getConfigBool("server.https", false) != ssl) { // validate port - final HttpServer theServerCore = env.getHttpServer(); + final YaCyHttpServer theServerCore = env.getHttpServer(); env.setConfig("port", port); env.setConfig("server.https", ssl); diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index a35ec8921..e78f501c3 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -31,7 +31,7 @@ import java.util.Map; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.http.HttpServer; +import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.rwi.IndexCell; import net.yacy.kelondro.util.FileUtils; @@ -39,11 +39,9 @@ import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.OS; import net.yacy.kelondro.workflow.BusyThread; -import net.yacy.kelondro.workflow.WorkflowThread; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; -import net.yacy.server.serverCore; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -267,7 +265,7 @@ public class PerformanceQueues_p { /* * configuring the http pool */ - final HttpServer httpd = sb.getHttpServer(); + final YaCyHttpServer httpd = sb.getHttpServer(); try { maxBusy = post.getInt("httpd Session Pool_maxActive", 8); } catch (final NumberFormatException e) { @@ -311,7 +309,7 @@ public class PerformanceQueues_p { prop.put("pool_0_maxActive", sb.getConfigLong("crawler.MaxActiveThreads", 0)); prop.put("pool_0_numActive",sb.crawlQueues.workerSize()); - final HttpServer httpd = sb.getHttpServer(); + final YaCyHttpServer httpd = sb.getHttpServer(); prop.put("pool_1_name", "httpd Session Pool"); prop.put("pool_1_maxActive", httpd.getMaxSessionCount()); prop.put("pool_1_numActive", httpd.getJobCount()); diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index ae5b02743..218df8b32 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -40,7 +40,7 @@ import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.http.HttpServer; +import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.util.Formatter; import net.yacy.peers.Network; import net.yacy.peers.Seed; @@ -109,7 +109,7 @@ public class SettingsAck_p { prop.putHTML("info_port", port); if (!env.getConfig("port", port).equals(port)) { // validation port - final HttpServer theServerCore = env.getHttpServer(); + final YaCyHttpServer theServerCore = env.getHttpServer(); try { final InetSocketAddress theNewAddress = theServerCore.generateSocketAddress(port); final String hostName = Domains.getHostName(theNewAddress.getAddress()); diff --git a/htroot/Status.java b/htroot/Status.java index faf84d0f1..3caa37dfc 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -33,7 +33,7 @@ import java.util.Date; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.Memory; -import net.yacy.http.HttpServer; +import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.io.ByteCount; import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.MemoryControl; @@ -327,7 +327,7 @@ public class Status prop.put("trafficCrawler", Formatter.bytesToString(ByteCount.getAccountCount(ByteCount.CRAWLER))); // connection information - final HttpServer httpd = sb.getHttpServer(); + final YaCyHttpServer httpd = sb.getHttpServer(); prop.putNum("connectionsActive", httpd.getJobCount()); prop.putNum("connectionsMax", httpd.getMaxSessionCount()); diff --git a/lib/jetty-continuation-8.1.13.v20130916.jar b/lib/jetty-continuation-8.1.13.v20130916.jar new file mode 100644 index 000000000..7d9990f75 Binary files /dev/null and b/lib/jetty-continuation-8.1.13.v20130916.jar differ diff --git a/lib/jetty-http-8.1.13.v20130916.jar b/lib/jetty-http-8.1.13.v20130916.jar new file mode 100644 index 000000000..e12cfe031 Binary files /dev/null and b/lib/jetty-http-8.1.13.v20130916.jar differ diff --git a/lib/jetty-io-8.1.13.v20130916.jar b/lib/jetty-io-8.1.13.v20130916.jar new file mode 100644 index 000000000..51e527b15 Binary files /dev/null and b/lib/jetty-io-8.1.13.v20130916.jar differ diff --git a/lib/jetty-security-8.1.13.v20130916.jar b/lib/jetty-security-8.1.13.v20130916.jar new file mode 100644 index 000000000..d1d7c2737 Binary files /dev/null and b/lib/jetty-security-8.1.13.v20130916.jar differ diff --git a/lib/jetty-server-8.1.13.v20130916.jar b/lib/jetty-server-8.1.13.v20130916.jar new file mode 100644 index 000000000..d2273bb16 Binary files /dev/null and b/lib/jetty-server-8.1.13.v20130916.jar differ diff --git a/lib/jetty-servlet-8.1.13.v20130916.jar b/lib/jetty-servlet-8.1.13.v20130916.jar new file mode 100644 index 000000000..4346d768c Binary files /dev/null and b/lib/jetty-servlet-8.1.13.v20130916.jar differ diff --git a/lib/jetty-servlets-8.1.13.v20130916.jar b/lib/jetty-servlets-8.1.13.v20130916.jar new file mode 100644 index 000000000..44e30dcb9 Binary files /dev/null and b/lib/jetty-servlets-8.1.13.v20130916.jar differ diff --git a/lib/jetty-util-8.1.13.v20130916.jar b/lib/jetty-util-8.1.13.v20130916.jar new file mode 100644 index 000000000..86e4d881c Binary files /dev/null and b/lib/jetty-util-8.1.13.v20130916.jar differ diff --git a/lib/jetty-continuation-9.0.5.v20130815.jar b/libt/jetty-continuation-9.0.5.v20130815.jar similarity index 100% rename from lib/jetty-continuation-9.0.5.v20130815.jar rename to libt/jetty-continuation-9.0.5.v20130815.jar diff --git a/lib/jetty-http-9.0.5.v20130815.jar b/libt/jetty-http-9.0.5.v20130815.jar similarity index 100% rename from lib/jetty-http-9.0.5.v20130815.jar rename to libt/jetty-http-9.0.5.v20130815.jar diff --git a/lib/jetty-io-9.0.5.v20130815.jar b/libt/jetty-io-9.0.5.v20130815.jar similarity index 100% rename from lib/jetty-io-9.0.5.v20130815.jar rename to libt/jetty-io-9.0.5.v20130815.jar diff --git a/lib/jetty-security-9.0.5.v20130815.jar b/libt/jetty-security-9.0.5.v20130815.jar similarity index 100% rename from lib/jetty-security-9.0.5.v20130815.jar rename to libt/jetty-security-9.0.5.v20130815.jar diff --git a/lib/jetty-server-9.0.5.v20130815.jar b/libt/jetty-server-9.0.5.v20130815.jar similarity index 100% rename from lib/jetty-server-9.0.5.v20130815.jar rename to libt/jetty-server-9.0.5.v20130815.jar diff --git a/lib/jetty-servlet-9.0.5.v20130815.jar b/libt/jetty-servlet-9.0.5.v20130815.jar similarity index 100% rename from lib/jetty-servlet-9.0.5.v20130815.jar rename to libt/jetty-servlet-9.0.5.v20130815.jar diff --git a/lib/jetty-servlets-9.0.5.v20130815.jar b/libt/jetty-servlets-9.0.5.v20130815.jar similarity index 100% rename from lib/jetty-servlets-9.0.5.v20130815.jar rename to libt/jetty-servlets-9.0.5.v20130815.jar diff --git a/lib/jetty-util-9.0.5.v20130815.jar b/libt/jetty-util-9.0.5.v20130815.jar similarity index 100% rename from lib/jetty-util-9.0.5.v20130815.jar rename to libt/jetty-util-9.0.5.v20130815.jar diff --git a/source/net/yacy/http/Jetty8HttpServerImpl.java b/source/net/yacy/http/Jetty8HttpServerImpl.java new file mode 100644 index 000000000..16131e239 --- /dev/null +++ b/source/net/yacy/http/Jetty8HttpServerImpl.java @@ -0,0 +1,190 @@ +// +// Jetty8HttpServerImpl +// Copyright 2011 by Florian Richter +// First released 13.04.2011 at http://yacy.net +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program in the file lgpl21.txt +// If not, see . +// + +package net.yacy.http; + +import java.net.InetSocketAddress; +import java.net.SocketException; +import java.util.EnumSet; +import javax.servlet.DispatcherType; +import net.yacy.cora.federate.solr.SolrServlet; +import net.yacy.cora.federate.solr.SolrServlet.Servlet404; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.search.Switchboard; + +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.ContextHandlerCollection; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.nio.SelectChannelConnector; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +/** + * class to embedded Jetty 8 http server into YaCy + */ +public class Jetty8HttpServerImpl implements YaCyHttpServer { + + private Server server; + + /** + * @param port TCP Port to listen for http requests + */ + public Jetty8HttpServerImpl(int port) { + Switchboard sb = Switchboard.getSwitchboard(); + + server = new Server(); + SelectChannelConnector connector = new SelectChannelConnector(); + connector.setPort(port); + connector.setName("httpd:"+Integer.toString(port)); + //connector.setThreadPool(new QueuedThreadPool(20)); + server.addConnector(connector); + + YacyDomainHandler domainHandler = new YacyDomainHandler(); + domainHandler.setAlternativeResolver(sb.peers); + + /* this is now handled by YaCyDefaultServlet + ResourceHandler resource_handler = new ResourceHandler(); + resource_handler.setDirectoriesListed(true); + resource_handler.setWelcomeFiles(new String[]{"index.html"}); + resource_handler.setResourceBase("htroot/"); + */ + + //add SolrServlet + ServletContextHandler solrContext = new ServletContextHandler(ServletContextHandler.SESSIONS); + solrContext.setContextPath("/solr"); + solrContext.addServlet(new ServletHolder(Servlet404.class),"/*"); + + SolrServlet.initCore(sb.index.fulltext().getDefaultEmbeddedConnector()); + solrContext.addFilter(new FilterHolder(SolrServlet.class), "/*", EnumSet.of(DispatcherType.REQUEST)); + + // configure root context + ServletContextHandler htrootContext = new ServletContextHandler(ServletContextHandler.SESSIONS); + htrootContext.setContextPath("/"); + ServletHolder sholder = new ServletHolder(Jetty8YaCyDefaultServlet.class); + sholder.setInitParameter("resourceBase", "htroot"); + //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html + sholder.setInitParameter("gzip","false"); + htrootContext.addServlet(sholder,"/*"); + + // assemble the servlet handlers + ContextHandlerCollection servletContext = new ContextHandlerCollection(); + servletContext.setHandlers(new Handler[] { solrContext, htrootContext }); + + // define list of YaCy specific general handlers + HandlerList handlers = new HandlerList(); + handlers.setHandlers(new Handler[] + {domainHandler, new ProxyCacheHandler(), new ProxyHandler() + /*, resource_handler, new DefaultHandler() */}); + + // context handler for dispatcher and security (hint: dispatcher requires a context) + ContextHandler context = new ContextHandler(); + context.setContextPath("/"); + context.setHandler(handlers); + + // make YaCy handlers (in context) and servlet context handlers available (both contain root context "/") + // logic: 1. YaCy handlers are called if request not handled (e.g. proxy) then servlets handle it + ContextHandlerCollection allrequesthandlers = new ContextHandlerCollection(); + allrequesthandlers.addHandler(context); + allrequesthandlers.addHandler(servletContext); + allrequesthandlers.addHandler(new DefaultHandler()); // if not handled by other handler + + // wrap all handlers by security handler + Jetty8YaCySecurityHandler securityHandler = new Jetty8YaCySecurityHandler(); + securityHandler.setLoginService(new YaCyLoginService()); + securityHandler.setRealmName("YaCy Admin Interface"); + securityHandler.setHandler(new CrashProtectionHandler(allrequesthandlers)); + + server.setHandler(securityHandler); + } + + /** + * start http server + */ + @Override + public void startupServer() throws Exception { + server.start(); + } + + /** + * stop http server and wait for it + */ + @Override + public void stop() throws Exception { + server.stop(); + server.join(); + } + + @Override + public void setMaxSessionCount(int maxBusy) { + // TODO: + } + + @Override + public boolean withSSL() { + return false; // TODO: + } + + @Override + public void reconnect(int milsec) { + try { + Thread.sleep(milsec); + } catch (final InterruptedException e) { + ConcurrentLog.logException(e); + } catch (final Exception e) { + ConcurrentLog.logException(e); + } + try { + server.stop(); + server.join(); + server.start(); + } catch (Exception ex) { + ConcurrentLog.logException(ex); + } + } + + @Override + public InetSocketAddress generateSocketAddress(String port) throws SocketException { + return null; // TODO: + } + + @Override + public int getMaxSessionCount() { + return server.getThreadPool().getThreads(); + } + + @Override + public int getJobCount() { + return getMaxSessionCount() - server.getThreadPool().getIdleThreads(); // TODO: + } + + @Override + public String getVersion() { + return "Jetty " + server.getVersion(); + } + +} diff --git a/source/net/yacy/http/Jetty8YaCyDefaultServlet.java b/source/net/yacy/http/Jetty8YaCyDefaultServlet.java new file mode 100644 index 000000000..b5a9c5041 --- /dev/null +++ b/source/net/yacy/http/Jetty8YaCyDefaultServlet.java @@ -0,0 +1,691 @@ +// +// ======================================================================== +// Copyright (c) 1995-2013 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// +package net.yacy.http; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.util.Enumeration; +import java.util.List; + +import javax.servlet.RequestDispatcher; +import javax.servlet.ServletException; +import javax.servlet.UnavailableException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.kelondro.util.FileUtils; + +import org.eclipse.jetty.http.HttpContent; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeaderValues; +import org.eclipse.jetty.http.HttpHeaders; +import org.eclipse.jetty.http.HttpMethods; +import org.eclipse.jetty.io.Buffer; +import org.eclipse.jetty.io.WriterOutputStream; +import org.eclipse.jetty.server.AbstractHttpConnection; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Dispatcher; +import org.eclipse.jetty.server.HttpOutput; +import org.eclipse.jetty.server.InclusiveByteRange; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.nio.NIOConnector; +import org.eclipse.jetty.server.ssl.SslConnector; +import org.eclipse.jetty.util.MultiPartOutputStream; +import org.eclipse.jetty.util.QuotedStringTokenizer; +import org.eclipse.jetty.util.URIUtil; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; + +/* ------------------------------------------------------------ */ +/** + * YaCyDefaultServlet base on Jetty DefaultServlet.java + * handles static files and the YaCy servlets. + */ + +/** + * The default servlet. This servlet, normally mapped to /, provides the + * handling for static content, OPTION and TRACE methods for the context. The + * following initParameters are supported, these can be set either on the + * servlet itself or as ServletContext initParameters : + * 
+ *  acceptRanges      If true, range requests and responses are
+ *                    supported
+ *
+ *  dirAllowed        If true, directory listings are returned if no
+ *                    welcome file is found. Else 403 Forbidden.
+ *
+ *  welcomeFile       name of the welcome file (default is "index.html", "welcome.html")
+ *
+ *  gzip              If set to true, then static content will be served as
+ *                    gzip content encoded if a matching resource is
+ *                    found ending with ".gz"
+ *
+ *  resourceBase      Set to replace the context resource base
+ *
+ *  resourceCache     If set, this is a context attribute name, which the servlet
+ *                    will use to look for a shared ResourceCache instance.
+ *
+ *  relativeResourceBase
+ *                    Set with a pathname relative to the base of the
+ *                    servlet context root. Useful for only serving static content out
+ *                    of only specific subdirectories.
+ *
+ *  pathInfoOnly      If true, only the path info will be applied to the resourceBase
+ *
+ *
+ *  etags             If True, weak etags will be generated and handled.
+ *
+ *
+ * + * + * + * + */ +public class Jetty8YaCyDefaultServlet extends YaCyDefaultServlet implements ResourceFactory { + + private static final long serialVersionUID = 4900000000000001110L; + + private boolean _gzip = true; + + /* ------------------------------------------------------------ */ + @Override + public void init() throws UnavailableException { + super.init(); + + _gzip=getInitBoolean("gzip",_gzip); + + } + + /* ------------------------------------------------------------ */ + /** + * get Resource to serve. Map a path to a resource. The default + * implementation calls HttpContext.getResource but derived servlets may + * provide their own mapping. + * + * @param pathInContext The path to find a resource for. + * @return The resource to serve. + */ + @Override + public Resource getResource(String pathInContext) { + Resource r = null; + if (_relativeResourceBase != null) { + pathInContext = URIUtil.addPaths(_relativeResourceBase, pathInContext); + } + + try { + if (_resourceBase != null) { + r = _resourceBase.addPath(pathInContext); + } else { + URL u = _servletContext.getResource(pathInContext); + r = Resource.newResource(u); + } + + if (ConcurrentLog.isFine("YaCyDefaultServlet")) { + ConcurrentLog.fine("YaCyDefaultServlet","Resource " + pathInContext + "=" + r); + } + } catch (IOException e) { + // ConcurrentLog.logException(e); + } + + return r; + } + + /* ------------------------------------------------------------ */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String servletPath = null; + String pathInfo = null; + Enumeration reqRanges = null; + Boolean included = request.getAttribute(Dispatcher.INCLUDE_REQUEST_URI) != null; + if (included != null && included.booleanValue()) { + servletPath = (String) request.getAttribute(Dispatcher.INCLUDE_SERVLET_PATH); + pathInfo = (String) request.getAttribute(Dispatcher.INCLUDE_PATH_INFO); + if (servletPath == null) { + servletPath = request.getServletPath(); + pathInfo = request.getPathInfo(); + } + } else { + included = Boolean.FALSE; + servletPath = _pathInfoOnly ? "/" : request.getServletPath(); + pathInfo = request.getPathInfo(); + + // Is this a Range request? + reqRanges = request.getHeaders(HeaderFramework.RANGE); + if (!hasDefinedRange(reqRanges)) { + reqRanges = null; + } + } + + if (pathInfo.startsWith("/currentyacypeer/")) pathInfo = pathInfo.substring(16); + String pathInContext = URIUtil.addPaths(servletPath, pathInfo); + boolean endsWithSlash = (pathInfo == null ? request.getServletPath() : pathInfo).endsWith(URIUtil.SLASH); + + // Find the resource and content + Resource resource = null; + HttpContent content = null; + try { + // is gzip enabled? + String pathInContextGz = null; + boolean gzip = false; + if (!included.booleanValue() && _gzip && reqRanges == null && !endsWithSlash) { + // Look for a gzip resource + pathInContextGz = pathInContext + ".gz"; + resource = getResource(pathInContextGz); + + // Does a gzip resource exist? + if (resource != null && resource.exists() && !resource.isDirectory()) { + // Tell caches that response may vary by accept-encoding + response.addHeader(HttpHeaders.VARY, HttpHeaders.ACCEPT_ENCODING); + + // Does the client accept gzip? + String accept = request.getHeader(HttpHeaders.ACCEPT_ENCODING); + if (accept != null && accept.indexOf("gzip") >= 0) { + gzip = true; + } + } + } + + // find resource + if (!gzip) resource = getResource(pathInContext); + + // Look for a class resource + boolean hasClass = false; + if (reqRanges == null && !endsWithSlash) { + final int p = pathInContext.lastIndexOf('.'); + if (p >= 0) { + String pathofClass = pathInContext.substring(0, p) + ".class"; + Resource classresource = getResource(pathofClass); + // Does a class resource exist? + if (classresource != null && classresource.exists() && !classresource.isDirectory()) { + hasClass = true; + } + } + } + + if (ConcurrentLog.isFine("YaCyDefaultServlet")) { + ConcurrentLog.fine("YaCyDefaultServlet","uri=" + request.getRequestURI() + " resource=" + resource + (content != null ? " content" : "")); + } + + // Handle resource + if (!hasClass && (resource == null || !resource.exists())) { + if (included) { + throw new FileNotFoundException("!" + pathInContext); + } + response.sendError(HttpServletResponse.SC_NOT_FOUND); + } else if (!resource.isDirectory()) { + if (endsWithSlash && pathInContext.length() > 1) { + String q = request.getQueryString(); + pathInContext = pathInContext.substring(0, pathInContext.length() - 1); + if (q != null && q.length() != 0) { + pathInContext += "?" + q; + } + response.sendRedirect(response.encodeRedirectURL(URIUtil.addPaths(_servletContext.getContextPath(), pathInContext))); + } else { + // ensure we have content + if (content == null) { + content = new HttpContent.ResourceAsHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), response.getBufferSize(), _etags); + } + + if (hasClass) { // this is a YaCy servlet, handle the template + handleTemplate(pathInfo, request, response); + } else { + if (included.booleanValue() || passConditionalHeaders(request, response, resource, content)) { + if (gzip) { + response.setHeader(HttpHeaders.CONTENT_ENCODING, "gzip"); + String mt = _servletContext.getMimeType(pathInContext); + if (mt != null) { + response.setContentType(mt); + } + } + sendData(request, response, included.booleanValue(), resource, content, reqRanges); + } + } + } + } else { + String welcome = null; + + if (!endsWithSlash || (pathInContext.length() == 1 && request.getAttribute("org.eclipse.jetty.server.nullPathInfo") != null)) { + StringBuffer buf = request.getRequestURL(); + synchronized (buf) { + int param = buf.lastIndexOf(";"); + if (param < 0) { + buf.append('/'); + } else { + buf.insert(param, '/'); + } + String q = request.getQueryString(); + if (q != null && q.length() != 0) { + buf.append('?'); + buf.append(q); + } + response.setContentLength(0); + response.sendRedirect(response.encodeRedirectURL(buf.toString())); + } + } // else look for a welcome file + else if (null != (welcome = getWelcomeFile(pathInContext))) { + ConcurrentLog.fine("welcome={}", welcome); + + + // Forward to the index + RequestDispatcher dispatcher = request.getRequestDispatcher(welcome); + if (dispatcher != null) { + if (included.booleanValue()) { + dispatcher.include(request, response); + } else { + request.setAttribute("org.eclipse.jetty.server.welcome", welcome); + dispatcher.forward(request, response); + } + } + + } else { + content = new HttpContent.ResourceAsHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), _etags); + if (included.booleanValue() || passConditionalHeaders(request, response, resource, content)) { + sendDirectory(request, response, resource, pathInContext); + } + } + } + } catch (IllegalArgumentException e) { + ConcurrentLog.logException(e); + if (!response.isCommitted()) { + response.sendError(500, e.getMessage()); + } + } finally { + if (content != null) { + content.release(); + } else if (resource != null) { + resource.release(); + } + } + } + + /* ------------------------------------------------------------ */ + @Override + protected boolean hasDefinedRange(Enumeration reqRanges) { + return (reqRanges != null && reqRanges.hasMoreElements()); + } + + /* ------------------------------------------------------------ */ + /* Check modification date headers. + */ + @Override + protected boolean passConditionalHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, HttpContent content) + throws IOException { + try { + if (!request.getMethod().equals(HttpMethods.HEAD)) { + if (_etags) { + String ifm = request.getHeader(HttpHeaders.IF_MATCH); + if (ifm != null) { + boolean match = false; + if (content != null && content.getETag() != null) { + QuotedStringTokenizer quoted = new QuotedStringTokenizer(ifm, ", ", false, true); + while (!match && quoted.hasMoreTokens()) { + String tag = quoted.nextToken(); + if (content.getETag().toString().equals(tag)) { + match = true; + } + } + } + + if (!match) { + Response r = Response.getResponse(response); + r.reset(true); + r.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED); + return false; + } + } + + String ifnm = request.getHeader(HttpHeaders.IF_NONE_MATCH); + if (ifnm != null && content != null && content.getETag() != null) { + // Look for GzipFiltered version of etag + if (content.getETag().toString().equals(request.getAttribute("o.e.j.s.GzipFilter.ETag"))) { + Response r = Response.getResponse(response); + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + r.getHttpFields().put(HttpHeaders.ETAG_BUFFER, ifnm); + return false; + } + + + // Handle special case of exact match. + if (content.getETag().toString().equals(ifnm)) { + Response r = Response.getResponse(response); + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + r.getHttpFields().put(HttpHeaders.ETAG_BUFFER, content.getETag()); + return false; + } + + // Handle list of tags + QuotedStringTokenizer quoted = new QuotedStringTokenizer(ifnm, ", ", false, true); + while (quoted.hasMoreTokens()) { + String tag = quoted.nextToken(); + if (content.getETag().toString().equals(tag)) { + Response r = Response.getResponse(response); + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + r.getHttpFields().put(HttpHeaders.ETAG_BUFFER, content.getETag()); + return false; + } + } + + return true; + } + } + + String ifms = request.getHeader(HttpHeaders.IF_MODIFIED_SINCE); + if (ifms != null) { + //Get jetty's Response impl + Response r = Response.getResponse(response); + + if (content != null) { + Buffer mdlm = content.getLastModified(); + if (mdlm != null) { + if (ifms.equals(mdlm.toString())) { + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + if (_etags) { + r.getHttpFields().add(HttpHeaders.ETAG_BUFFER, content.getETag()); + } + r.flushBuffer(); + return false; + } + } + } + + long ifmsl = request.getDateHeader(HttpHeaders.IF_MODIFIED_SINCE); + if (ifmsl != -1) { + if (resource.lastModified() / 1000 <= ifmsl / 1000) { + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + if (_etags) { + r.getHttpFields().add(HttpHeaders.ETAG_BUFFER, content.getETag()); + } + r.flushBuffer(); + return false; + } + } + } + + // Parse the if[un]modified dates and compare to resource + long date = request.getDateHeader(HttpHeaders.IF_UNMODIFIED_SINCE); + + if (date != -1) { + if (resource.lastModified() / 1000 > date / 1000) { + response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED); + return false; + } + } + + } + } catch (IllegalArgumentException iae) { + if (!response.isCommitted()) { + response.sendError(400, iae.getMessage()); + } + throw iae; + } + return true; + } + + /* ------------------------------------------------------------ */ + protected void sendData(HttpServletRequest request, + HttpServletResponse response, + boolean include, + Resource resource, + HttpContent content, + Enumeration reqRanges) + throws IOException { + boolean direct; + long content_length; + if (content == null) { + direct = false; + content_length = resource.length(); + } else { + Connector connector = AbstractHttpConnection.getCurrentConnection().getConnector(); + direct = connector instanceof NIOConnector && ((NIOConnector) connector).getUseDirectBuffers() && !(connector instanceof SslConnector); + content_length = content.getContentLength(); + } + + + // Get the output stream (or writer) + OutputStream out = null; + boolean written; + try { + out = response.getOutputStream(); + + // has a filter already written to the response? + written = out instanceof HttpOutput + ? ((HttpOutput) out).isWritten() + : AbstractHttpConnection.getCurrentConnection().getGenerator().isWritten(); + } catch (IllegalStateException e) { + out = new WriterOutputStream(response.getWriter()); + written = true; // there may be data in writer buffer, so assume written + } + + if (reqRanges == null || !reqRanges.hasMoreElements() || content_length < 0) { + // if there were no ranges, send entire entity + if (include) { + resource.writeTo(out, 0, content_length); + } else { + // See if a direct methods can be used? + if (content != null && !written && out instanceof HttpOutput) { + if (response instanceof Response) { + writeOptionHeaders(((Response) response).getHttpFields()); + ((AbstractHttpConnection.Output) out).sendContent(content); + } else { + Buffer buffer = direct ? content.getDirectBuffer() : content.getIndirectBuffer(); + if (buffer != null) { + writeHeaders(response, content, content_length); + ((AbstractHttpConnection.Output) out).sendContent(buffer); + } else { + writeHeaders(response, content, content_length); + resource.writeTo(out, 0, content_length); + } + } + } else { + // Write headers normally + writeHeaders(response, content, written ? -1 : content_length); + + // Write content normally + Buffer buffer = (content == null) ? null : content.getIndirectBuffer(); + if (buffer != null) { + buffer.writeTo(out); + } else { + resource.writeTo(out, 0, content_length); + } + } + } + } else { + // Parse the satisfiable ranges + List ranges = InclusiveByteRange.satisfiableRanges(reqRanges, content_length); + + // if there are no satisfiable ranges, send 416 response + if (ranges == null || ranges.size() == 0) { + writeHeaders(response, content, content_length); + response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); + response.setHeader(HttpHeaders.CONTENT_RANGE, + InclusiveByteRange.to416HeaderRangeString(content_length)); + resource.writeTo(out, 0, content_length); + return; + } + + // if there is only a single valid range (must be satisfiable + // since were here now), send that range with a 216 response + if (ranges.size() == 1) { + InclusiveByteRange singleSatisfiableRange = + (InclusiveByteRange) ranges.get(0); + long singleLength = singleSatisfiableRange.getSize(content_length); + writeHeaders(response, content, singleLength); + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader(HttpHeaders.CONTENT_RANGE, + singleSatisfiableRange.toHeaderRangeString(content_length)); + resource.writeTo(out, singleSatisfiableRange.getFirst(content_length), singleLength); + return; + } + + // multiple non-overlapping valid ranges cause a multipart + // 216 response which does not require an overall + // content-length header + // + writeHeaders(response, content, -1); + String mimetype = (content.getContentType() == null ? null : content.getContentType().toString()); + if (mimetype == null) { + ConcurrentLog.warn("YaCyDefaultServlet", "Unknown mimetype for " + request.getRequestURI()); + } + MultiPartOutputStream multi = new MultiPartOutputStream(out); + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + + // If the request has a "Request-Range" header then we need to + // send an old style multipart/x-byteranges Content-Type. This + // keeps Netscape and acrobat happy. This is what Apache does. + String ctp; + if (request.getHeader(HttpHeaders.REQUEST_RANGE) != null) { + ctp = "multipart/x-byteranges; boundary="; + } else { + ctp = "multipart/byteranges; boundary="; + } + response.setContentType(ctp + multi.getBoundary()); + + InputStream in = resource.getInputStream(); + long pos = 0; + + // calculate the content-length + int length = 0; + String[] header = new String[ranges.size()]; + for (int i = 0; i < ranges.size(); i++) { + InclusiveByteRange ibr = (InclusiveByteRange) ranges.get(i); + header[i] = ibr.toHeaderRangeString(content_length); + length += + ((i > 0) ? 2 : 0) + + 2 + multi.getBoundary().length() + 2 + + (mimetype == null ? 0 : HeaderFramework.CONTENT_TYPE.length() + 2 + mimetype.length()) + 2 + + HeaderFramework.CONTENT_RANGE.length() + 2 + header[i].length() + 2 + + 2 + + (ibr.getLast(content_length) - ibr.getFirst(content_length)) + 1; + } + length += 2 + 2 + multi.getBoundary().length() + 2 + 2; + response.setContentLength(length); + + for (int i = 0; i < ranges.size(); i++) { + InclusiveByteRange ibr = (InclusiveByteRange) ranges.get(i); + multi.startPart(mimetype, new String[]{HeaderFramework.CONTENT_RANGE + ": " + header[i]}); + + long start = ibr.getFirst(content_length); + long size = ibr.getSize(content_length); + if (in != null) { + // Handle non cached resource + if (start < pos) { + in.close(); + in = resource.getInputStream(); + pos = 0; + } + if (pos < start) { + in.skip(start - pos); + pos = start; + } + + FileUtils.copy(in, multi, size); + pos += size; + } else // Handle cached resource + { + (resource).writeTo(multi, start, size); + } + + } + if (in != null) { + in.close(); + } + multi.close(); + } + return; + } + + /* ------------------------------------------------------------ */ + @Override + protected void writeHeaders(HttpServletResponse response, HttpContent content, long count) { + if (content.getContentType() != null && response.getContentType() == null) { + response.setContentType(content.getContentType().toString()); + } + + if (response instanceof Response) { + Response r = (Response) response; + HttpFields fields = r.getHttpFields(); + + if (content.getLastModified() != null) { + fields.put(HttpHeaders.LAST_MODIFIED_BUFFER, content.getLastModified()); + } else if (content.getResource() != null) { + long lml = content.getResource().lastModified(); + if (lml != -1) { + fields.putDateField(HttpHeaders.LAST_MODIFIED_BUFFER, lml); + } + } + + if (count != -1) { + r.setLongContentLength(count); + } + + writeOptionHeaders(fields); + + if (_etags) { + fields.put(HttpHeaders.ETAG_BUFFER, content.getETag()); + } + } else { + long lml = content.getResource().lastModified(); + if (lml >= 0) { + response.setDateHeader(HeaderFramework.LAST_MODIFIED, lml); + } + + if (count != -1) { + if (count < Integer.MAX_VALUE) { + response.setContentLength((int) count); + } else { + response.setHeader(HeaderFramework.CONTENT_LENGTH, Long.toString(count)); + } + } + + writeOptionHeaders(response); + + if (_etags) { + response.setHeader(HeaderFramework.ETAG, content.getETag().toString()); + } + } + } + + /* ------------------------------------------------------------ */ + @Override + protected void writeOptionHeaders(HttpFields fields) { + if (_acceptRanges) { + fields.put(HttpHeaders.ACCEPT_RANGES_BUFFER, HttpHeaderValues.BYTES_BUFFER); + } + + } + + /* ------------------------------------------------------------ */ + /* + * @see javax.servlet.Servlet#destroy() + */ + @Override + public void destroy() { + + super.destroy(); + } +} diff --git a/source/net/yacy/http/Jetty8YaCySecurityHandler.java b/source/net/yacy/http/Jetty8YaCySecurityHandler.java new file mode 100644 index 000000000..1261342ec --- /dev/null +++ b/source/net/yacy/http/Jetty8YaCySecurityHandler.java @@ -0,0 +1,200 @@ +// +// YaCySecurityHandler +// Copyright 2011 by Florian Richter +// First released 16.04.2011 at http://yacy.net +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program in the file lgpl21.txt +// If not, see . +// + +package net.yacy.http; + +import java.io.IOException; +import java.net.MalformedURLException; +import net.yacy.cora.document.id.MultiProtocolURL; + +import net.yacy.cora.protocol.Domains; +import net.yacy.search.Switchboard; +import org.eclipse.jetty.http.HttpSchemes; +import org.eclipse.jetty.security.RoleInfo; + +import org.eclipse.jetty.security.SecurityHandler; +import org.eclipse.jetty.security.UserDataConstraint; +import org.eclipse.jetty.server.AbstractHttpConnection; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.UserIdentity; + +/** + * jetty security handler + * demands authentication for pages with _p. inside + * and updates AccessTracker + */ +public class Jetty8YaCySecurityHandler extends SecurityHandler { + + @Override + protected boolean checkUserDataPermissions(String pathInContext, Request request, Response response, Object constraintInfo) throws IOException + // check the SecurityHandler code, denying here does not provide authentication + // - identical with ConstraintSecurityHandler.checkUserDataPermissions implementation of Jetty source distribution + { + if (constraintInfo == null) + return true; + + RoleInfo roleInfo = (RoleInfo)constraintInfo; + if (roleInfo.isForbidden()) + return false; + + + UserDataConstraint dataConstraint = roleInfo.getUserDataConstraint(); + if (dataConstraint == null || dataConstraint == UserDataConstraint.None) + { + return true; + } + AbstractHttpConnection connection = AbstractHttpConnection.getCurrentConnection(); + Connector connector = connection.getConnector(); + + if (dataConstraint == UserDataConstraint.Integral) + { + if (connector.isIntegral(request)) + return true; + if (connector.getIntegralPort() > 0) + { + String scheme=connector.getIntegralScheme(); + int port=connector.getIntegralPort(); + String url = (HttpSchemes.HTTPS.equalsIgnoreCase(scheme) && port==443) + ? "https://"+request.getServerName()+request.getRequestURI() + : scheme + "://" + request.getServerName() + ":" + port + request.getRequestURI(); + if (request.getQueryString() != null) + url += "?" + request.getQueryString(); + response.setContentLength(0); + response.sendRedirect(url); + } + else + response.sendError(Response.SC_FORBIDDEN,"!Integral"); + + request.setHandled(true); + return false; + } + else if (dataConstraint == UserDataConstraint.Confidential) + { + if (connector.isConfidential(request)) + return true; + + if (connector.getConfidentialPort() > 0) + { + String scheme=connector.getConfidentialScheme(); + int port=connector.getConfidentialPort(); + String url = (HttpSchemes.HTTPS.equalsIgnoreCase(scheme) && port==443) + ? "https://"+request.getServerName()+request.getRequestURI() + : scheme + "://" + request.getServerName() + ":" + port + request.getRequestURI(); + if (request.getQueryString() != null) + url += "?" + request.getQueryString(); + response.setContentLength(0); + response.sendRedirect(url); + } + else + response.sendError(Response.SC_FORBIDDEN,"!Confidential"); + + request.setHandled(true); + return false; + } + else + { + throw new IllegalArgumentException("Invalid dataConstraint value: " + dataConstraint); + } + } + + @Override + protected boolean checkWebResourcePermissions(String pathInContext, Request request, + Response response, Object constraintInfo, UserIdentity userIdentity) throws IOException { + // deny and request for authentication, if necessary + // - identical with ConstraintSecurityHandler.checkWebResourcePermissions implementation of Jetty source distribution + if (constraintInfo == null) { + return true; + } + RoleInfo roleInfo = (RoleInfo) constraintInfo; + + if (!roleInfo.isChecked()) { + return true; + } + + if (roleInfo.isAnyRole() && request.getAuthType() != null) { + return true; + } + + for (String role : roleInfo.getRoles()) { + if (userIdentity.isUserInRole(role, null)) { + return true; + } + } + return false; + } + + @Override + protected boolean isAuthMandatory(Request baseRequest, Response base_response, Object constraintInfo) { + // identical with ConstraintSecurityHandler.isAuthMandatory implementation of Jetty source distribution + return constraintInfo != null && ((RoleInfo) constraintInfo).isChecked(); + } + + /** + * create the constraint for the given path + * for urls containing *_p. (like info_p.html) admin access is required, + * on localhost = admin setting no constraint is set + * @param pathInContext + * @param request + * @return RoleInfo with + * isChecked=true if any security contraint applies (compare reference implementation org.eclipse.jetty.security.ConstraintSecurityHandler) + * role = "admin" for resource name containint _p. + */ + @Override + protected RoleInfo prepareConstraintInfo(String pathInContext, Request request) { + final Switchboard sb = Switchboard.getSwitchboard(); + final boolean adminAccountForLocalhost = sb.getConfigBool("adminAccountForLocalhost", false); + //final String adminAccountBase64MD5 = sb.getConfig(YaCyLegacyCredential.ADMIN_ACCOUNT_B64MD5, ""); + + String refererHost; + // update AccessTracker + refererHost = request.getRemoteAddr(); + sb.track(refererHost, pathInContext); + + try { + refererHost = new MultiProtocolURL(request.getHeader("Referer")).getHost(); + } catch (MalformedURLException e) { + refererHost = null; + } + final boolean accessFromLocalhost = Domains.isLocalhost(request.getRemoteHost()) && (refererHost == null || refererHost.length() == 0 || Domains.isLocalhost(refererHost)); + // ! note : accessFromLocalhost compares localhost ip pattern ( ! currently also any intranet host is a local host) + final boolean grantedForLocalhost = adminAccountForLocalhost && accessFromLocalhost; + final boolean protectedPage = pathInContext.indexOf("_p.") > 0; + //final boolean accountEmpty = adminAccountBase64MD5.length() == 0; + //final boolean yacyBot = request.getHeader("User-Agent").startsWith("yacybot"); + + if (protectedPage) { // TODO: none public site + if (!grantedForLocalhost) { + RoleInfo roleinfo = new RoleInfo(); + roleinfo.setChecked(true); // RoleInfo.setChecked() : in Jetty this means - marked to have any security constraint + roleinfo.addRole("admin"); //YaCyLoginService assigns "admin" role to admin user + return roleinfo; + } // can omit else, as if grantedForLocalhost==true no constraint applies + // TODO: is this correct or adminAccountBase64MD5 not empty check neccessary ? + } + return null; + } + +} diff --git a/source/net/yacy/http/YaCyDefaultServlet.java b/source/net/yacy/http/YaCyDefaultServlet.java index 6cb3ebf32..6858b94ee 100644 --- a/source/net/yacy/http/YaCyDefaultServlet.java +++ b/source/net/yacy/http/YaCyDefaultServlet.java @@ -33,13 +33,11 @@ import java.lang.ref.SoftReference; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; -import java.nio.ByteBuffer; import java.util.Enumeration; import java.util.Iterator; import java.util.List; import java.util.concurrent.ConcurrentHashMap; -import javax.servlet.AsyncContext; import javax.servlet.RequestDispatcher; import javax.servlet.ServletContext; import javax.servlet.ServletException; @@ -72,24 +70,18 @@ import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.eclipse.jetty.http.HttpContent; import org.eclipse.jetty.http.HttpFields; -import org.eclipse.jetty.http.HttpHeader; -import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.http.MimeTypes; -import org.eclipse.jetty.io.WriterOutputStream; -import org.eclipse.jetty.server.HttpOutput; -import org.eclipse.jetty.server.InclusiveByteRange; -import org.eclipse.jetty.server.Response; -import org.eclipse.jetty.util.BufferUtil; -import org.eclipse.jetty.util.Callback; -import org.eclipse.jetty.util.MultiPartOutputStream; -import org.eclipse.jetty.util.QuotedStringTokenizer; import org.eclipse.jetty.util.URIUtil; import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.resource.ResourceFactory; /** - * YaCyDefaultServlet base on Jetty DefaultServlet.java + * YaCyDefaultServlet based on Jetty DefaultServlet.java * handles static files and the YaCy servlets. + * + * This interface impements the YaCy specific and standard Servlet routines + * which should not have a dependency on the implemented Jetty version. + * The Jetty version specific code is moved to the Jetty8HttpServerImpl.java implementation */ /** @@ -123,29 +115,29 @@ import org.eclipse.jetty.util.resource.ResourceFactory; * * */ -public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { +public abstract class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { private static final long serialVersionUID = 4900000000000001110L; - private ServletContext _servletContext; - private boolean _acceptRanges = true; - private boolean _dirAllowed = true; - private boolean _pathInfoOnly = false; - private boolean _etags = false; - private Resource _resourceBase; - private MimeTypes _mimeTypes; - private String[] _welcomes; - private String _relativeResourceBase; + protected ServletContext _servletContext; + protected boolean _acceptRanges = true; + protected boolean _dirAllowed = true; + protected boolean _pathInfoOnly = false; + protected boolean _etags = false; + protected Resource _resourceBase; + protected MimeTypes _mimeTypes; + protected String[] _welcomes; + protected String _relativeResourceBase; - private File htLocalePath; - private File htDocsPath; - private static final serverClassLoader provider = new serverClassLoader(/*this.getClass().getClassLoader()*/); - private ConcurrentHashMap> templateMethodCache = null; + protected File _htLocalePath; + protected File _htDocsPath; + protected static final serverClassLoader provider = new serverClassLoader(/*this.getClass().getClassLoader()*/); + protected ConcurrentHashMap> templateMethodCache = null; /* ------------------------------------------------------------ */ @Override public void init() throws UnavailableException { - htDocsPath = Switchboard.getSwitchboard().htDocsPath; - htLocalePath = Switchboard.getSwitchboard().getDataPath("locale.translated_html", "DATA/LOCALE/htroot"); + _htDocsPath = Switchboard.getSwitchboard().htDocsPath; + _htLocalePath = Switchboard.getSwitchboard().getDataPath("locale.translated_html", "DATA/LOCALE/htroot"); _servletContext = getServletContext(); @@ -185,7 +177,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { /* ------------------------------------------------------------ */ - private boolean getInitBoolean(String name, boolean dft) { + protected boolean getInitBoolean(String name, boolean dft) { String value = getInitParameter(name); if (value == null || value.length() == 0) { return dft; @@ -357,13 +349,13 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { if (content != null) { content.release(); } else if (resource != null) { - resource.close(); + resource.release(); } } } /* ------------------------------------------------------------ */ - private boolean hasDefinedRange(Enumeration reqRanges) { + protected boolean hasDefinedRange(Enumeration reqRanges) { return (reqRanges != null && reqRanges.hasMoreElements()); } @@ -398,7 +390,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { * @param path in context * @return The path of the matching welcome file in context or null. */ - private String getWelcomeFile(String pathInContext) { + protected String getWelcomeFile(String pathInContext) { if (_welcomes == null) { return null; } @@ -415,104 +407,8 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { /* ------------------------------------------------------------ */ /* Check modification date headers. */ - protected boolean passConditionalHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, HttpContent content) - throws IOException { - try { - if (!HttpMethod.HEAD.is(request.getMethod())) { - if (_etags) { - String ifm = request.getHeader(HttpHeader.IF_MATCH.asString()); - if (ifm != null) { - boolean match = false; - if (content.getETag() != null) { - QuotedStringTokenizer quoted = new QuotedStringTokenizer(ifm, ", ", false, true); - while (!match && quoted.hasMoreTokens()) { - String tag = quoted.nextToken(); - if (content.getETag().toString().equals(tag)) { - match = true; - } - } - } - - if (!match) { - response.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED); - return false; - } - } - - String if_non_match_etag = request.getHeader(HttpHeader.IF_NONE_MATCH.asString()); - if (if_non_match_etag != null && content.getETag() != null) { - // Look for GzipFiltered version of etag - if (content.getETag().toString().equals(request.getAttribute("o.e.j.s.GzipFilter.ETag"))) { - response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); - response.setHeader(HeaderFramework.ETAG, if_non_match_etag); - return false; - } - - // Handle special case of exact match. - if (content.getETag().toString().equals(if_non_match_etag)) { - response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); - response.setHeader(HeaderFramework.ETAG, content.getETag()); - return false; - } - - // Handle list of tags - QuotedStringTokenizer quoted = new QuotedStringTokenizer(if_non_match_etag, ", ", false, true); - while (quoted.hasMoreTokens()) { - String tag = quoted.nextToken(); - if (content.getETag().toString().equals(tag)) { - response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); - response.setHeader(HeaderFramework.ETAG, content.getETag()); - return false; - } - } - - // If etag requires content to be served, then do not check if-modified-since - return true; - } - } - - // Handle if modified since - String ifms = request.getHeader(HttpHeader.IF_MODIFIED_SINCE.asString()); - if (ifms != null) { - //Get jetty's Response impl - String mdlm = content.getLastModified(); - if (mdlm != null && ifms.equals(mdlm)) { - response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); - if (_etags) { - response.setHeader(HeaderFramework.ETAG, content.getETag()); - } - response.flushBuffer(); - return false; - } - - long ifmsl = request.getDateHeader(HttpHeader.IF_MODIFIED_SINCE.asString()); - if (ifmsl != -1 && resource.lastModified() / 1000 <= ifmsl / 1000) { - response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); - if (_etags) { - response.setHeader(HeaderFramework.ETAG, content.getETag()); - } - response.flushBuffer(); - return false; - } - } - - // Parse the if[un]modified dates and compare to resource - long date = request.getDateHeader(HttpHeader.IF_UNMODIFIED_SINCE.asString()); - if (date != -1 && resource.lastModified() / 1000 > date / 1000) { - response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED); - return false; - } - - } - } catch (IllegalArgumentException iae) { - if (!response.isCommitted()) { - response.sendError(400, iae.getMessage()); - } - throw iae; - } - return true; - } - + abstract protected boolean passConditionalHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, HttpContent content) + throws IOException; /* ------------------------------------------------------------------- */ protected void sendDirectory(HttpServletRequest request, @@ -542,228 +438,16 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { } /* ------------------------------------------------------------ */ - protected void sendData(HttpServletRequest request, + abstract protected void sendData(HttpServletRequest request, HttpServletResponse response, boolean include, Resource resource, HttpContent content, Enumeration reqRanges) - throws IOException { - final long content_length = (content == null) ? resource.length() : content.getContentLength(); - - // Get the output stream (or writer) - OutputStream out = null; - boolean written; - try { - out = response.getOutputStream(); - - // has a filter already written to the response? - written = out instanceof HttpOutput - ? ((HttpOutput) out).isWritten() - : true; - } catch (IllegalStateException e) { - out = new WriterOutputStream(response.getWriter()); - written = true; // there may be data in writer buffer, so assume written - } - - if (reqRanges == null || !reqRanges.hasMoreElements() || content_length < 0) { - // if there were no ranges, send entire entity - if (include) { - resource.writeTo(out, 0, content_length); - } // else if we can't do a bypass write because of wrapping - else if (content == null || written || !(out instanceof HttpOutput)) { - // write normally - writeHeaders(response, content, written ? -1 : content_length); - ByteBuffer buffer = (content == null) ? null : content.getIndirectBuffer(); - if (buffer != null) { - BufferUtil.writeTo(buffer, out); - } else { - resource.writeTo(out, 0, content_length); - } - } // else do a bypass write - else { - // write the headers - if (response instanceof Response) { - Response r = (Response) response; - writeOptionHeaders(r.getHttpFields()); - r.setHeaders(content); - } else { - writeHeaders(response, content, content_length); - } - - // write the content asynchronously if supported - if (request.isAsyncSupported()) { - final AsyncContext context = request.startAsync(); - - ((HttpOutput) out).sendContent(content, new Callback() { - @Override - public void succeeded() { - context.complete(); - } - - @Override - public void failed(Throwable x) { - ConcurrentLog.logException(x); - context.complete(); - } - }); - } // otherwise write content blocking - else { - ((HttpOutput) out).sendContent(content); - } - } - } else { - // Parse the satisfiable ranges - List ranges = InclusiveByteRange.satisfiableRanges(reqRanges, content_length); - - // if there are no satisfiable ranges, send 416 response - if (ranges == null || ranges.size() == 0) { - writeHeaders(response, content, content_length); - response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); - response.setHeader(HeaderFramework.CONTENT_RANGE, - InclusiveByteRange.to416HeaderRangeString(content_length)); - resource.writeTo(out, 0, content_length); - return; - } - - // if there is only a single valid range (must be satisfiable - // since were here now), send that range with a 216 response - if (ranges.size() == 1) { - InclusiveByteRange singleSatisfiableRange = ranges.get(0); - long singleLength = singleSatisfiableRange.getSize(content_length); - writeHeaders(response, content, singleLength); - response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); - response.setHeader(HeaderFramework.CONTENT_RANGE, - singleSatisfiableRange.toHeaderRangeString(content_length)); - resource.writeTo(out, singleSatisfiableRange.getFirst(content_length), singleLength); - return; - } - - // multiple non-overlapping valid ranges cause a multipart - // 216 response which does not require an overall - // content-length header - // - writeHeaders(response, content, -1); - String mimetype = (content == null || content.getContentType() == null ? null : content.getContentType().toString()); - if (mimetype == null) { - ConcurrentLog.warn("YaCyDefaultServlet", "Unknown mimetype for " + request.getRequestURI()); - } - MultiPartOutputStream multi = new MultiPartOutputStream(out); - response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); - - // If the request has a "Request-Range" header then we need to - // send an old style multipart/x-byteranges Content-Type. This - // keeps Netscape and acrobat happy. This is what Apache does. - String ctp; - if (request.getHeader(HttpHeader.REQUEST_RANGE.asString()) != null) { - ctp = "multipart/x-byteranges; boundary="; - } else { - ctp = "multipart/byteranges; boundary="; - } - response.setContentType(ctp + multi.getBoundary()); - - InputStream in = resource.getInputStream(); - long pos = 0; - - // calculate the content-length - int length = 0; - String[] header = new String[ranges.size()]; - for (int i = 0; i < ranges.size(); i++) { - InclusiveByteRange ibr = ranges.get(i); - header[i] = ibr.toHeaderRangeString(content_length); - length += - ((i > 0) ? 2 : 0) - + 2 + multi.getBoundary().length() + 2 - + (mimetype == null ? 0 : HeaderFramework.CONTENT_TYPE.length() + 2 + mimetype.length()) + 2 - + HeaderFramework.CONTENT_RANGE.length() + 2 + header[i].length() + 2 - + 2 - + (ibr.getLast(content_length) - ibr.getFirst(content_length)) + 1; - } - length += 2 + 2 + multi.getBoundary().length() + 2 + 2; - response.setContentLength(length); - - for (int i = 0; i < ranges.size(); i++) { - InclusiveByteRange ibr = ranges.get(i); - multi.startPart(mimetype, new String[]{HeaderFramework.CONTENT_RANGE + ": " + header[i]}); - - long start = ibr.getFirst(content_length); - long size = ibr.getSize(content_length); - if (in != null) { - // Handle non cached resource - if (start < pos) { - in.close(); - in = resource.getInputStream(); - pos = 0; - } - if (pos < start) { - in.skip(start - pos); - pos = start; - } - - FileUtils.copy(in, multi, size); - pos += size; - } else // Handle cached resource - { - (resource).writeTo(multi, start, size); - } - - } - if (in != null) { - in.close(); - } - multi.close(); - } - } + throws IOException; /* ------------------------------------------------------------ */ - protected void writeHeaders(HttpServletResponse response, HttpContent content, long count) { - if (content.getContentType() != null && response.getContentType() == null) { - response.setContentType(content.getContentType().toString()); - } - - if (response instanceof Response) { - Response r = (Response) response; - HttpFields fields = r.getHttpFields(); - - if (content.getLastModified() != null) { - fields.put(HeaderFramework.LAST_MODIFIED, content.getLastModified()); - } else if (content.getResource() != null) { - long lml = content.getResource().lastModified(); - if (lml != -1) { - fields.putDateField(HeaderFramework.LAST_MODIFIED, lml); - } - } - - if (count != -1) { - r.setLongContentLength(count); - } - - writeOptionHeaders(fields); - - if (_etags) { - fields.put(HeaderFramework.ETAG, content.getETag()); - } - } else { - long lml = content.getResource().lastModified(); - if (lml >= 0) { - response.setDateHeader(HeaderFramework.LAST_MODIFIED, lml); - } - - if (count != -1) { - if (count < Integer.MAX_VALUE) { - response.setContentLength((int) count); - } else { - response.setHeader(HeaderFramework.CONTENT_LENGTH, Long.toString(count)); - } - } - - writeOptionHeaders(response); - - if (_etags) { - response.setHeader(HeaderFramework.ETAG, content.getETag().toString()); - } - } - } + abstract protected void writeHeaders(HttpServletResponse response, HttpContent content, long count); /* ------------------------------------------------------------ */ protected void writeOptionHeaders(HttpFields fields) { @@ -780,11 +464,11 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { } - private Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { + protected Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { return rewriteMethod(targetClass).invoke(null, new Object[]{request, args, Switchboard.getSwitchboard()}); // add switchboard } - private RequestHeader generateLegacyRequestHeader(HttpServletRequest request, String target, String targetExt) { + protected RequestHeader generateLegacyRequestHeader(HttpServletRequest request, String target, String targetExt) { RequestHeader legacyRequestHeader = new RequestHeader(); @SuppressWarnings("unchecked") Enumeration headers = request.getHeaderNames(); @@ -814,20 +498,20 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { */ public File getLocalizedFile(final String path, final String localeSelection) throws IOException { if (!(localeSelection.equals("default"))) { - final File localePath = new File(htLocalePath, localeSelection + '/' + path); + final File localePath = new File(_htLocalePath, localeSelection + '/' + path); if (localePath.exists()) { return localePath; // avoid "NoSuchFile" troubles if the "localeSelection" is misspelled } } - File docsPath = new File(htDocsPath, path); + File docsPath = new File(_htDocsPath, path); if (docsPath.exists()) { return docsPath; } return _resourceBase.addPath(path).getFile(); } - private File rewriteClassFile(final File template) { + protected File rewriteClassFile(final File template) { try { String f = template.getCanonicalPath(); final int p = f.lastIndexOf('.'); @@ -845,7 +529,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { } } - private Method rewriteMethod(final File classFile) throws InvocationTargetException { + protected Method rewriteMethod(final File classFile) throws InvocationTargetException { Method m = null; // now make a class out of the stream try { @@ -879,7 +563,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { return m; } - public void handleTemplate(String target, HttpServletRequest request, + protected void handleTemplate(String target, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { Switchboard sb = Switchboard.getSwitchboard(); @@ -1047,7 +731,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { } // parse SSI line and include resource - private void parseSSI(final net.yacy.cora.util.ByteBuffer in, final int off, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + protected void parseSSI(final net.yacy.cora.util.ByteBuffer in, final int off, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { if (in.startsWith("".getBytes(), p + 10); + + out.write(in, off, p - off); + out.flush(); + parseSSI(buffer, p, request, response); + off = q + 3; + p = buffer.indexOf("