// // YaCyHttpServer // Copyright 2011 by Florian Richter // First released 13.04.2011 at http://yacy.net // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public // License as published by the Free Software Foundation; either // version 2.1 of the License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with this program in the file lgpl21.txt // If not, see . // package net.yacy.http; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.InetAddress; import java.security.KeyStore; import java.util.StringTokenizer; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.DefaultHandler; import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.InetAccessHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.ProcessorUtils; import org.eclipse.jetty.util.log.Log; import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.webapp.WebAppContext; import net.yacy.cora.util.ConcurrentLog; import net.yacy.http.servlets.YaCyDefaultServlet; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.utils.PKCS12Tool; /** * class to embedded Jetty 9 http server into YaCy */ public class YaCyHttpServer { private final Server server; /** * @param port TCP Port to listen for http requests */ public YaCyHttpServer(final int port, final String host) { final Switchboard sb = Switchboard.getSwitchboard(); this.server = new Server(); final int cores = ProcessorUtils.availableProcessors(); final int acceptors = Math.max(1, Math.min(4, cores/2)); // original: Math.max(1, Math.min(4,cores/8)); final HttpConfiguration httpConfig = new HttpConfiguration(); httpConfig.setRequestHeaderSize(16384); final HttpConnectionFactory hcf = new HttpConnectionFactory(httpConfig); final ServerConnector connector = new ServerConnector(this.server, null, null, null, acceptors, -1, hcf); connector.setPort(port); connector.setHost(host); connector.setName("httpd-" + host + ":" + Integer.toString(port)); connector.setIdleTimeout(9000); // timout in ms when no bytes send / received connector.setAcceptQueueSize(128); this.server.addConnector(connector); // add ssl/https connector final boolean useSSL = sb.getConfigBool("server.https", false); if (useSSL) { final SslContextFactory sslContextFactory = new SslContextFactory.Server(); final SSLContext sslContext = initSslContext(sb); if (sslContext != null) { final int sslport = sb.getConfigInt(SwitchboardConstants.SERVER_SSLPORT, 8443); sslContextFactory.setSslContext(sslContext); // SSL HTTP Configuration final HttpConfiguration https_config = new HttpConfiguration(); https_config.addCustomizer(new SecureRequestCustomizer()); // SSL Connector final ServerConnector sslConnector = new ServerConnector(this.server, new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), new HttpConnectionFactory(https_config)); sslConnector.setPort(sslport); sslConnector.setName("ssld:" + Integer.toString(sslport)); // name must start with ssl (for withSSL() to work correctly) sslConnector.setIdleTimeout(9000); // timout in ms when no bytes send / received this.server.addConnector(sslConnector); ConcurrentLog.info("SERVER", "SSL support initialized successfully on port " + sslport); } } final YacyDomainHandler domainHandler = new YacyDomainHandler(); domainHandler.setAlternativeResolver(sb.peers); // configure root context final WebAppContext htrootContext = new WebAppContext(); htrootContext.setContextPath("/"); final String htrootpath = sb.appPath + "/" + sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT); ConcurrentLog.info("Jetty9HttpServerImpl", "htrootpath = " + htrootpath); htrootContext.setErrorHandler(new YaCyErrorHandler()); // handler for custom error page try { htrootContext.setBaseResource(Resource.newResource(htrootpath)); // set web.xml to use // make use of Jetty feature to define web.xml other as default WEB-INF/web.xml // and to use a DefaultsDescriptor merged with a individual web.xml // use defaults/web.xml as default and look in DATA/SETTINGS for local addition/changes htrootContext.setDefaultsDescriptor(sb.appPath + "/defaults/web.xml"); final Resource webxml = Resource.newResource(sb.dataPath + "/DATA/SETTINGS/web.xml"); if (webxml.exists()) { htrootContext.setDescriptor(webxml.getName()); } } catch (final IOException ex) { if (htrootContext.getBaseResource() == null) { ConcurrentLog.severe("SERVER", "could not find directory: htroot "); } else { ConcurrentLog.warn("SERVER", "could not find: defaults/web.xml or DATA/SETTINGS/web.xml"); } } // as fundamental component leave this hardcoded, other servlets may be defined in web.xml only final ServletHolder sholder = new ServletHolder(YaCyDefaultServlet.class); sholder.setInitParameter("resourceBase", htrootpath); sholder.setAsyncSupported(true); // needed for YaCyQoSFilter //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html htrootContext.addServlet(sholder, "/*"); final GzipHandler gzipHandler = new GzipHandler(); /* * Decompression of incoming requests body is required for index distribution * APIs /yacy/transferRWI.html and /yacy/transferURL.html This was previously * handled by a GZIPRequestWrapper in the YaCyDefaultServlet. */ gzipHandler.setInflateBufferSize(4096); if (!sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP, SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) { /* Gzip compression of responses can be disabled by user configuration */ gzipHandler.setExcludedMethods(HttpMethod.GET.asString(), HttpMethod.POST.asString()); } htrootContext.setGzipHandler(gzipHandler); // ----------------------------------------------------------------------------- // here we set and map the mandatory servlets, needed for typical YaCy operation // to make sure they are available even if removed in individual web.xml // additional, optional or individual servlets or servlet mappings can be set in web.xml // in Jetty 9 servlet should be set only once // therefore only the settings in web.xml is used //add SolrSelectServlet //htrootContext.addServlet(SolrSelectServlet.class, "/solr/select"); // uses the default core, collection1 //htrootContext.addServlet(SolrSelectServlet.class, "/solr/collection1/select"); // the same servlet, identifies the collection1 core using the path //htrootContext.addServlet(SolrSelectServlet.class, "/solr/webgraph/select"); // the same servlet, identifies the webgraph core using the path //htrootContext.addServlet(SolrServlet.class, "/solr/collection1/admin/luke"); //htrootContext.addServlet(SolrServlet.class, "/solr/webgraph/admin/luke"); // add proxy?url= servlet //htrootContext.addServlet(YaCyProxyServlet.class,"/proxy.html"); // add GSA servlet //htrootContext.addServlet(GSAsearchServlet.class,"/gsa/search"); // --- eof default servlet mappings -------------------------------------------- // define list of YaCy specific general handlers final HandlerList handlers = new HandlerList(); if (sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false)) { // Proxyhandlers are only needed if feature activated (save resources if not used) ConcurrentLog.info("SERVER", "load Jetty handler for transparent proxy"); handlers.setHandlers(new Handler[]{new MonitorHandler(), domainHandler, new ProxyCacheHandler(), new ProxyHandler()}); } else { handlers.setHandlers(new Handler[]{new MonitorHandler(), domainHandler}); } // context handler for dispatcher and security (hint: dispatcher requires a context) final ContextHandler context = new ContextHandler(); context.setServer(this.server); context.setContextPath("/"); context.setHandler(handlers); context.setMaxFormContentSize(1024 * 1024 * 10); // allow 10MB, large forms may be required during crawl starts with long lists final org.eclipse.jetty.util.log.Logger log = Log.getRootLogger(); context.setLogger(log); // make YaCy handlers (in context) and servlet context handlers available (both contain root context "/") // logic: 1. YaCy handlers are called if request not handled (e.g. proxy) then servlets handle it final ContextHandlerCollection allrequesthandlers = new ContextHandlerCollection(); allrequesthandlers.setServer(this.server); allrequesthandlers.addHandler(context); allrequesthandlers.addHandler(htrootContext); allrequesthandlers.addHandler(new DefaultHandler()); // if not handled by other handler final YaCyLoginService loginService = new YaCyLoginService(); // this is very important (as it is part of the user password hash) // changes will ivalidate all current existing user-password-hashes (from userDB) loginService.setName(sb.getConfig(SwitchboardConstants.ADMIN_REALM,"YaCy")); final YaCySecurityHandler securityHandler = new YaCySecurityHandler(); securityHandler.setLoginService(loginService); htrootContext.setSecurityHandler(securityHandler); // wrap all handlers final Handler crashHandler = new CrashProtectionHandler(this.server, allrequesthandlers); // check server access restriction and add InetAccessHandler if restrictions are needed // otherwise don't (to save performance) final String white = sb.getConfig("serverClient", "*"); if (!white.equals("*")) { // full ip (allowed ranges 0-255 or prefix 10.0-255,0,0-100 or CIDR notation 192.168.1.0/24) final StringTokenizer st = new StringTokenizer(white, ","); final InetAccessHandler whiteListHandler; if (white.contains("|")) { /* * At least one pattern includes a path definition : we must use the * InetPathAccessHandler as InetAccessHandler doesn't support path patterns */ whiteListHandler = new InetPathAccessHandler(); } else { whiteListHandler = new InetAccessHandler(); } int i = 0; while (st.hasMoreTokens()) { final String pattern = st.nextToken(); try { whiteListHandler.include(pattern); } catch (final IllegalArgumentException nex) { // catch format exception on wrong ip address pattern ConcurrentLog.severe("SERVER", "Server Access Settings - IP filter: " + nex.getMessage()); continue; } i++; } if (i > 0) { final String loopbackAddress = InetAddress.getLoopbackAddress().getHostAddress(); whiteListHandler.include(loopbackAddress); whiteListHandler.setHandler(crashHandler); this.server.setHandler(whiteListHandler); ConcurrentLog.info("SERVER","activated IP access restriction to: [" + loopbackAddress + "," + white +"]"); } else { this.server.setHandler(crashHandler); // InetAccessHandler not needed } } else { this.server.setHandler(crashHandler); // InetAccessHandler not needed } } /** * start http server */ public void startupServer() throws Exception { // option to finish running requests on shutdown // server.setGracefulShutdown(3000); this.server.setStopAtShutdown(true); this.server.start(); } /** * stop http server and wait for it */ public void stop() throws Exception { this.server.stop(); this.server.join(); } /** * @return true if ssl/https connector is available */ public boolean withSSL() { final Connector[] clist = this.server.getConnectors(); for (final Connector c:clist) { if (c.getName().startsWith("ssl")) return true; } return false; } /** * The port of actual running ssl connector * @return the ssl/https port or -1 if not active */ public int getSslPort() { final Connector[] clist = this.server.getConnectors(); for (final Connector c:clist) { if (c.getName().startsWith("ssl")) { final int port =((ServerConnector)c).getLocalPort(); return port; } } return -1; } /** * reconnect with new port settings (after waiting milsec) - routine returns * immediately * checks http and ssl connector for new port settings * @param milsec wait time */ public void reconnect(final int milsec) { new Thread("Jetty8HttpServer.reconnect") { @Override public void run() { if (milsec > 0) try { Thread.sleep(milsec); } catch (final Exception e) { ConcurrentLog.logException(e); } try { if (!YaCyHttpServer.this.server.isRunning() || YaCyHttpServer.this.server.isStopped()) { YaCyHttpServer.this.server.start(); } // reconnect with new settings (instead to stop/start server, just manipulate connectors final Connector[] cons = YaCyHttpServer.this.server.getConnectors(); final int port = Switchboard.getSwitchboard().getLocalPort(); final int sslport = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.SERVER_SSLPORT, 8443); for (final Connector con : cons) { // check http connector if (con.getName().startsWith("httpd") && ((ServerConnector)con).getPort() != port) { ((ServerConnector)con).close(); con.stop(); if (!con.isStopped()) { ConcurrentLog.warn("SERVER", "Reconnect: Jetty Connector failed to stop"); } ((ServerConnector)con).setPort(port); con.start(); ConcurrentLog.info("SERVER", "set new port for Jetty connector " + con.getName()); continue; } // check https connector if (con.getName().startsWith("ssl") && ((ServerConnector)con).getPort() != sslport) { ((ServerConnector)con).close(); con.stop(); if (!con.isStopped()) { ConcurrentLog.warn("SERVER", "Reconnect: Jetty Connector failed to stop"); } ((ServerConnector)con).setPort(sslport); con.start(); ConcurrentLog.info("SERVER", "set new port for Jetty connector " + con.getName()); } } } catch (final Exception ex) { ConcurrentLog.logException(ex); } } }.start(); } /** * forces loginservice to reload user credentials * (used after setting new pwd in cfg file/db) * @param username */ public void resetUser(final String username) { final YaCySecurityHandler hx = this.server.getChildHandlerByClass(YaCySecurityHandler.class); if (hx != null) { final YaCyLoginService loginservice = (YaCyLoginService) hx.getLoginService(); if (loginservice.removeUser(username)) { // remove old credential from cache loginservice.loadUserInfo(username); } } } /** * removes user from knowuser cache of loginservice * @param username */ public void removeUser(final String username) { final YaCySecurityHandler hx = this.server.getChildHandlerByClass(YaCySecurityHandler.class); if (hx != null) { final YaCyLoginService loginservice = (YaCyLoginService) hx.getLoginService(); loginservice.removeUser(username); } } /** * get Jetty version * @return version_string */ public String getVersion() { return "Jetty " + Server.getVersion(); } /** * Init SSL Context from config settings * @param sb Switchboard * @return default or sslcontext according to config */ private SSLContext initSslContext(final Switchboard sb) { // getting the keystore file name String keyStoreFileName = sb.getConfig("keyStore", "").trim(); // getting the keystore pwd String keyStorePwd = sb.getConfig("keyStorePassword", "").trim(); // take a look if we have something to import final String pkcs12ImportFile = sb.getConfig("pkcs12ImportFile", "").trim(); // if no keyStore and no import is defined, then set the default key if (keyStoreFileName.isEmpty() && keyStorePwd.isEmpty() && pkcs12ImportFile.isEmpty()) { keyStoreFileName = "defaults/freeworldKeystore"; keyStorePwd = "freeworld"; sb.setConfig("keyStore", keyStoreFileName); sb.setConfig("keyStorePassword", keyStorePwd); } if (pkcs12ImportFile.length() > 0) { ConcurrentLog.info("SERVER", "Import certificates from import file '" + pkcs12ImportFile + "'."); try { // getting the password final String pkcs12ImportPwd = sb.getConfig("pkcs12ImportPwd", "").trim(); // creating tool to import cert final PKCS12Tool pkcsTool = new PKCS12Tool(pkcs12ImportFile,pkcs12ImportPwd); // creating a new keystore file if (keyStoreFileName.isEmpty()) { // using the default keystore name keyStoreFileName = "DATA/SETTINGS/myPeerKeystore"; // creating an empty java keystore final KeyStore ks = KeyStore.getInstance("JKS"); ks.load(null,keyStorePwd.toCharArray()); try ( /* Automatically closed by this try-with-resources statement */ final FileOutputStream ksOut = new FileOutputStream(keyStoreFileName); ) { ks.store(ksOut, keyStorePwd.toCharArray()); } // storing path to keystore into config file sb.setConfig("keyStore", keyStoreFileName); } // importing certificate pkcsTool.importToJKS(keyStoreFileName, keyStorePwd); // removing entries from config file sb.setConfig("pkcs12ImportFile", ""); sb.setConfig("pkcs12ImportPwd", ""); // deleting original import file // TODO: should we do this } catch (final Exception e) { ConcurrentLog.severe("SERVER", "Unable to import certificate from import file '" + pkcs12ImportFile + "'.",e); } } else if (keyStoreFileName.isEmpty()) return null; // get the ssl context try { ConcurrentLog.info("SERVER","Initializing SSL support ..."); // creating a new keystore instance of type (java key store) if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER", "Initializing keystore ..."); final KeyStore ks = KeyStore.getInstance("JKS"); // loading keystore data from file if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER","Loading keystore file " + keyStoreFileName); final FileInputStream stream = new FileInputStream(keyStoreFileName); try { ks.load(stream, keyStorePwd.toCharArray()); } finally { try { stream.close(); } catch(final IOException ioe) { ConcurrentLog.warn("SERVER", "Could not close input stream on file " + keyStoreFileName); } } // creating a keystore factory if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER","Initializing key manager factory ..."); final KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); kmf.init(ks,keyStorePwd.toCharArray()); // initializing the ssl context if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER","Initializing SSL context ..."); final SSLContext sslcontext = SSLContext.getInstance("TLS"); sslcontext.init(kmf.getKeyManagers(), null, null); return sslcontext; } catch (final Exception e) { final String errorMsg = "FATAL ERROR: Unable to initialize the SSL Socket factory. " + e.getMessage(); ConcurrentLog.severe("SERVER",errorMsg); System.out.println(errorMsg); return null; } } public int getServerThreads() { return this.server == null ? 0 : this.server.getThreadPool().getThreads() - this.server.getThreadPool().getIdleThreads(); } @Override public String toString() { return this.server.dump() + "\n\n" + this.server.getState(); } }