From 2d2df64870ff0af13dca67227f453250cee276f0 Mon Sep 17 00:00:00 2001 From: Florian Richter Date: Fri, 11 Mar 2011 13:15:56 +0100 Subject: [PATCH] * jetty works for many pages now --- build.xml | 2 +- htroot/Status.java | 4 +- source/net/yacy/http/HttpServer.java | 31 +++-- source/net/yacy/http/TemplateHandler.java | 149 ++++++++++++++++++++-- source/net/yacy/yacy.java | 43 ++++--- 5 files changed, 185 insertions(+), 44 deletions(-) diff --git a/build.xml b/build.xml index 1ad0d07ca..e879ac737 100644 --- a/build.xml +++ b/build.xml @@ -644,7 +644,7 @@ - + diff --git a/htroot/Status.java b/htroot/Status.java index 60dd20f4f..bfa1432af 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -309,8 +309,8 @@ public class Status { // connection information final serverCore httpd = (serverCore) sb.getThread("10_httpd"); - prop.putNum("connectionsActive", httpd.getJobCount()); - prop.putNum("connectionsMax", httpd.getMaxSessionCount()); + //prop.putNum("connectionsActive", httpd.getJobCount()); + //prop.putNum("connectionsMax", httpd.getMaxSessionCount()); // Queue information final int loaderJobCount = sb.crawlQueues.workerSize(); diff --git a/source/net/yacy/http/HttpServer.java b/source/net/yacy/http/HttpServer.java index 7d95f8241..6dabfd9bf 100644 --- a/source/net/yacy/http/HttpServer.java +++ b/source/net/yacy/http/HttpServer.java @@ -6,20 +6,16 @@ import org.eclipse.jetty.server.handler.DefaultHandler; import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; import org.eclipse.jetty.server.nio.SelectChannelConnector; -import org.eclipse.jetty.util.thread.QueuedThreadPool; public class HttpServer { - public static void initYaCyServer() { - // create directories - } + private Server server = new Server(); /** * @param port TCP Port to listen for http requests - * @throws Exception */ - public static void runHttpServer(int port) throws Exception { - Server server = new Server(); + public HttpServer(int port) { + server = new Server(); SelectChannelConnector connector = new SelectChannelConnector(); connector.setPort(port); //connector.setThreadPool(new QueuedThreadPool(20)); @@ -29,21 +25,34 @@ public class HttpServer { resource_handler.setDirectoriesListed(true); resource_handler.setWelcomeFiles(new String[]{ "index.html" }); - resource_handler.setResourceBase("DATA/HTDOCS"); + resource_handler.setResourceBase("htroot/"); HandlerList handlers = new HandlerList(); handlers.setHandlers(new Handler[] { new TemplateHandler(), resource_handler, new DefaultHandler() }); server.setHandler(handlers); - + } + + public void start() throws Exception { server.start(); + } + + public void initYaCyServer() { + // TODO: delete? + // create directories + } + + public void stop() throws Exception { + server.stop(); server.join(); - } + } /** * just for testing and debugging */ public static void main(String[] args) throws Exception { - runHttpServer(8080); + HttpServer server = new HttpServer(8090); + server.start(); + server.stop(); } } diff --git a/source/net/yacy/http/TemplateHandler.java b/source/net/yacy/http/TemplateHandler.java index 4c02eedd2..cd607361e 100644 --- a/source/net/yacy/http/TemplateHandler.java +++ b/source/net/yacy/http/TemplateHandler.java @@ -1,28 +1,49 @@ package net.yacy.http; +import java.awt.Image; +import java.awt.image.BufferedImage; +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.lang.ref.SoftReference; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Date; +import java.util.Enumeration; import java.util.concurrent.ConcurrentHashMap; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import net.yacy.cora.date.GenericFormatter; +import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.ByteBuffer; +import net.yacy.kelondro.util.FileUtils; +import net.yacy.kelondro.util.MemoryControl; +import net.yacy.visualization.RasterPlotter; +import org.eclipse.jetty.http.MimeTypes; import org.eclipse.jetty.server.HttpConnection; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; +import de.anomic.data.MimeTable; +import de.anomic.http.server.HTTPDemon; +import de.anomic.http.server.TemplateEngine; +import de.anomic.search.Switchboard; import de.anomic.server.serverClassLoader; +import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.servletProperties; +import de.anomic.yacy.yacyBuildProperties; +import de.anomic.yacy.yacySeed; public class TemplateHandler extends AbstractHandler { @@ -35,6 +56,8 @@ public class TemplateHandler extends AbstractHandler { boolean useTemplateCache = false; private ConcurrentHashMap> templateCache = null; private ConcurrentHashMap> templateMethodCache = null; + + private MimeTypes mimeTypes = new MimeTypes(); @Override protected void doStart() throws Exception { @@ -119,22 +142,54 @@ public class TemplateHandler extends AbstractHandler { } - private final Object invokeServlet(final File targetClass, final HttpServletRequest request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { - return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, null}); // add switchboard + private final Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { + return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, Switchboard.getSwitchboard()}); // add switchboard + } + + private RequestHeader generateLegacyRequestHeader(HttpServletRequest request, String target, String targetExt) { + RequestHeader legacyRequestHeader = new RequestHeader(); + @SuppressWarnings("unchecked") + Enumeration headers = request.getHeaderNames(); + while (headers.hasMoreElements()) { + String headerName = headers.nextElement(); + @SuppressWarnings("unchecked") + Enumeration header = request.getHeaders(headerName); + while(header.hasMoreElements()) + legacyRequestHeader.add(headerName, header.nextElement()); + } + + legacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, request.getRemoteAddr()); + legacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, target); + legacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, targetExt); + + return legacyRequestHeader; } @Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + Switchboard sb = Switchboard.getSwitchboard(); + + System.err.println("Page: " + target); String localeSelection = "default"; File targetFile = getLocalizedFile(target, localeSelection); File targetClass = rewriteClassFile(new File(htDefaultPath, target)); + String targetExt = target.substring(target.lastIndexOf('.') + 1, target.length()); if ((targetClass != null)) { - serverObjects args = new serverObjects(request.getParameterMap()); + @SuppressWarnings("unchecked") + serverObjects args = new serverObjects(); + @SuppressWarnings("unchecked") + Enumeration argNames = request.getParameterNames(); + while (argNames.hasMoreElements()) { + String argName = argNames.nextElement(); + args.put(argName, request.getParameter(argName)); + } + RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt); + Object tmp; try { - tmp = invokeServlet(targetClass, request, args); + tmp = invokeServlet(targetClass, legacyRequestHeader, args); } catch (InvocationTargetException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -147,8 +202,52 @@ public class TemplateHandler extends AbstractHandler { // TODO Auto-generated catch block e.printStackTrace(); throw new ServletException(); - } + + if (tmp instanceof RasterPlotter) { + final RasterPlotter yp = (RasterPlotter) tmp; + // send an image to client + final String mimeType = MimeTable.ext2mime(targetExt, "text/html"); + final ByteBuffer result = RasterPlotter.exportImage(yp.getImage(), "png"); + + response.setContentType(mimeType); + response.setContentLength(result.length()); + response.setStatus(HttpServletResponse.SC_OK); + + result.writeTo(response.getOutputStream()); + + // we handled this request, break out of handler chain + Request base_request = (request instanceof Request) ? (Request)request:HttpConnection.getCurrentConnection().getRequest(); + base_request.setHandled(true); + + return; + } + + if (tmp instanceof Image) { + final Image i = (Image) tmp; + final String mimeType = MimeTable.ext2mime(targetExt, "text/html"); + + // generate an byte array from the generated image + int width = i.getWidth(null); if (width < 0) width = 96; // bad hack + int height = i.getHeight(null); if (height < 0) height = 96; // bad hack + final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + bi.createGraphics().drawImage(i, 0, 0, width, height, null); + final ByteBuffer result = RasterPlotter.exportImage(bi, targetExt); + + response.setContentType(targetExt); + response.setContentLength(result.length()); + response.setStatus(HttpServletResponse.SC_OK); + + result.writeTo(response.getOutputStream()); + + // we handled this request, break out of handler chain + Request base_request = (request instanceof Request) ? (Request)request:HttpConnection.getCurrentConnection().getRequest(); + base_request.setHandled(true); + + return; + } + + servletProperties templatePatterns = null; if (tmp == null) { // if no args given, then tp will be an empty Hashtable object (not null) @@ -158,13 +257,41 @@ public class TemplateHandler extends AbstractHandler { } else { templatePatterns = new servletProperties((serverObjects) tmp); } + // add the application version, the uptime and the client name to every rewrite table + templatePatterns.put(servletProperties.PEER_STAT_VERSION, yacyBuildProperties.getVersion()); + templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() - serverCore.startupTime) / 1000) / 60); // uptime in minutes + templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName()); + templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, sb.peers.myID()); + templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format()); + yacySeed myPeer = sb.peers.mySeed(); + templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1); + templatePatterns.putHTML("newpeer_peerhash", myPeer.hash); - response.setContentType("text/html"); - response.setStatus(HttpServletResponse.SC_OK); - response.getWriter().println("

Hello OneHandler

"); - - Request base_request = (request instanceof Request) ? (Request)request:HttpConnection.getCurrentConnection().getRequest(); - base_request.setHandled(true); + if(targetFile.exists() && targetFile.isFile() && targetFile.canRead()) { + String mimeType = MimeTable.ext2mime(targetExt, "text/html"); + + InputStream fis = null; + long fileSize = targetFile.length(); + + if (fileSize <= Math.min(4 * 1024 * 1204, MemoryControl.available() / 100)) { + // read file completely into ram, avoid that too many files are open at the same time + fis = new ByteArrayInputStream(FileUtils.read(targetFile)); + } else { + fis = new BufferedInputStream(new FileInputStream(targetFile)); + } + + // set response header + response.setContentType(mimeType); + response.setStatus(HttpServletResponse.SC_OK); + + // apply templates + TemplateEngine.writeTemplate(fis, response.getOutputStream(), templatePatterns, "-UNRESOLVED_PATTERN-".getBytes("UTF-8")); + fis.close(); + + // we handled this request, break out of handler chain + Request base_request = (request instanceof Request) ? (Request)request:HttpConnection.getCurrentConnection().getRequest(); + base_request.setHandled(true); + } } } diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index e35c443ef..de71661ac 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -55,6 +55,7 @@ import net.yacy.cora.storage.DynamicScore; import net.yacy.cora.storage.ScoreCluster; import net.yacy.gui.YaCyApp; import net.yacy.gui.framework.Browser; +import net.yacy.http.HttpServer; import net.yacy.kelondro.blob.MapDataMining; //import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; @@ -290,21 +291,24 @@ public final class yacy { HTTPClient.setDefaultUserAgent(userAgent); // start main threads - final String port = sb.getConfig("port", "8090"); + final long port = sb.getConfigLong("port", 8090); try { - final HTTPDemon protocolHandler = new HTTPDemon(sb); - final serverCore server = new serverCore( - timeout /*control socket timeout in milliseconds*/, - true /* block attacks (wrong protocol) */, - protocolHandler /*command class*/, - sb, - 30000 /*command max length incl. GET args*/); - server.setName("httpd:"+port); - server.setPriority(Thread.MAX_PRIORITY); - server.setObeyIntermission(false); + // start jetty http server + HttpServer httpServer = new HttpServer((int)port); + httpServer.start(); + //final HTTPDemon protocolHandler = new HTTPDemon(sb); + //final serverCore server = new serverCore( + // timeout /*control socket timeout in milliseconds*/, + // true /* block attacks (wrong protocol) */, + // protocolHandler /*command class*/, + // sb, + // 30000 /*command max length incl. GET args*/); + //server.setName("httpd:"+port); + //server.setPriority(Thread.MAX_PRIORITY); + //server.setObeyIntermission(false); // start the server - sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0); + //sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0); //server.start(); // open the browser window @@ -313,7 +317,7 @@ public final class yacy { final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html"); //boolean properPW = (sb.getConfig("adminAccount", "").length() == 0) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0); //if (!properPW) browserPopUpPage = "ConfigBasic.html"; - Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage); + Browser.openBrowser((false?"https":"http") + "://localhost:" + port + "/" + browserPopUpPage); } // unlock yacyTray browser popup @@ -387,9 +391,10 @@ public final class yacy { // shut down if (RowCollection.sortingthreadexecutor != null) RowCollection.sortingthreadexecutor.shutdown(); Log.logConfig("SHUTDOWN", "caught termination signal"); - server.terminate(false); - server.interrupt(); - server.close(); + //server.terminate(false); + //server.interrupt(); + //server.close(); + httpServer.stop(); /* if (server.isAlive()) try { // TODO only send request, don't read response (cause server is already down resulting in error) @@ -404,11 +409,11 @@ public final class yacy { // MultiThreadedHttpConnectionManager.shutdownAll(); // idle until the processes are down - if (server.isAlive()) { + //if (server.isAlive()) { //Thread.sleep(2000); // wait a while - server.interrupt(); + // server.interrupt(); // MultiThreadedHttpConnectionManager.shutdownAll(); - } + //} // MultiThreadedHttpConnectionManager.shutdownAll(); Log.logConfig("SHUTDOWN", "server has terminated"); sb.close();