boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

* jetty works for many pages now

Browse Source
pull/1/head
Florian Richter 14 years ago
parent 1989ba64c0
commit 2d2df64870
5 changed files with 185 additions and 44 deletions
Show Stats Download Patch File Download Diff File

2
build.xml
Unescape View File

@ -644,7 +644,7 @@
<jvmarg line="-Xdebug"/> <jvmarg line="-Xdebug"/>
<jvmarg line="-Xnoagent"/> <jvmarg line="-Xnoagent"/>
<jvmarg line="-Djava.compiler=none"/> <jvmarg line="-Djava.compiler=none"/>
<jvmarg line="-Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n"/> <jvmarg line="-Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=y"/>
</java> </java>
</target> </target>

4
htroot/Status.java
Unescape View File

@ -309,8 +309,8 @@ public class Status {
// connection information // connection information
final serverCore httpd = (serverCore) sb.getThread("10_httpd"); final serverCore httpd = (serverCore) sb.getThread("10_httpd");
prop.putNum("connectionsActive", httpd.getJobCount()); //prop.putNum("connectionsActive", httpd.getJobCount());
prop.putNum("connectionsMax", httpd.getMaxSessionCount()); //prop.putNum("connectionsMax", httpd.getMaxSessionCount());
// Queue information // Queue information
final int loaderJobCount = sb.crawlQueues.workerSize(); final int loaderJobCount = sb.crawlQueues.workerSize();

31
source/net/yacy/http/HttpServer.java
Unescape View File

@ -6,20 +6,16 @@ import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler; import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.server.nio.SelectChannelConnector; import org.eclipse.jetty.server.nio.SelectChannelConnector;
import org.eclipse.jetty.util.thread.QueuedThreadPool;
public class HttpServer { public class HttpServer {
public static void initYaCyServer() { private Server server = new Server();
// create directories
}
/** /**
* @param port TCP Port to listen for http requests * @param port TCP Port to listen for http requests
* @throws Exception
*/ */
public static void runHttpServer(int port) throws Exception { public HttpServer(int port) {
Server server = new Server(); server = new Server();
SelectChannelConnector connector = new SelectChannelConnector(); SelectChannelConnector connector = new SelectChannelConnector();
connector.setPort(port); connector.setPort(port);
//connector.setThreadPool(new QueuedThreadPool(20)); //connector.setThreadPool(new QueuedThreadPool(20));
@ -29,21 +25,34 @@ public class HttpServer {
resource_handler.setDirectoriesListed(true); resource_handler.setDirectoriesListed(true);
resource_handler.setWelcomeFiles(new String[]{ "index.html" }); resource_handler.setWelcomeFiles(new String[]{ "index.html" });
resource_handler.setResourceBase("DATA/HTDOCS"); resource_handler.setResourceBase("htroot/");
HandlerList handlers = new HandlerList(); HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] { new TemplateHandler(), resource_handler, new DefaultHandler() }); handlers.setHandlers(new Handler[] { new TemplateHandler(), resource_handler, new DefaultHandler() });
server.setHandler(handlers); server.setHandler(handlers);
}
public void start() throws Exception {
server.start(); server.start();
}
public void initYaCyServer() {
// TODO: delete?
// create directories
}
public void stop() throws Exception {
server.stop();
server.join(); server.join();
} }
/** /**
* just for testing and debugging * just for testing and debugging
*/ */
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
runHttpServer(8080); HttpServer server = new HttpServer(8090);
server.start();
server.stop();
} }
} }

149
source/net/yacy/http/TemplateHandler.java
Unescape View File

@ -1,28 +1,49 @@
package net.yacy.http; package net.yacy.http;
import java.awt.Image;
import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.lang.ref.SoftReference; import java.lang.ref.SoftReference;
import java.lang.reflect.InvocationTargetException; import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.util.Date; import java.util.Date;
import java.util.Enumeration;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.visualization.RasterPlotter;
import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.server.HttpConnection; import org.eclipse.jetty.server.HttpConnection;
import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.server.handler.AbstractHandler;
import de.anomic.data.MimeTable;
import de.anomic.http.server.HTTPDemon;
import de.anomic.http.server.TemplateEngine;
import de.anomic.search.Switchboard;
import de.anomic.server.serverClassLoader; import de.anomic.server.serverClassLoader;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.server.servletProperties; import de.anomic.server.servletProperties;
import de.anomic.yacy.yacyBuildProperties;
import de.anomic.yacy.yacySeed;
public class TemplateHandler extends AbstractHandler { public class TemplateHandler extends AbstractHandler {
@ -35,6 +56,8 @@ public class TemplateHandler extends AbstractHandler {
boolean useTemplateCache = false; boolean useTemplateCache = false;
private ConcurrentHashMap<File, SoftReference<TemplateCacheEntry>> templateCache = null; private ConcurrentHashMap<File, SoftReference<TemplateCacheEntry>> templateCache = null;
private ConcurrentHashMap<File, SoftReference<Method>> templateMethodCache = null; private ConcurrentHashMap<File, SoftReference<Method>> templateMethodCache = null;
private MimeTypes mimeTypes = new MimeTypes();
@Override @Override
protected void doStart() throws Exception { protected void doStart() throws Exception {
@ -119,22 +142,54 @@ public class TemplateHandler extends AbstractHandler {
} }
private final Object invokeServlet(final File targetClass, final HttpServletRequest request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { private final Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, null}); // add switchboard return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, Switchboard.getSwitchboard()}); // add switchboard
}
private RequestHeader generateLegacyRequestHeader(HttpServletRequest request, String target, String targetExt) {
RequestHeader legacyRequestHeader = new RequestHeader();
@SuppressWarnings("unchecked")
Enumeration<String> headers = request.getHeaderNames();
while (headers.hasMoreElements()) {
String headerName = headers.nextElement();
@SuppressWarnings("unchecked")
Enumeration<String> header = request.getHeaders(headerName);
while(header.hasMoreElements())
legacyRequestHeader.add(headerName, header.nextElement());
}
legacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, request.getRemoteAddr());
legacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, target);
legacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, targetExt);
return legacyRequestHeader;
} }
@Override @Override
public void handle(String target, Request baseRequest, HttpServletRequest request, public void handle(String target, Request baseRequest, HttpServletRequest request,
HttpServletResponse response) throws IOException, ServletException { HttpServletResponse response) throws IOException, ServletException {
Switchboard sb = Switchboard.getSwitchboard();
System.err.println("Page: " + target);
String localeSelection = "default"; String localeSelection = "default";
File targetFile = getLocalizedFile(target, localeSelection); File targetFile = getLocalizedFile(target, localeSelection);
File targetClass = rewriteClassFile(new File(htDefaultPath, target)); File targetClass = rewriteClassFile(new File(htDefaultPath, target));
String targetExt = target.substring(target.lastIndexOf('.') + 1, target.length());
if ((targetClass != null)) { if ((targetClass != null)) {
serverObjects args = new serverObjects(request.getParameterMap()); @SuppressWarnings("unchecked")
serverObjects args = new serverObjects();
@SuppressWarnings("unchecked")
Enumeration<String> argNames = request.getParameterNames();
while (argNames.hasMoreElements()) {
String argName = argNames.nextElement();
args.put(argName, request.getParameter(argName));
}
RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
Object tmp; Object tmp;
try { try {
tmp = invokeServlet(targetClass, request, args); tmp = invokeServlet(targetClass, legacyRequestHeader, args);
} catch (InvocationTargetException e) { } catch (InvocationTargetException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
@ -147,8 +202,52 @@ public class TemplateHandler extends AbstractHandler {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
throw new ServletException(); throw new ServletException();
} }
if (tmp instanceof RasterPlotter) {
final RasterPlotter yp = (RasterPlotter) tmp;
// send an image to client
final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
final ByteBuffer result = RasterPlotter.exportImage(yp.getImage(), "png");
response.setContentType(mimeType);
response.setContentLength(result.length());
response.setStatus(HttpServletResponse.SC_OK);
result.writeTo(response.getOutputStream());
// we handled this request, break out of handler chain
Request base_request = (request instanceof Request) ? (Request)request:HttpConnection.getCurrentConnection().getRequest();
base_request.setHandled(true);
return;
}
if (tmp instanceof Image) {
final Image i = (Image) tmp;
final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
// generate an byte array from the generated image
int width = i.getWidth(null); if (width < 0) width = 96; // bad hack
int height = i.getHeight(null); if (height < 0) height = 96; // bad hack
final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
bi.createGraphics().drawImage(i, 0, 0, width, height, null);
final ByteBuffer result = RasterPlotter.exportImage(bi, targetExt);
response.setContentType(targetExt);
response.setContentLength(result.length());
response.setStatus(HttpServletResponse.SC_OK);
result.writeTo(response.getOutputStream());
// we handled this request, break out of handler chain
Request base_request = (request instanceof Request) ? (Request)request:HttpConnection.getCurrentConnection().getRequest();
base_request.setHandled(true);
return;
}
servletProperties templatePatterns = null; servletProperties templatePatterns = null;
if (tmp == null) { if (tmp == null) {
// if no args given, then tp will be an empty Hashtable object (not null) // if no args given, then tp will be an empty Hashtable object (not null)
@ -158,13 +257,41 @@ public class TemplateHandler extends AbstractHandler {
} else { } else {
templatePatterns = new servletProperties((serverObjects) tmp); templatePatterns = new servletProperties((serverObjects) tmp);
} }
// add the application version, the uptime and the client name to every rewrite table
templatePatterns.put(servletProperties.PEER_STAT_VERSION, yacyBuildProperties.getVersion());
templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() - serverCore.startupTime) / 1000) / 60); // uptime in minutes
templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName());
templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, sb.peers.myID());
templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
yacySeed myPeer = sb.peers.mySeed();
templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1);
templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);
response.setContentType("text/html"); if(targetFile.exists() && targetFile.isFile() && targetFile.canRead()) {
response.setStatus(HttpServletResponse.SC_OK); String mimeType = MimeTable.ext2mime(targetExt, "text/html");
response.getWriter().println("<h1>Hello OneHandler</h1>");
InputStream fis = null;
Request base_request = (request instanceof Request) ? (Request)request:HttpConnection.getCurrentConnection().getRequest(); long fileSize = targetFile.length();
base_request.setHandled(true);
if (fileSize <= Math.min(4 * 1024 * 1204, MemoryControl.available() / 100)) {
// read file completely into ram, avoid that too many files are open at the same time
fis = new ByteArrayInputStream(FileUtils.read(targetFile));
} else {
fis = new BufferedInputStream(new FileInputStream(targetFile));
}
// set response header
response.setContentType(mimeType);
response.setStatus(HttpServletResponse.SC_OK);
// apply templates
TemplateEngine.writeTemplate(fis, response.getOutputStream(), templatePatterns, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
fis.close();
// we handled this request, break out of handler chain
Request base_request = (request instanceof Request) ? (Request)request:HttpConnection.getCurrentConnection().getRequest();
base_request.setHandled(true);
}
} }
} }

43
source/net/yacy/yacy.java
Unescape View File

@ -55,6 +55,7 @@ import net.yacy.cora.storage.DynamicScore;
import net.yacy.cora.storage.ScoreCluster; import net.yacy.cora.storage.ScoreCluster;
import net.yacy.gui.YaCyApp; import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser; import net.yacy.gui.framework.Browser;
import net.yacy.http.HttpServer;
import net.yacy.kelondro.blob.MapDataMining; import net.yacy.kelondro.blob.MapDataMining;
//import net.yacy.kelondro.data.meta.DigestURI; //import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.meta.URIMetadataRow;
@ -290,21 +291,24 @@ public final class yacy {
HTTPClient.setDefaultUserAgent(userAgent); HTTPClient.setDefaultUserAgent(userAgent);
// start main threads // start main threads
final String port = sb.getConfig("port", "8090"); final long port = sb.getConfigLong("port", 8090);
try { try {
final HTTPDemon protocolHandler = new HTTPDemon(sb); // start jetty http server
final serverCore server = new serverCore( HttpServer httpServer = new HttpServer((int)port);
timeout /*control socket timeout in milliseconds*/, httpServer.start();
true /* block attacks (wrong protocol) */, //final HTTPDemon protocolHandler = new HTTPDemon(sb);
protocolHandler /*command class*/, //final serverCore server = new serverCore(
sb, // timeout /*control socket timeout in milliseconds*/,
30000 /*command max length incl. GET args*/); // true /* block attacks (wrong protocol) */,
server.setName("httpd:"+port); // protocolHandler /*command class*/,
server.setPriority(Thread.MAX_PRIORITY); // sb,
server.setObeyIntermission(false); // 30000 /*command max length incl. GET args*/);
//server.setName("httpd:"+port);
//server.setPriority(Thread.MAX_PRIORITY);
//server.setObeyIntermission(false);
// start the server // start the server
sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0); //sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
//server.start(); //server.start();
// open the browser window // open the browser window
@ -313,7 +317,7 @@ public final class yacy {
final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html"); final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
//boolean properPW = (sb.getConfig("adminAccount", "").length() == 0) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0); //boolean properPW = (sb.getConfig("adminAccount", "").length() == 0) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
//if (!properPW) browserPopUpPage = "ConfigBasic.html"; //if (!properPW) browserPopUpPage = "ConfigBasic.html";
Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage); Browser.openBrowser((false?"https":"http") + "://localhost:" + port + "/" + browserPopUpPage);
} }
// unlock yacyTray browser popup // unlock yacyTray browser popup
@ -387,9 +391,10 @@ public final class yacy {
// shut down // shut down
if (RowCollection.sortingthreadexecutor != null) RowCollection.sortingthreadexecutor.shutdown(); if (RowCollection.sortingthreadexecutor != null) RowCollection.sortingthreadexecutor.shutdown();
Log.logConfig("SHUTDOWN", "caught termination signal"); Log.logConfig("SHUTDOWN", "caught termination signal");
server.terminate(false); //server.terminate(false);
server.interrupt(); //server.interrupt();
server.close(); //server.close();
httpServer.stop();
/* /*
if (server.isAlive()) try { if (server.isAlive()) try {
// TODO only send request, don't read response (cause server is already down resulting in error) // TODO only send request, don't read response (cause server is already down resulting in error)
@ -404,11 +409,11 @@ public final class yacy {
// MultiThreadedHttpConnectionManager.shutdownAll(); // MultiThreadedHttpConnectionManager.shutdownAll();
// idle until the processes are down // idle until the processes are down
if (server.isAlive()) { //if (server.isAlive()) {
//Thread.sleep(2000); // wait a while //Thread.sleep(2000); // wait a while
server.interrupt(); // server.interrupt();
// MultiThreadedHttpConnectionManager.shutdownAll(); // MultiThreadedHttpConnectionManager.shutdownAll();
} //}
// MultiThreadedHttpConnectionManager.shutdownAll(); // MultiThreadedHttpConnectionManager.shutdownAll();
Log.logConfig("SHUTDOWN", "server has terminated"); Log.logConfig("SHUTDOWN", "server has terminated");
sb.close(); sb.close();

Write Preview
Loading…
Cancel
Save