boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced SSI server-side support:

Browse Source 
- SSIs may now refer to servlets, not only files
- calling a servlet, the servlet/SSI engine is called recursively
- SSIs now work also for non-chunked-encoding supporting clients
This will support the new search page functionality, to show search results
dynamically without using javascript. To test this method, a test page has been added
http://localhost:8080/ssitest.html
..calls dynamicalls 3 servlets, which produce some delays during their execution
please verify that you can see the result step-by-step on your browser
To implement this feature, some refactoring had been taken place, mostly code
had been made static and will execute faster.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4037 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent c8e5a4a6b7
commit 24e25e1141
12 changed files with 337 additions and 529 deletions
Show Stats Download Patch File Download Diff File

7
htroot/TestApplet.java
Unescape View File

@ -23,7 +23,6 @@ import java.util.Iterator;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpdFileHandler; import de.anomic.http.httpdFileHandler;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverByteBuffer; import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -31,8 +30,6 @@ import de.anomic.server.serverSwitch;
public class TestApplet { public class TestApplet {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
plasmaSwitchboard sb = (plasmaSwitchboard) env;
httpdFileHandler filehandler=new httpdFileHandler(sb);
if(post== null || !post.containsKey("url")){ if(post== null || !post.containsKey("url")){
prop.put("mode", "0"); prop.put("mode", "0");
@ -51,7 +48,7 @@ public class TestApplet {
prop.put("mode", "1"); prop.put("mode", "1");
//File templatefile=filehandler.getOverlayedFile((String)post.get("url")); //File templatefile=filehandler.getOverlayedFile((String)post.get("url"));
File classfile=filehandler.getOverlayedClass((String)post.get("url")); File classfile = httpdFileHandler.getOverlayedClass((String)post.get("url"));
httpHeader header2=new httpHeader(); httpHeader header2=new httpHeader();
header2.put("CLIENTIP", "127.0.0.1"); header2.put("CLIENTIP", "127.0.0.1");
header2.put("PATH", post.get("url")); header2.put("PATH", post.get("url"));
@ -61,7 +58,7 @@ public class TestApplet {
prop.put("mode_templates", "classfile does not exist"); prop.put("mode_templates", "classfile does not exist");
return prop; return prop;
} }
tp=(serverObjects)filehandler.invokeServlet(classfile, header2, args); tp=(serverObjects) httpdFileHandler.invokeServlet(classfile, header2, args);
} }
catch (IllegalArgumentException e) {} catch (IllegalArgumentException e) {}
catch (IllegalAccessException e) {} catch (IllegalAccessException e) {}

3
htroot/ssitest.html
Unescape View File

@ -7,5 +7,8 @@
<body> <body>
<h1>Dynamisches HTML mit Server Side Includes</h1> <h1>Dynamisches HTML mit Server Side Includes</h1>
<!--#include virtual="ssitest.inc" --> <!--#include virtual="ssitest.inc" -->
<!--#include virtual="ssitestservlet.html?delay=1000" -->
<!--#include virtual="ssitestservlet.html?delay=2000" -->
<!--#include virtual="ssitestservlet.html?delay=1000" -->
</body> </body>
</html> </html>

4
htroot/ssitestservlet.html
Unescape View File

@ -0,0 +1,4 @@
<p>
testservlet ok nach #[delay]# millisekunden Pause.<br>
start = #[start]#, stop = #[stop]#
</p>

55
htroot/ssitestservlet.java
Unescape View File

@ -0,0 +1,55 @@
// ssitestservlet.java
// --------------------
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 09.08.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class ssitestservlet {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
//plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
int delay = 0;
long start = System.currentTimeMillis();
if (post != null) {
delay = post.getInt("delay", 1000);
}
// make a delay to see how the ssi loads and displays this page
try {Thread.sleep(delay);} catch (InterruptedException e) {}
prop.put("delay", delay);
prop.put("start", start);
prop.put("stop", System.currentTimeMillis());
return prop;
}
}

78
source/de/anomic/http/httpSSI.java
Unescape View File

@ -27,50 +27,70 @@
package de.anomic.http; package de.anomic.http;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream;
import java.util.Properties;
import de.anomic.server.serverByteBuffer; import de.anomic.server.serverByteBuffer;
public class httpSSI { public class httpSSI {
public static void writeSSI(File referenceFile, serverByteBuffer in, httpChunkedOutputStream out) throws IOException { public static void writeSSI(serverByteBuffer in, OutputStream out) throws IOException {
writeSSI(referenceFile, in, 0, out); writeSSI(in, 0, out);
} }
public static void writeSSI(File referenceFile, serverByteBuffer in, int start, httpChunkedOutputStream out) throws IOException { public static void writeSSI(serverByteBuffer in, int off, OutputStream out) throws IOException {
int p = in.indexOf("<!--#".getBytes(), start); int p = in.indexOf("<!--#".getBytes(), off);
if (p == 0) { if (p >= 0) {
int q = in.indexOf("-->".getBytes(), start + 10);
assert q >= 0;
parseSSI(referenceFile, in, start, q + 3 - start, out);
writeSSI(referenceFile, in, start + q + 3, out);
} else if (p > 0) {
int q = in.indexOf("-->".getBytes(), p + 10); int q = in.indexOf("-->".getBytes(), p + 10);
out.write(in, start, p - start); if (out instanceof httpChunkedOutputStream) {
parseSSI(referenceFile, in, start + p, q + 3 - start - p, out); ((httpChunkedOutputStream) out).write(in, off, p - off);
writeSSI(referenceFile, in, start + q + 3, out); } else {
out.write(in.getBytes(off, p - off));
}
parseSSI(in, p, q + 3 - p, out);
writeSSI(in, q + 3, out);
} else /* p < 0 */ { } else /* p < 0 */ {
out.write(in, start, in.length() - start); if (out instanceof httpChunkedOutputStream) {
((httpChunkedOutputStream) out).write(in, off, in.length() - off);
} else {
out.write(in.getBytes(off, in.length() - off));
}
} }
} }
private static void parseSSI(File referenceFile, serverByteBuffer in, int start, int length, httpChunkedOutputStream out) { private static void parseSSI(serverByteBuffer in, int off, int len, OutputStream out) {
if (in.startsWith("<!--#include virtual=\"".getBytes(), start)) { if (in.startsWith("<!--#include virtual=\"".getBytes(), off)) {
int q = in.indexOf("\"".getBytes(), start + 22); int q = in.indexOf("\"".getBytes(), off + 22);
if (q > 0) { if (q > 0) {
String path = in.toString(start + 22, q); String path = in.toString(off + 22, q);
File loadFile = new File(referenceFile.getParentFile(), path); writeContent(path, out);
try {
out.write(new FileInputStream(loadFile));
} catch (FileNotFoundException e) {
// do nothing
} catch (IOException e) {
// do nothing
}
} }
} }
} }
private static void writeContent(String path, OutputStream out) {
// check if there are arguments in path string
String args = "";
int argpos = path.indexOf('?');
if (argpos > 0) {
args = path.substring(argpos + 1);
path = path.substring(0, argpos);
}
// set up virtual connection properties to call httpdFileHander.doGet()
Properties conProp = new Properties();
httpHeader header = new httpHeader(httpd.reverseMappingCache);
conProp.setProperty(httpHeader.CONNECTION_PROP_METHOD, httpHeader.METHOD_GET);
conProp.setProperty(httpHeader.CONNECTION_PROP_PATH, path);
conProp.setProperty(httpHeader.CONNECTION_PROP_ARGS, args);
conProp.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9);
conProp.setProperty("CLIENTIP", "127.0.0.1");
try {
httpdFileHandler.doGet(conProp, header, out);
} catch (IOException e) {
// do nothing
}
}
} }

32
source/de/anomic/http/httpd.java
Unescape View File

@ -78,6 +78,7 @@ import de.anomic.server.serverHandler;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.soap.httpdSoapHandler;
import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeed;
@ -118,9 +119,7 @@ public final class httpd implements serverHandler {
public static final String hline = "-------------------------------------------------------------------------------"; public static final String hline = "-------------------------------------------------------------------------------";
public static HashMap reverseMappingCache = new HashMap(); public static HashMap reverseMappingCache = new HashMap();
private httpdHandler proxyHandler = null; // a servlet that holds the proxy functions private httpdSoapHandler soapHandler = null;
private httpdHandler fileHandler = null; // a servlet that holds the file serving functions
private httpdHandler soapHandler = null;
private static plasmaSwitchboard switchboard = null; private static plasmaSwitchboard switchboard = null;
private static String virtualHost = null; private static String virtualHost = null;
@ -151,11 +150,9 @@ public final class httpd implements serverHandler {
private final serverLog log = new serverLog("HTTPD"); private final serverLog log = new serverLog("HTTPD");
// class methods // class methods
public httpd(serverSwitch s, httpdHandler fileHandler, httpdHandler proxyHandler) { public httpd(serverSwitch s) {
// handler info // handler info
httpd.switchboard = (plasmaSwitchboard)s; httpd.switchboard = (plasmaSwitchboard)s;
this.fileHandler = fileHandler;
this.proxyHandler = proxyHandler;
httpd.virtualHost = switchboard.getConfig("fileHost","localhost"); httpd.virtualHost = switchboard.getConfig("fileHost","localhost");
// authentication: by default none // authentication: by default none
@ -500,7 +497,7 @@ public final class httpd implements serverHandler {
try { try {
Class soapHandlerClass = Class.forName("de.anomic.soap.httpdSoapHandler"); Class soapHandlerClass = Class.forName("de.anomic.soap.httpdSoapHandler");
Constructor classConstructor = soapHandlerClass.getConstructor( new Class[] { serverSwitch.class } ); Constructor classConstructor = soapHandlerClass.getConstructor( new Class[] { serverSwitch.class } );
this.soapHandler = (httpdHandler) classConstructor.newInstance(new Object[] { switchboard }); this.soapHandler = (httpdSoapHandler) classConstructor.newInstance(new Object[] { switchboard });
} catch (Exception e) { } catch (Exception e) {
sendRespondError(this.prop,this.session.out,4,501,null,"Error while initializing SOAP Excension",e); sendRespondError(this.prop,this.session.out,4,501,null,"Error while initializing SOAP Excension",e);
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
@ -519,8 +516,7 @@ public final class httpd implements serverHandler {
*/ */
} else { } else {
if (this.handleServerAuthentication(header)) { if (this.handleServerAuthentication(header)) {
if (fileHandler == null) fileHandler = new httpdFileHandler(switchboard); httpdFileHandler.doGet(this.prop, header, this.session.out);
fileHandler.doGet(this.prop, header, this.session.out);
} }
} }
} else { } else {
@ -532,7 +528,7 @@ public final class httpd implements serverHandler {
// pass to proxy // pass to proxy
if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) || if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) ||
((this.allowProxy) && (handleProxyAuthentication(header)))) { ((this.allowProxy) && (handleProxyAuthentication(header)))) {
proxyHandler.doGet(this.prop, header, this.session.out); httpdProxyHandler.doGet(this.prop, header, this.session.out);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
this.session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.crlfString + serverCore.crlfString + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + clientIP + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.crlfString).getBytes()); this.session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.crlfString + serverCore.crlfString + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + clientIP + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.crlfString).getBytes());
@ -592,8 +588,7 @@ public final class httpd implements serverHandler {
// pass to server // pass to server
if (allowServer) { if (allowServer) {
if (handleServerAuthentication(header)) { if (handleServerAuthentication(header)) {
if (fileHandler == null) fileHandler = new httpdFileHandler(switchboard); httpdFileHandler.doHead(prop, header, this.session.out);
fileHandler.doHead(prop, header, this.session.out);
} }
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
@ -605,7 +600,7 @@ public final class httpd implements serverHandler {
// pass to proxy // pass to proxy
if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) || if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) ||
((this.allowProxy) && (handleProxyAuthentication(header)))) { ((this.allowProxy) && (handleProxyAuthentication(header)))) {
proxyHandler.doHead(prop, header, this.session.out); httpdProxyHandler.doHead(prop, header, this.session.out);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + session.out.write((httpVersion + " 403 refused (IP not granted)" +
@ -656,7 +651,7 @@ public final class httpd implements serverHandler {
Constructor soapHandlerConstructor = soapHandlerClass.getConstructor( new Class[] { serverSwitch.class } ); Constructor soapHandlerConstructor = soapHandlerClass.getConstructor( new Class[] { serverSwitch.class } );
// creating the new object // creating the new object
this.soapHandler = (httpdHandler)soapHandlerConstructor.newInstance( new Object[] { switchboard } ); this.soapHandler = (httpdSoapHandler)soapHandlerConstructor.newInstance( new Object[] { switchboard } );
} catch (Exception e) { } catch (Exception e) {
sendRespondError(this.prop,this.session.out,4,501,null,"Error while initializing SOAP Excension",e); sendRespondError(this.prop,this.session.out,4,501,null,"Error while initializing SOAP Excension",e);
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
@ -674,8 +669,7 @@ public final class httpd implements serverHandler {
*/ */
} else { } else {
if (handleServerAuthentication(header)) { if (handleServerAuthentication(header)) {
if (fileHandler == null) fileHandler = new httpdFileHandler(switchboard); httpdFileHandler.doPost(prop, header, this.session.out, this.session.in);
fileHandler.doPost(prop, header, this.session.out, this.session.in);
} }
} }
} else { } else {
@ -687,7 +681,7 @@ public final class httpd implements serverHandler {
// pass to proxy // pass to proxy
if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) || if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) ||
((this.allowProxy) && (handleProxyAuthentication(header)))) { ((this.allowProxy) && (handleProxyAuthentication(header)))) {
proxyHandler.doPost(prop, header, this.session.out, this.session.in); httpdProxyHandler.doPost(prop, header, this.session.out, this.session.in);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.crlfString + serverCore.crlfString + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + clientIP + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.crlfString).getBytes()); session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.crlfString + serverCore.crlfString + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + clientIP + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.crlfString).getBytes());
@ -754,7 +748,7 @@ public final class httpd implements serverHandler {
// pass to proxy // pass to proxy
if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) || if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) ||
((this.allowProxy) && (this.handleProxyAuthentication(header)))) { ((this.allowProxy) && (this.handleProxyAuthentication(header)))) {
proxyHandler.doConnect(prop, header, this.session.in, this.session.out); httpdProxyHandler.doConnect(prop, header, this.session.in, this.session.out);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.crlfString + serverCore.crlfString + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + clientIP + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.crlfString).getBytes()); session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.crlfString + serverCore.crlfString + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + clientIP + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.crlfString).getBytes());
@ -1106,7 +1100,7 @@ public final class httpd implements serverHandler {
} }
public Object clone() { public Object clone() {
return new httpd(switchboard, new httpdFileHandler(switchboard), new httpdProxyHandler(switchboard)); return new httpd(switchboard);
} }
public static final void sendRespondBody( public static final void sendRespondBody(

79
source/de/anomic/http/httpdAbstractHandler.java
Unescape View File

@ -1,79 +0,0 @@
// httpdAbstractHandler.java
// -----------------------
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 25.10.2004
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
/*
Documentation:
the servlet interface
an actual servlet for the AnomicHTTPD must provide a class that implements
this interface. The resulting class is then placed in a folder that contains
all servlets and is configured in the httpd.conf configuration file.
servlet classes in that directory are then automatically selected as CGI
extensions to the server.
The core functionality of file serving is also implemented as servlet.
*/
package de.anomic.http;
import java.text.SimpleDateFormat;
import java.util.Properties;
import de.anomic.server.logging.serverLog;
public abstract class httpdAbstractHandler {
// static tools
private static int fileCounter = 0; // for unique file names
private static SimpleDateFormat DateFileNameFormatter =
new SimpleDateFormat("yyyyMMddHHmmss");
protected static String uniqueDateString() {
String c = Integer.toString(fileCounter);
fileCounter++; if (fileCounter>9999) fileCounter = 0;
while (c.length() < 4) { c = "0" + c; }
return "FILE" + DateFileNameFormatter.format(httpc.nowDate()) + c;
}
protected Properties connectionProperties = null;
protected serverLog theLogger;
}

108
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -113,7 +113,7 @@ import de.anomic.server.servletProperties;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.ymage.ymageMatrix; import de.anomic.ymage.ymageMatrix;
public final class httpdFileHandler extends httpdAbstractHandler implements httpdHandler { public final class httpdFileHandler {
private static final boolean safeServletsMode = false; // if true then all servlets are called synchronized private static final boolean safeServletsMode = false; // if true then all servlets are called synchronized
@ -136,19 +136,20 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
public static boolean useTemplateCache = false; public static boolean useTemplateCache = false;
//private Properties connectionProperties = null;
private static serverLog theLogger;
static { static {
useTemplateCache = plasmaSwitchboard.getSwitchboard().getConfig("enableTemplateCache","true").equalsIgnoreCase("true"); serverSwitch switchboard = plasmaSwitchboard.getSwitchboard();
useTemplateCache = switchboard.getConfig("enableTemplateCache","true").equalsIgnoreCase("true");
templateCache = (useTemplateCache)? new HashMap() : new HashMap(0); templateCache = (useTemplateCache)? new HashMap() : new HashMap(0);
templateMethodCache = (useTemplateCache) ? new HashMap() : new HashMap(0); templateMethodCache = (useTemplateCache) ? new HashMap() : new HashMap(0);
// create a class loader // create a class loader
provider = new serverClassLoader(/*this.getClass().getClassLoader()*/); provider = new serverClassLoader(/*this.getClass().getClassLoader()*/);
}
public httpdFileHandler(serverSwitch switchboard) {
// creating a logger // creating a logger
this.theLogger = new serverLog("FILEHANDLER"); theLogger = new serverLog("FILEHANDLER");
if (httpdFileHandler.switchboard == null) { if (httpdFileHandler.switchboard == null) {
httpdFileHandler.switchboard = switchboard; httpdFileHandler.switchboard = switchboard;
@ -244,7 +245,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// out.flush(); // out.flush();
// } // }
private httpHeader getDefaultHeaders(String path) { private static final httpHeader getDefaultHeaders(String path) {
httpHeader headers = new httpHeader(); httpHeader headers = new httpHeader();
String ext; String ext;
int pos; int pos;
@ -261,21 +262,19 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
return headers; return headers;
} }
public void doGet(Properties conProp, httpHeader requestHeader, OutputStream response) throws IOException { public static void doGet(Properties conProp, httpHeader requestHeader, OutputStream response) throws IOException {
doResponse(conProp, requestHeader, response, null); doResponse(conProp, requestHeader, response, null);
} }
public void doHead(Properties conProp, httpHeader requestHeader, OutputStream response) throws IOException { public static void doHead(Properties conProp, httpHeader requestHeader, OutputStream response) throws IOException {
doResponse(conProp, requestHeader, response, null); doResponse(conProp, requestHeader, response, null);
} }
public void doPost(Properties conProp, httpHeader requestHeader, OutputStream response, PushbackInputStream body) throws IOException { public static void doPost(Properties conProp, httpHeader requestHeader, OutputStream response, PushbackInputStream body) throws IOException {
doResponse(conProp, requestHeader, response, body); doResponse(conProp, requestHeader, response, body);
} }
public void doResponse(Properties conProp, httpHeader requestHeader, OutputStream out, InputStream body) { public static void doResponse(Properties conProp, httpHeader requestHeader, OutputStream out, InputStream body) {
this.connectionProperties = conProp;
String path = null; String path = null;
try { try {
@ -456,7 +455,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
targetFile = getOverlayedFile("/htdocsdefault/dir." + dirlistFormat); targetFile = getOverlayedFile("/htdocsdefault/dir." + dirlistFormat);
targetClass=getOverlayedClass("/htdocsdefault/dir." + dirlistFormat); targetClass=getOverlayedClass("/htdocsdefault/dir." + dirlistFormat);
if(! (( targetFile != null && targetFile.exists()) && ( targetClass != null && targetClass.exists())) ){ if(! (( targetFile != null && targetFile.exists()) && ( targetClass != null && targetClass.exists())) ){
httpd.sendRespondError(this.connectionProperties,out,3,500,"dir." + dirlistFormat + " or dir.class not found.",null,null); httpd.sendRespondError(conProp, out, 3, 500, "dir." + dirlistFormat + " or dir.class not found.", null, null);
} }
} }
}else{ }else{
@ -483,7 +482,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// in case that there are no args given, args = null or empty hashmap // in case that there are no args given, args = null or empty hashmap
img = invokeServlet(targetClass, requestHeader, args); img = invokeServlet(targetClass, requestHeader, args);
} catch (InvocationTargetException e) { } catch (InvocationTargetException e) {
this.theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" + theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
e.getMessage() + e.getMessage() +
" target exception at " + targetClass + ": " + " target exception at " + targetClass + ": " +
e.getTargetException().toString() + ":" + e.getTargetException().toString() + ":" +
@ -493,7 +492,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
} }
if (img == null) { if (img == null) {
// error with image generation; send file-not-found // error with image generation; send file-not-found
httpd.sendRespondError(this.connectionProperties,out,3,404,"File not Found",null,null); httpd.sendRespondError(conProp, out, 3, 404, "File not Found", null, null);
} else { } else {
if (img instanceof ymageMatrix) { if (img instanceof ymageMatrix) {
ymageMatrix yp = (ymageMatrix) img; ymageMatrix yp = (ymageMatrix) img;
@ -514,10 +513,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
baos = null; baos = null;
// write the array to the client // write the array to the client
httpd.sendRespondHeader(this.connectionProperties, out, httpd.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length, targetDate, null, null, null, null, nocache);
httpVersion, 200, null, mimeType,
result.length, targetDate, null, null, null,
null, nocache);
if (!method.equals(httpHeader.METHOD_HEAD)) { if (!method.equals(httpHeader.METHOD_HEAD)) {
//Thread.sleep(200); // see below //Thread.sleep(200); // see below
serverFileUtils.write(result, out); serverFileUtils.write(result, out);
@ -543,10 +539,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
baos = null; baos = null;
// write the array to the client // write the array to the client
httpd.sendRespondHeader(this.connectionProperties, out, httpd.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length, targetDate, null, null, null, null, nocache);
httpVersion, 200, null, mimeType,
result.length, targetDate, null, null, null,
null, nocache);
if (!method.equals(httpHeader.METHOD_HEAD)) { if (!method.equals(httpHeader.METHOD_HEAD)) {
//Thread.sleep(200); // see below //Thread.sleep(200); // see below
serverFileUtils.write(result, out); serverFileUtils.write(result, out);
@ -560,11 +553,11 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
requestHeader.put(httpHeader.CONNECTION_PROP_INPUTSTREAM, body); requestHeader.put(httpHeader.CONNECTION_PROP_INPUTSTREAM, body);
requestHeader.put(httpHeader.CONNECTION_PROP_OUTPUTSTREAM, out); requestHeader.put(httpHeader.CONNECTION_PROP_OUTPUTSTREAM, out);
httpd.sendRespondHeader(this.connectionProperties, out, httpVersion, 200, null); httpd.sendRespondHeader(conProp, out, httpVersion, 200, null);
// in case that there are no args given, args = null or empty hashmap // in case that there are no args given, args = null or empty hashmap
/* servletProperties tp = (servlerObjects) */ invokeServlet(targetClass, requestHeader, args); /* servletProperties tp = (servlerObjects) */ invokeServlet(targetClass, requestHeader, args);
this.forceConnectionClose(); forceConnectionClose(conProp);
return; return;
} else if ((targetFile.exists()) && (targetFile.canRead())) { } else if ((targetFile.exists()) && (targetFile.canRead())) {
// we have found a file that can be written to the client // we have found a file that can be written to the client
@ -646,7 +639,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
throw new InterruptedException(e.getCause().getMessage()); throw new InterruptedException(e.getCause().getMessage());
} }
this.theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" + theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
e.getMessage() + e.getMessage() +
" target exception at " + targetClass + ": " + " target exception at " + targetClass + ": " +
e.getTargetException().toString() + ":" + e.getTargetException().toString() + ":" +
@ -683,11 +676,11 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// storing the content into the cache // storing the content into the cache
ref = new SoftReference(templateContent); ref = new SoftReference(templateContent);
templateCache.put(targetFile, ref); templateCache.put(targetFile, ref);
if (this.theLogger.isLoggable(Level.FINEST)) if (theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache MISS for file " + targetFile); theLogger.logFinest("Cache MISS for file " + targetFile);
} else { } else {
if (this.theLogger.isLoggable(Level.FINEST)) if (theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache HIT for file " + targetFile); theLogger.logFinest("Cache HIT for file " + targetFile);
} }
// creating an inputstream needed by the template // creating an inputstream needed by the template
@ -704,7 +697,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// write the array to the client // write the array to the client
// we can do that either in standard mode (whole thing completely) or in chunked mode // we can do that either in standard mode (whole thing completely) or in chunked mode
// since yacy clients do not understand chunked mode, we use this only for communication with the administrator // since yacy clients do not understand chunked mode (yet), we use this only for communication with the administrator
boolean yacyClient = requestHeader.userAgent().startsWith("yacy"); boolean yacyClient = requestHeader.userAgent().startsWith("yacy");
boolean chunked = !method.equals(httpHeader.METHOD_HEAD) && !yacyClient && httpVersion.equals(httpHeader.HTTP_VERSION_1_1); boolean chunked = !method.equals(httpHeader.METHOD_HEAD) && !yacyClient && httpVersion.equals(httpHeader.HTTP_VERSION_1_1);
if (chunked) { if (chunked) {
@ -712,38 +705,41 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
serverByteBuffer o = new serverByteBuffer(); serverByteBuffer o = new serverByteBuffer();
// apply templates // apply templates
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8")); httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
httpd.sendRespondHeader(this.connectionProperties, out, httpd.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, -1, targetDate, null, tp.getOutgoingHeader(), null, "chunked", nocache);
httpVersion, 200, null, mimeType, -1,
targetDate, null, tp.getOutgoingHeader(),
null, "chunked", nocache);
// send the content in chunked parts, see RFC 2616 section 3.6.1 // send the content in chunked parts, see RFC 2616 section 3.6.1
httpChunkedOutputStream chos = new httpChunkedOutputStream(out); httpChunkedOutputStream chos = new httpChunkedOutputStream(out);
httpSSI.writeSSI(targetFile, o, chos); httpSSI.writeSSI(o, chos);
//chos.write(result); //chos.write(result);
chos.finish(); chos.finish();
} else { } else {
// send page as whole thing, SSIs are not possible // send page as whole thing, SSIs are not possible
String contentEncoding = (zipContent) ? "gzip" : null; String contentEncoding = (zipContent) ? "gzip" : null;
// apply templates // apply templates
serverByteBuffer o1 = new serverByteBuffer();
httpTemplate.writeTemplate(fis, o1, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
serverByteBuffer o = new serverByteBuffer(); serverByteBuffer o = new serverByteBuffer();
if (zipContent) { if (zipContent) {
GZIPOutputStream zippedOut = new GZIPOutputStream(o); GZIPOutputStream zippedOut = new GZIPOutputStream(o);
httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8")); httpSSI.writeSSI(o1, zippedOut);
//httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
zippedOut.finish(); zippedOut.finish();
zippedOut.flush(); zippedOut.flush();
zippedOut.close(); zippedOut.close();
zippedOut = null; zippedOut = null;
} else { } else {
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8")); httpSSI.writeSSI(o1, o);
//httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
} }
if (method.equals(httpHeader.METHOD_HEAD)) { if (method.equals(httpHeader.METHOD_HEAD)) {
httpd.sendRespondHeader(this.connectionProperties, out, httpd.sendRespondHeader(conProp, out,
httpVersion, 200, null, mimeType, o.length(), httpVersion, 200, null, mimeType, o.length(),
targetDate, null, tp.getOutgoingHeader(), targetDate, null, tp.getOutgoingHeader(),
contentEncoding, null, nocache); contentEncoding, null, nocache);
} else { } else {
byte[] result = o.toByteArray(); // this interrupts streaming (bad idea!) byte[] result = o.toByteArray(); // this interrupts streaming (bad idea!)
httpd.sendRespondHeader(this.connectionProperties, out, httpd.sendRespondHeader(conProp, out,
httpVersion, 200, null, mimeType, result.length, httpVersion, 200, null, mimeType, result.length,
targetDate, null, tp.getOutgoingHeader(), targetDate, null, tp.getOutgoingHeader(),
contentEncoding, null, nocache); contentEncoding, null, nocache);
@ -803,9 +799,9 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
long contentLength = (zipContent)?-1:targetFile.length()-rangeStartOffset; long contentLength = (zipContent)?-1:targetFile.length()-rangeStartOffset;
String contentEncoding = (zipContent)?"gzip":null; String contentEncoding = (zipContent)?"gzip":null;
String transferEncoding = (!httpVersion.equals(httpHeader.HTTP_VERSION_1_1))?null:(zipContent)?"chunked":null; String transferEncoding = (!httpVersion.equals(httpHeader.HTTP_VERSION_1_1))?null:(zipContent)?"chunked":null;
if (!httpVersion.equals(httpHeader.HTTP_VERSION_1_1) && zipContent) forceConnectionClose(); if (!httpVersion.equals(httpHeader.HTTP_VERSION_1_1) && zipContent) forceConnectionClose(conProp);
httpd.sendRespondHeader(this.connectionProperties, out, httpVersion, statusCode, null, mimeType, contentLength, targetDate, null, header, contentEncoding, transferEncoding, nocache); httpd.sendRespondHeader(conProp, out, httpVersion, statusCode, null, mimeType, contentLength, targetDate, null, header, contentEncoding, transferEncoding, nocache);
if (!method.equals(httpHeader.METHOD_HEAD)) { if (!method.equals(httpHeader.METHOD_HEAD)) {
httpChunkedOutputStream chunkedOut = null; httpChunkedOutputStream chunkedOut = null;
@ -886,16 +882,16 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
httpd.sendRespondError(conProp,out, 4, httpStatusCode, httpStatusText, new String(errorMessage),errorExc); httpd.sendRespondError(conProp,out, 4, httpStatusCode, httpStatusText, new String(errorMessage),errorExc);
} else { } else {
// otherwise we close the connection // otherwise we close the connection
this.forceConnectionClose(); forceConnectionClose(conProp);
} }
// if it is an unexpected error we log it // if it is an unexpected error we log it
if (httpStatusCode == 500) { if (httpStatusCode == 500) {
this.theLogger.logWarning(new String(errorMessage),e); theLogger.logWarning(new String(errorMessage),e);
} }
} catch (Exception ee) { } catch (Exception ee) {
this.forceConnectionClose(); forceConnectionClose(conProp);
} }
} finally { } finally {
@ -907,7 +903,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
} }
} }
public File getOverlayedClass(String path) { public static final File getOverlayedClass(String path) {
File targetClass; File targetClass;
targetClass=rewriteClassFile(new File(htDefaultPath, path)); //works for default and localized files targetClass=rewriteClassFile(new File(htDefaultPath, path)); //works for default and localized files
if(targetClass == null || !targetClass.exists()){ if(targetClass == null || !targetClass.exists()){
@ -917,7 +913,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
return targetClass; return targetClass;
} }
public File getOverlayedFile(String path) { public static final File getOverlayedFile(String path) {
File targetFile; File targetFile;
targetFile = getLocalizedFile(path); targetFile = getLocalizedFile(path);
if (!(targetFile.exists())){ if (!(targetFile.exists())){
@ -926,13 +922,13 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
return targetFile; return targetFile;
} }
private void forceConnectionClose() { private static final void forceConnectionClose(Properties conprop) {
if (this.connectionProperties != null) { if (conprop != null) {
this.connectionProperties.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); conprop.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
} }
} }
private File rewriteClassFile(File template) { private static final File rewriteClassFile(File template) {
try { try {
String f = template.getCanonicalPath(); String f = template.getCanonicalPath();
int p = f.lastIndexOf("."); int p = f.lastIndexOf(".");
@ -947,7 +943,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
} }
} }
private final Method rewriteMethod(File classFile) { private static final Method rewriteMethod(File classFile) {
Method m = null; Method m = null;
// now make a class out of the stream // now make a class out of the stream
try { try {
@ -958,7 +954,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
if (m == null) { if (m == null) {
templateMethodCache.remove(classFile); templateMethodCache.remove(classFile);
} else { } else {
this.theLogger.logFine("Cache HIT for file " + classFile);
return m; return m;
} }
@ -977,7 +972,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// storing the method into the cache // storing the method into the cache
SoftReference ref = new SoftReference(m); SoftReference ref = new SoftReference(m);
templateMethodCache.put(classFile,ref); templateMethodCache.put(classFile,ref);
this.theLogger.logFine("Cache MISS for file " + classFile);
} }
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
@ -989,7 +983,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
return m; return m;
} }
public Object invokeServlet(File targetClass, httpHeader request, serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { public static final Object invokeServlet(File targetClass, httpHeader request, serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
Object result; Object result;
if (safeServletsMode) synchronized (switchboard) { if (safeServletsMode) synchronized (switchboard) {
result = rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard}); result = rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard});

157
source/de/anomic/http/httpdHandler.java
Unescape View File

@ -1,157 +0,0 @@
// httpdHandler.java
// -----------------------
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 03.01.2004
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
/*
Documentation:
the servlet interface
an actual servlet for the AnomicHTTPD must provide a class that implements
this interface. The resulting class is then placed in a folder that contains
all servlets and is configured in the httpd.conf configuration file.
servlet classes in that directory are then automatically selected as CGI
extensions to the server.
The core functionality of file serving is also implemented as servlet.
*/
package de.anomic.http;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PushbackInputStream;
import java.util.Properties;
public interface httpdHandler {
void doGet(Properties conProp, httpHeader header, OutputStream response) throws IOException;
/*
The GET method means retrieve whatever information (in the form of an
entity) is identified by the Request-URI. If the Request-URI refers
to a data-producing process, it is the produced data which shall be
returned as the entity in the response and not the source text of the
process, unless that text happens to be the output of the process.
The semantics of the GET method change to a "conditional GET" if the
request message includes an If-Modified-Since, If-Unmodified-Since,
If-Match, If-None-Match, or If-Range header field. A conditional GET
method requests that the entity be transferred only under the
circumstances described by the conditional header field(s). The
conditional GET method is intended to reduce unnecessary network
usage by allowing cached entities to be refreshed without requiring
multiple requests or transferring data already held by the client.
The semantics of the GET method change to a "partial GET" if the
request message includes a Range header field. A partial GET requests
that only part of the entity be transferred, as described in section
14.35. The partial GET method is intended to reduce unnecessary
network usage by allowing partially-retrieved entities to be
completed without transferring data already held by the client.
*/
void doHead(Properties conProp, httpHeader header, OutputStream response) throws IOException;
/*
The HEAD method is identical to GET except that the server MUST NOT
return a message-body in the response. The metainformation contained
in the HTTP headers in response to a HEAD request SHOULD be identical
to the information sent in response to a GET request. This method can
be used for obtaining metainformation about the entity implied by the
request without transferring the entity-body itself. This method is
often used for testing hypertext links for validity, accessibility,
and recent modification.
The response to a HEAD request MAY be cacheable in the sense that the
information contained in the response MAY be used to update a
previously cached entity from that resource. If the new field values
indicate that the cached entity differs from the current entity (as
would be indicated by a change in Content-Length, Content-MD5, ETag
or Last-Modified), then the cache MUST treat the cache entry as
stale.
*/
void doPost(Properties conProp, httpHeader header, OutputStream response, PushbackInputStream body) throws IOException;
/*
The POST method is used to request that the origin server accept the
entity enclosed in the request as a new subordinate of the resource
identified by the Request-URI in the Request-Line. POST is designed
to allow a uniform method to cover the following functions:
- Annotation of existing resources;
- Posting a message to a bulletin board, newsgroup, mailing list,
or similar group of articles;
- Providing a block of data, such as the result of submitting a
form, to a data-handling process;
- Extending a database through an append operation.
The actual function performed by the POST method is determined by the
server and is usually dependent on the Request-URI. The posted entity
is subordinate to that URI in the same way that a file is subordinate
to a directory containing it, a news article is subordinate to a
newsgroup to which it is posted, or a record is subordinate to a
database.
The action performed by the POST method might not result in a
resource that can be identified by a URI. In this case, either 200
(OK) or 204 (No Content) is the appropriate response status,
depending on whether or not the response includes an entity that
describes the result.
If a resource has been created on the origin server, the response
SHOULD be 201 (Created) and contain an entity which describes the
status of the request and refers to the new resource, and a Location
header (see section 14.30).
Responses to this method are not cacheable, unless the response
includes appropriate Cache-Control or Expires header fields. However,
the 303 (See Other) response can be used to direct the user agent to
retrieve a cacheable resource.
POST requests MUST obey the message transmission requirements set out
in section 8.2.
*/
void doConnect(Properties conProp, httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) throws IOException;
/* this is only needed for https proxies. http daemons should throw a
* UnsupportedOperationException
*/
//public long getLastModified(Properties conProp);
}

331
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -104,12 +104,11 @@ import de.anomic.server.serverCore;
import de.anomic.server.serverDomains; import de.anomic.server.serverDomains;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.server.logging.serverMiniLogFormatter; import de.anomic.server.logging.serverMiniLogFormatter;
import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyCore;
public final class httpdProxyHandler extends httpdAbstractHandler implements httpdHandler { public final class httpdProxyHandler {
// static variables // static variables
// can only be instantiated upon first instantiation of this class object // can only be instantiated upon first instantiation of this class object
@ -127,8 +126,11 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
private static htmlFilterTransformer transformer = null; private static htmlFilterTransformer transformer = null;
public static final String proxyUserAgent = "yacy (" + httpc.systemOST +") yacy.net"; public static final String proxyUserAgent = "yacy (" + httpc.systemOST +") yacy.net";
public static final String crawlerUserAgent = "yacybot (" + httpc.systemOST +") http://yacy.net/yacy/bot.html"; public static final String crawlerUserAgent = "yacybot (" + httpc.systemOST +") http://yacy.net/yacy/bot.html";
private File htRootPath = null; private static File htRootPath = null;
//private Properties connectionProperties = null;
private static serverLog theLogger;
private static boolean doAccessLogging = false; private static boolean doAccessLogging = false;
/** /**
* Do logging configuration for special proxy access log file * Do logging configuration for special proxy access log file
@ -177,73 +179,40 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
} catch (Exception e) { } catch (Exception e) {
serverLog.logSevere("PROXY","Unable to configure proxy access logging.",e); serverLog.logSevere("PROXY","Unable to configure proxy access logging.",e);
} }
}
switchboard = plasmaSwitchboard.getSwitchboard();
/**
* Special logger instance for proxy access logging much similar
* to the squid access.log file
*/
private final serverLog proxyLog = new serverLog("PROXY.access");
/**
* Reusable {@link StringBuffer} for logging
*/
private final StringBuffer logMessage = new StringBuffer();
/**
* Reusable {@link StringBuffer} to generate the useragent string
*/
private final StringBuffer userAgentStr = new StringBuffer();
// class methods
public httpdProxyHandler(serverSwitch sb) {
// creating a logger // creating a logger
this.theLogger = new serverLog("PROXY"); theLogger = new serverLog("PROXY");
if (switchboard == null) { cacheManager = switchboard.getCacheManager();
switchboard = (plasmaSwitchboard) sb;
cacheManager = switchboard.getCacheManager();
isTransparentProxy = Boolean.valueOf(switchboard.getConfig("isTransparentProxy","false")).booleanValue();
// // load remote proxy data
// remoteProxyHost = switchboard.getConfig("remoteProxyHost","");
// try {
// remoteProxyPort = Integer.parseInt(switchboard.getConfig("remoteProxyPort","3128"));
// } catch (NumberFormatException e) {
// remoteProxyPort = 3128;
// }
// remoteProxyUse = switchboard.getConfig("remoteProxyUse","false").equals("true");
// remoteProxyNoProxy = switchboard.getConfig("remoteProxyNoProxy","");
// remoteProxyNoProxyPatterns = remoteProxyNoProxy.split(",");
// set timeout
timeout = Integer.parseInt(switchboard.getConfig("proxy.clientTimeout", "10000"));
// create a htRootPath: system pages
if (htRootPath == null) {
htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot"));
if (!(htRootPath.exists())) htRootPath.mkdir();
}
// load a transformer isTransparentProxy = Boolean.valueOf(switchboard.getConfig("isTransparentProxy","false")).booleanValue();
transformer = new htmlFilterContentTransformer();
transformer.init(new File(switchboard.getRootPath(), switchboard.getConfig(plasmaSwitchboard.LIST_BLUE, "")).toString());
String f; // set timeout
// load the yellow-list timeout = Integer.parseInt(switchboard.getConfig("proxy.clientTimeout", "10000"));
f = switchboard.getConfig("proxyYellowList", null);
if (f != null) { // create a htRootPath: system pages
yellowList = serverFileUtils.loadList(new File(f)); htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot"));
this.theLogger.logConfig("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries"); if (!(htRootPath.exists())) htRootPath.mkdir();
} else {
yellowList = new HashSet(); // load a transformer
} transformer = new htmlFilterContentTransformer();
transformer.init(new File(switchboard.getRootPath(), switchboard.getConfig(plasmaSwitchboard.LIST_BLUE, "")).toString());
String f;
// load the yellow-list
f = switchboard.getConfig("proxyYellowList", null);
if (f != null) {
yellowList = serverFileUtils.loadList(new File(f));
theLogger.logConfig("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
} else {
yellowList = new HashSet();
} }
String redirectorPath=switchboard.getConfig("externalRedirector", "");
if(redirectorPath.length() >0 && redirectorEnabled==false){ String redirectorPath = switchboard.getConfig("externalRedirector", "");
if (redirectorPath.length() > 0 && redirectorEnabled == false){
try { try {
redirectorProcess=Runtime.getRuntime().exec(redirectorPath); redirectorProcess=Runtime.getRuntime().exec(redirectorPath);
redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream()); redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());
@ -255,6 +224,23 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
} }
/**
* Special logger instance for proxy access logging much similar
* to the squid access.log file
*/
private static final serverLog proxyLog = new serverLog("PROXY.access");
/**
* Reusable {@link StringBuffer} for logging
*/
private static final StringBuffer logMessage = new StringBuffer();
/**
* Reusable {@link StringBuffer} to generate the useragent string
*/
private static final StringBuffer userAgentStr = new StringBuffer();
private static String domain(String host) { private static String domain(String host) {
String domain = host; String domain = host;
int pos = domain.lastIndexOf("."); int pos = domain.lastIndexOf(".");
@ -270,7 +256,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
return domain; return domain;
} }
public void handleOutgoingCookies(httpHeader requestHeader, String targethost, String clienthost) { public static void handleOutgoingCookies(httpHeader requestHeader, String targethost, String clienthost) {
/* /*
The syntax for the header is: The syntax for the header is:
@ -289,7 +275,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
} }
public void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) { public static void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) {
/* /*
The syntax for the Set-Cookie response header is The syntax for the Set-Cookie response header is
@ -317,14 +303,12 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
* @param respond the OutputStream to the client * @param respond the OutputStream to the client
* @see de.anomic.http.httpdHandler#doGet(java.util.Properties, de.anomic.http.httpHeader, java.io.OutputStream) * @see de.anomic.http.httpdHandler#doGet(java.util.Properties, de.anomic.http.httpHeader, java.io.OutputStream)
*/ */
public void doGet(Properties conProp, httpHeader requestHeader, OutputStream respond) throws IOException { public static void doGet(Properties conProp, httpHeader requestHeader, OutputStream respond) throws IOException {
this.connectionProperties = conProp;
try { try {
// remembering the starting time of the request // remembering the starting time of the request
final Date requestDate = new Date(); // remember the time... final Date requestDate = new Date(); // remember the time...
this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis(); switchboard.proxyLastAccess = System.currentTimeMillis();
@ -389,7 +373,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_PROXY, hostlow, path)) { if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_PROXY, hostlow, path)) {
httpd.sendRespondError(conProp,respond,4,403,null, httpd.sendRespondError(conProp,respond,4,403,null,
"URL '" + hostlow + "' blocked by yacy proxy (blacklisted)",null); "URL '" + hostlow + "' blocked by yacy proxy (blacklisted)",null);
this.theLogger.logInfo("AGIS blocking of host '" + hostlow + "'"); theLogger.logInfo("AGIS blocking of host '" + hostlow + "'");
return; return;
} }
@ -472,28 +456,28 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
try { try {
String exTxt = e.getMessage(); String exTxt = e.getMessage();
if ((exTxt!=null)&&(exTxt.startsWith("Socket closed"))) { if ((exTxt!=null)&&(exTxt.startsWith("Socket closed"))) {
this.forceConnectionClose(); forceConnectionClose(conProp);
} else if (!conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { } else if (!conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
String errorMsg = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage(); String errorMsg = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage();
httpd.sendRespondError(conProp,respond,4,501,null,errorMsg,e); httpd.sendRespondError(conProp,respond,4,501,null,errorMsg,e);
this.theLogger.logSevere(errorMsg); theLogger.logSevere(errorMsg);
} else { } else {
this.forceConnectionClose(); forceConnectionClose(conProp);
} }
} catch (Exception ee) { } catch (Exception ee) {
this.forceConnectionClose(); forceConnectionClose(conProp);
} }
} finally { } finally {
try { respond.flush(); } catch (Exception e) {} try { respond.flush(); } catch (Exception e) {}
if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish(); if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish();
this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis())); conProp.put(httpHeader.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis()));
this.connectionProperties.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount())); conProp.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount()));
this.logProxyAccess(); logProxyAccess(conProp);
} }
} }
private void fulfillRequestFromWeb(Properties conProp, URL url,String ext, httpHeader requestHeader, httpHeader cachedResponseHeader, File cacheFile, OutputStream respond) { private static void fulfillRequestFromWeb(Properties conProp, URL url,String ext, httpHeader requestHeader, httpHeader cachedResponseHeader, File cacheFile, OutputStream respond) {
GZIPOutputStream gzippedOut = null; GZIPOutputStream gzippedOut = null;
httpChunkedOutputStream chunkedOut = null; httpChunkedOutputStream chunkedOut = null;
@ -533,7 +517,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
remote = (yAddress == null) ? newhttpc(host, port, timeout) : newhttpc(yAddress, timeout); remote = (yAddress == null) ? newhttpc(host, port, timeout) : newhttpc(yAddress, timeout);
// removing hop by hop headers // removing hop by hop headers
this.removeHopByHopHeaders(requestHeader); removeHopByHopHeaders(requestHeader);
// adding additional headers // adding additional headers
setViaHeader(requestHeader, httpVer); setViaHeader(requestHeader, httpVer);
@ -603,14 +587,14 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
(plasmaParser.supportedRealTimeContent(url,res.responseHeader.mime())) (plasmaParser.supportedRealTimeContent(url,res.responseHeader.mime()))
) { ) {
// make a transformer // make a transformer
this.theLogger.logFine("create transformer for URL " + url); theLogger.logFine("create transformer for URL " + url);
//hfos = new htmlFilterOutputStream((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), null, transformer, (ext.length() == 0)); //hfos = new htmlFilterOutputStream((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), null, transformer, (ext.length() == 0));
String charSet = res.responseHeader.getCharacterEncoding(); String charSet = res.responseHeader.getCharacterEncoding();
if (charSet == null) charSet = httpHeader.DEFAULT_CHARSET; if (charSet == null) charSet = httpHeader.DEFAULT_CHARSET;
hfos = new htmlFilterWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond),charSet, null, transformer, (ext.length() == 0)); hfos = new htmlFilterWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond),charSet, null, transformer, (ext.length() == 0));
} else { } else {
// simply pass through without parsing // simply pass through without parsing
this.theLogger.logFine("create passthrough for URL " + url + ", extension '" + ext + "', mime-type '" + res.responseHeader.mime() + "'"); theLogger.logFine("create passthrough for URL " + url + ", extension '" + ext + "', mime-type '" + res.responseHeader.mime() + "'");
hfos = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond); hfos = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
} }
@ -618,7 +602,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
handleIncomingCookies(res.responseHeader, host, ip); handleIncomingCookies(res.responseHeader, host, ip);
// remove hop by hop headers // remove hop by hop headers
this.removeHopByHopHeaders(res.responseHeader); removeHopByHopHeaders(res.responseHeader);
// adding additional headers // adding additional headers
setViaHeader(res.responseHeader, res.httpVer); setViaHeader(res.responseHeader, res.httpVer);
@ -656,7 +640,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
{ {
// ok, we don't write actually into a file, only to RAM, and schedule writing the file. // ok, we don't write actually into a file, only to RAM, and schedule writing the file.
byte[] cacheArray = res.writeContent(hfos,true); byte[] cacheArray = res.writeContent(hfos,true);
this.theLogger.logFine("writeContent of " + url + " produced cacheArray = " + ((cacheArray == null) ? "null" : ("size=" + cacheArray.length))); theLogger.logFine("writeContent of " + url + " produced cacheArray = " + ((cacheArray == null) ? "null" : ("size=" + cacheArray.length)));
if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize(); if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize();
@ -685,7 +669,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();
res.writeContent(hfos, cacheFile); res.writeContent(hfos, cacheFile);
if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize(); if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize();
this.theLogger.logFine("for write-file of " + url + ": contentLength = " + contentLength + ", sizeBeforeDelete = " + sizeBeforeDelete); theLogger.logFine("for write-file of " + url + ": contentLength = " + contentLength + ", sizeBeforeDelete = " + sizeBeforeDelete);
cacheManager.writeFileAnnouncement(cacheFile); cacheManager.writeFileAnnouncement(cacheFile);
if (sizeBeforeDelete == -1) { if (sizeBeforeDelete == -1) {
// totally fresh file // totally fresh file
@ -708,7 +692,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
} else { } else {
// no caching // no caching
this.theLogger.logFine(cacheFile.toString() + " not cached." + theLogger.logFine(cacheFile.toString() + " not cached." +
" StoreError=" + ((storeError==null)?"None":storeError) + " StoreError=" + ((storeError==null)?"None":storeError) +
" StoreHTCache=" + storeHTCache + " StoreHTCache=" + storeHTCache +
" SupportetContent=" + isSupportedContent); " SupportetContent=" + isSupportedContent);
@ -744,7 +728,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
private void fulfillRequestFromCache( private static void fulfillRequestFromCache(
Properties conProp, Properties conProp,
URL url, URL url,
String ext, String ext,
@ -763,7 +747,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
// we respond on the request by using the cache, the cache is fresh // we respond on the request by using the cache, the cache is fresh
try { try {
// remove hop by hop headers // remove hop by hop headers
this.removeHopByHopHeaders(cachedResponseHeader); removeHopByHopHeaders(cachedResponseHeader);
// adding additional headers // adding additional headers
setViaHeader(cachedResponseHeader, httpVer); setViaHeader(cachedResponseHeader, httpVer);
@ -786,7 +770,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if (requestHeader.containsKey(httpHeader.IF_MODIFIED_SINCE)) { if (requestHeader.containsKey(httpHeader.IF_MODIFIED_SINCE)) {
// conditional request: freshness of cache for that condition was already // conditional request: freshness of cache for that condition was already
// checked within shallUseCache(). Now send only a 304 response // checked within shallUseCache(). Now send only a 304 response
this.theLogger.logInfo("CACHE HIT/304 " + cacheFile.toString()); theLogger.logInfo("CACHE HIT/304 " + cacheFile.toString());
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_HIT"); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_HIT");
// setting the content length header to 0 // setting the content length header to 0
@ -797,7 +781,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
//respondHeader(respond, "304 OK", cachedResponseHeader); // respond with 'not modified' //respondHeader(respond, "304 OK", cachedResponseHeader); // respond with 'not modified'
} else { } else {
// unconditional request: send content of cache // unconditional request: send content of cache
this.theLogger.logInfo("CACHE HIT/203 " + cacheFile.toString()); theLogger.logInfo("CACHE HIT/203 " + cacheFile.toString());
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_HIT"); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_HIT");
// setting the content header to the proper length // setting the content header to the proper length
@ -837,7 +821,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
// this happens if the client stops loading the file // this happens if the client stops loading the file
// we do nothing here // we do nothing here
if (conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { if (conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
this.theLogger.logWarning("Error while trying to send cached message body."); theLogger.logWarning("Error while trying to send cached message body.");
conProp.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); conProp.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
} else { } else {
httpd.sendRespondError(conProp,respond,4,503,"socket error: " + e.getMessage(),"socket error: " + e.getMessage(), e); httpd.sendRespondError(conProp,respond,4,503,"socket error: " + e.getMessage(),"socket error: " + e.getMessage(), e);
@ -849,7 +833,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
private void removeHopByHopHeaders(httpHeader headers) { private static void removeHopByHopHeaders(httpHeader headers) {
/* /*
- Trailers - Trailers
*/ */
@ -874,14 +858,13 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
headers.remove(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE); headers.remove(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE);
} }
private void forceConnectionClose() { private static void forceConnectionClose(Properties conProp) {
if (this.connectionProperties != null) { if (conProp != null) {
this.connectionProperties.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); conProp.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
} }
} }
public void doHead(Properties conProp, httpHeader requestHeader, OutputStream respond) throws IOException { public static void doHead(Properties conProp, httpHeader requestHeader, OutputStream respond) throws IOException {
this.connectionProperties = conProp;
httpc remote = null; httpc remote = null;
httpc.response res = null; httpc.response res = null;
@ -889,7 +872,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
try { try {
// remembering the starting time of the request // remembering the starting time of the request
Date requestDate = new Date(); // remember the time... Date requestDate = new Date(); // remember the time...
this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis(); switchboard.proxyLastAccess = System.currentTimeMillis();
@ -930,7 +913,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_PROXY, hostlow, remotePath)) { if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_PROXY, hostlow, remotePath)) {
httpd.sendRespondError(conProp,respond,4,403,null, httpd.sendRespondError(conProp,respond,4,403,null,
"URL '" + hostlow + "' blocked by yacy proxy (blacklisted)",null); "URL '" + hostlow + "' blocked by yacy proxy (blacklisted)",null);
this.theLogger.logInfo("AGIS blocking of host '" + hostlow + "'"); theLogger.logInfo("AGIS blocking of host '" + hostlow + "'");
return; return;
} }
@ -952,7 +935,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if ((yAddress != null) && ((pos = yAddress.indexOf("/")) >= 0)) remotePath = yAddress.substring(pos) + remotePath; if ((yAddress != null) && ((pos = yAddress.indexOf("/")) >= 0)) remotePath = yAddress.substring(pos) + remotePath;
// removing hop by hop headers // removing hop by hop headers
this.removeHopByHopHeaders(requestHeader); removeHopByHopHeaders(requestHeader);
// adding outgoing headers // adding outgoing headers
setViaHeader(requestHeader, httpVer); setViaHeader(requestHeader, httpVer);
@ -969,7 +952,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
// removing hop by hop headers // removing hop by hop headers
this.removeHopByHopHeaders(res.responseHeader); removeHopByHopHeaders(res.responseHeader);
// adding outgoing headers // adding outgoing headers
setViaHeader(res.responseHeader, res.httpVer); setViaHeader(res.responseHeader, res.httpVer);
@ -985,16 +968,14 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
respond.flush(); respond.flush();
} }
public void doPost(Properties conProp, httpHeader requestHeader, OutputStream respond, PushbackInputStream body) throws IOException { public static void doPost(Properties conProp, httpHeader requestHeader, OutputStream respond, PushbackInputStream body) throws IOException {
this.connectionProperties = conProp;
httpc remote = null; httpc remote = null;
URL url = null; URL url = null;
try { try {
// remembering the starting time of the request // remembering the starting time of the request
Date requestDate = new Date(); // remember the time... Date requestDate = new Date(); // remember the time...
this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis(); switchboard.proxyLastAccess = System.currentTimeMillis();
@ -1045,7 +1026,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if ((yAddress != null) && ((pos = yAddress.indexOf("/")) >= 0)) remotePath = yAddress.substring(pos) + remotePath; if ((yAddress != null) && ((pos = yAddress.indexOf("/")) >= 0)) remotePath = yAddress.substring(pos) + remotePath;
// removing hop by hop headers // removing hop by hop headers
this.removeHopByHopHeaders(requestHeader); removeHopByHopHeaders(requestHeader);
// adding additional headers // adding additional headers
setViaHeader(requestHeader, httpVer); setViaHeader(requestHeader, httpVer);
@ -1071,7 +1052,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
res.responseHeader.put(httpHeader.CONTENT_LENGTH,"0"); res.responseHeader.put(httpHeader.CONTENT_LENGTH,"0");
} else { } else {
if (httpVer.equals("HTTP/0.9") || httpVer.equals("HTTP/1.0")) { if (httpVer.equals("HTTP/0.9") || httpVer.equals("HTTP/1.0")) {
forceConnectionClose(); forceConnectionClose(conProp);
} else { } else {
chunked = new httpChunkedOutputStream(respond); chunked = new httpChunkedOutputStream(respond);
} }
@ -1080,7 +1061,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
// remove hop by hop headers // remove hop by hop headers
this.removeHopByHopHeaders(res.responseHeader); removeHopByHopHeaders(res.responseHeader);
// adding additional headers // adding additional headers
setViaHeader(res.responseHeader, res.httpVer); setViaHeader(res.responseHeader, res.httpVer);
@ -1112,14 +1093,14 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
respond.flush(); respond.flush();
if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish(); if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish();
this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis())); conProp.put(httpHeader.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis()));
this.connectionProperties.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount())); conProp.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount()));
this.logProxyAccess(); logProxyAccess(conProp);
} }
} }
public void doConnect(Properties conProp, de.anomic.http.httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) throws IOException { public static void doConnect(Properties conProp, de.anomic.http.httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) throws IOException {
this.connectionProperties = conProp;
switchboard.proxyLastAccess = System.currentTimeMillis(); switchboard.proxyLastAccess = System.currentTimeMillis();
String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
@ -1143,8 +1124,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_PROXY, hostlow, path)) { if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_PROXY, hostlow, path)) {
httpd.sendRespondError(conProp,clientOut,4,403,null, httpd.sendRespondError(conProp,clientOut,4,403,null,
"URL '" + hostlow + "' blocked by yacy proxy (blacklisted)",null); "URL '" + hostlow + "' blocked by yacy proxy (blacklisted)",null);
this.theLogger.logInfo("AGIS blocking of host '" + hostlow + "'"); theLogger.logInfo("AGIS blocking of host '" + hostlow + "'");
forceConnectionClose(); forceConnectionClose(conProp);
return; return;
} }
@ -1176,7 +1157,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
// pass error response back to client // pass error response back to client
httpd.sendRespondHeader(conProp,clientOut,httpVersion,response.statusCode,response.statusText,response.responseHeader); httpd.sendRespondHeader(conProp,clientOut,httpVersion,response.statusCode,response.statusText,response.responseHeader);
//respondHeader(clientOut, response.status, response.responseHeader); //respondHeader(clientOut, response.status, response.responseHeader);
forceConnectionClose(); forceConnectionClose(conProp);
return; return;
} }
} catch (Exception e) { } catch (Exception e) {
@ -1198,7 +1179,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
"Proxy-agent: YACY" + serverCore.crlfString + "Proxy-agent: YACY" + serverCore.crlfString +
serverCore.crlfString).getBytes()); serverCore.crlfString).getBytes());
this.theLogger.logInfo("SSL connection to " + host + ":" + port + " established."); theLogger.logInfo("SSL connection to " + host + ":" + port + " established.");
// start stream passing with mediate processes // start stream passing with mediate processes
Mediate cs = new Mediate(sslSocket, clientIn, promiscuousOut); Mediate cs = new Mediate(sslSocket, clientIn, promiscuousOut);
@ -1222,7 +1203,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
// ...hope they have terminated... // ...hope they have terminated...
} }
public class Mediate extends Thread { public static class Mediate extends Thread {
boolean terminate; boolean terminate;
Socket socket; Socket socket;
@ -1259,7 +1240,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
} }
private httpc newhttpc(String server, int port, int timeout) throws IOException { private static httpc newhttpc(String server, int port, int timeout) throws IOException {
// getting the remote proxy configuration // getting the remote proxy configuration
httpRemoteProxyConfig remProxyConfig = switchboard.remoteProxyConfig; httpRemoteProxyConfig remProxyConfig = switchboard.remoteProxyConfig;
@ -1303,7 +1284,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
); );
} }
private httpc newhttpc(String address, int timeout) throws IOException { private static httpc newhttpc(String address, int timeout) throws IOException {
// a new httpc connection for <host>:<port>/<path> syntax // a new httpc connection for <host>:<port>/<path> syntax
// this is called when a '.yacy'-domain is used // this is called when a '.yacy'-domain is used
int p = address.indexOf(":"); int p = address.indexOf(":");
@ -1330,7 +1311,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
out.flush(); out.flush();
} }
*/ */
private void handleProxyException(Exception e, httpc remote, Properties conProp, OutputStream respond, URL url) { private static void handleProxyException(Exception e, httpc remote, Properties conProp, OutputStream respond, URL url) {
// this may happen if // this may happen if
// - the targeted host does not exist // - the targeted host does not exist
// - anything with the remote server was wrong. // - anything with the remote server was wrong.
@ -1373,8 +1354,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
String exceptionMsg = e.getMessage(); String exceptionMsg = e.getMessage();
if ((exceptionMsg != null) && (exceptionMsg.indexOf("Corrupt GZIP trailer") >= 0)) { if ((exceptionMsg != null) && (exceptionMsg.indexOf("Corrupt GZIP trailer") >= 0)) {
// just do nothing, we leave it this way // just do nothing, we leave it this way
this.theLogger.logFine("ignoring bad gzip trail for URL " + url + " (" + e.getMessage() + ")"); theLogger.logFine("ignoring bad gzip trail for URL " + url + " (" + e.getMessage() + ")");
this.forceConnectionClose(); forceConnectionClose(conProp);
} else if ((exceptionMsg != null) && (exceptionMsg.indexOf("Connection reset")>= 0)) { } else if ((exceptionMsg != null) && (exceptionMsg.indexOf("Connection reset")>= 0)) {
errorMessage = "Connection reset"; errorMessage = "Connection reset";
} else if ((exceptionMsg != null) && (exceptionMsg.indexOf("unknown host")>=0)) { } else if ((exceptionMsg != null) && (exceptionMsg.indexOf("unknown host")>=0)) {
@ -1413,25 +1394,25 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} }
} else { } else {
if (unknownError) { if (unknownError) {
this.theLogger.logFine("Error while processing request '" + theLogger.logFine("Error while processing request '" +
conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" + conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" +
"\n" + Thread.currentThread().getName() + "\n" + Thread.currentThread().getName() +
"\n" + errorMessage,e); "\n" + errorMessage,e);
} else { } else {
this.theLogger.logFine("Error while processing request '" + theLogger.logFine("Error while processing request '" +
conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" + conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" +
"\n" + Thread.currentThread().getName() + "\n" + Thread.currentThread().getName() +
"\n" + errorMessage); "\n" + errorMessage);
} }
this.forceConnectionClose(); forceConnectionClose(conProp);
} }
} catch (Exception ee) { } catch (Exception ee) {
this.forceConnectionClose(); forceConnectionClose(conProp);
} }
} }
private serverObjects unknownHostHandling(Properties conProp) throws Exception { private static serverObjects unknownHostHandling(Properties conProp) throws Exception {
serverObjects detailedErrorMsgMap = new serverObjects(); serverObjects detailedErrorMsgMap = new serverObjects();
// generic toplevel domains // generic toplevel domains
@ -1520,26 +1501,26 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
return detailedErrorMsgMap; return detailedErrorMsgMap;
} }
private String generateUserAgent(httpHeader requestHeaders) { private static String generateUserAgent(httpHeader requestHeaders) {
this.userAgentStr.setLength(0); userAgentStr.setLength(0);
String browserUserAgent = (String) requestHeaders.get(httpHeader.USER_AGENT, proxyUserAgent); String browserUserAgent = (String) requestHeaders.get(httpHeader.USER_AGENT, proxyUserAgent);
int pos = browserUserAgent.lastIndexOf(')'); int pos = browserUserAgent.lastIndexOf(')');
if (pos >= 0) { if (pos >= 0) {
this.userAgentStr userAgentStr
.append(browserUserAgent.substring(0,pos)) .append(browserUserAgent.substring(0,pos))
.append("; YaCy ") .append("; YaCy ")
.append(switchboard.getConfig("vString","0.1")) .append(switchboard.getConfig("vString","0.1"))
.append("; yacy.net") .append("; yacy.net")
.append(browserUserAgent.substring(pos)); .append(browserUserAgent.substring(pos));
} else { } else {
this.userAgentStr.append(browserUserAgent); userAgentStr.append(browserUserAgent);
} }
return new String(this.userAgentStr); return new String(userAgentStr);
} }
private void setViaHeader(httpHeader header, String httpVer) { private static void setViaHeader(httpHeader header, String httpVer) {
if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return; if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return;
// getting header set by other proxies in the chain // getting header set by other proxies in the chain
@ -1563,86 +1544,86 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
* e.g.<br> * e.g.<br>
* <code>1117528623.857 178 192.168.1.201 TCP_MISS/200 1069 GET http://www.yacy.de/ - DIRECT/81.169.145.74 text/html</code> * <code>1117528623.857 178 192.168.1.201 TCP_MISS/200 1069 GET http://www.yacy.de/ - DIRECT/81.169.145.74 text/html</code>
*/ */
private final void logProxyAccess() { private final static void logProxyAccess(Properties conProp) {
if (!doAccessLogging) return; if (!doAccessLogging) return;
this.logMessage.setLength(0); logMessage.setLength(0);
// Timestamp // Timestamp
String currentTimestamp = Long.toString(System.currentTimeMillis()); String currentTimestamp = Long.toString(System.currentTimeMillis());
int offset = currentTimestamp.length()-3; int offset = currentTimestamp.length()-3;
this.logMessage.append(currentTimestamp.substring(0,offset)); logMessage.append(currentTimestamp.substring(0,offset));
this.logMessage.append('.'); logMessage.append('.');
this.logMessage.append(currentTimestamp.substring(offset)); logMessage.append(currentTimestamp.substring(offset));
this.logMessage.append(' '); logMessage.append(' ');
// Elapsed time // Elapsed time
Long requestStart = (Long) this.connectionProperties.get(httpHeader.CONNECTION_PROP_REQUEST_START); Long requestStart = (Long) conProp.get(httpHeader.CONNECTION_PROP_REQUEST_START);
Long requestEnd = (Long) this.connectionProperties.get(httpHeader.CONNECTION_PROP_REQUEST_END); Long requestEnd = (Long) conProp.get(httpHeader.CONNECTION_PROP_REQUEST_END);
String elapsed = Long.toString(requestEnd.longValue()-requestStart.longValue()); String elapsed = Long.toString(requestEnd.longValue()-requestStart.longValue());
for (int i=0; i<6-elapsed.length(); i++) this.logMessage.append(' '); for (int i=0; i<6-elapsed.length(); i++) logMessage.append(' ');
this.logMessage.append(elapsed); logMessage.append(elapsed);
this.logMessage.append(' '); logMessage.append(' ');
// Remote Host // Remote Host
String clientIP = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); String clientIP = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP);
this.logMessage.append(clientIP); logMessage.append(clientIP);
this.logMessage.append(' '); logMessage.append(' ');
// Code/Status // Code/Status
String respondStatus = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS); String respondStatus = conProp.getProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS);
String respondCode = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"UNKNOWN"); String respondCode = conProp.getProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"UNKNOWN");
this.logMessage.append(respondCode); logMessage.append(respondCode);
this.logMessage.append("/"); logMessage.append("/");
this.logMessage.append(respondStatus); logMessage.append(respondStatus);
this.logMessage.append(' '); logMessage.append(' ');
// Bytes // Bytes
Long bytes = (Long) this.connectionProperties.get(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE); Long bytes = (Long) conProp.get(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE);
this.logMessage.append(bytes.toString()); logMessage.append(bytes.toString());
this.logMessage.append(' '); logMessage.append(' ');
// Method // Method
String requestMethod = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_METHOD); String requestMethod = conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD);
this.logMessage.append(requestMethod); logMessage.append(requestMethod);
this.logMessage.append(' '); logMessage.append(' ');
// URL // URL
String requestURL = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_URL); String requestURL = conProp.getProperty(httpHeader.CONNECTION_PROP_URL);
String requestArgs = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_ARGS); String requestArgs = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS);
this.logMessage.append(requestURL); logMessage.append(requestURL);
if (requestArgs != null) { if (requestArgs != null) {
this.logMessage.append("?") logMessage.append("?")
.append(requestArgs); .append(requestArgs);
} }
this.logMessage.append(' '); logMessage.append(' ');
// Rfc931 // Rfc931
this.logMessage.append("-"); logMessage.append("-");
this.logMessage.append(' '); logMessage.append(' ');
// Peerstatus/Peerhost // Peerstatus/Peerhost
String host = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_HOST); String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
this.logMessage.append("DIRECT/"); logMessage.append("DIRECT/");
this.logMessage.append(host); logMessage.append(host);
this.logMessage.append(' '); logMessage.append(' ');
// Type // Type
String mime = "-"; String mime = "-";
if (this.connectionProperties.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { if (conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
httpHeader proxyRespondHeader = (httpHeader) this.connectionProperties.get(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER); httpHeader proxyRespondHeader = (httpHeader) conProp.get(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER);
mime = proxyRespondHeader.mime(); mime = proxyRespondHeader.mime();
if (mime.indexOf(";") != -1) { if (mime.indexOf(";") != -1) {
mime = mime.substring(0,mime.indexOf(";")); mime = mime.substring(0,mime.indexOf(";"));
} }
} }
this.logMessage.append(mime); logMessage.append(mime);
// sending the logging message to the logger // sending the logging message to the logger
this.proxyLog.logFine(new String(this.logMessage)); proxyLog.logFine(new String(logMessage));
} }
} }

8
source/de/anomic/soap/httpdSoapHandler.java
Unescape View File

@ -81,8 +81,6 @@ import de.anomic.http.httpChunkedOutputStream;
import de.anomic.http.httpContentLengthInputStream; import de.anomic.http.httpContentLengthInputStream;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpd; import de.anomic.http.httpd;
import de.anomic.http.httpdAbstractHandler;
import de.anomic.http.httpdHandler;
import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaParser;
import de.anomic.server.serverClassLoader; import de.anomic.server.serverClassLoader;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
@ -118,11 +116,11 @@ import de.anomic.server.logging.serverLog;
* *
* @author Martin Thelian * @author Martin Thelian
*/ */
public final class httpdSoapHandler extends httpdAbstractHandler implements httpdHandler public final class httpdSoapHandler {
{
public static final String SOAP_HANDLER_VERSION = "YaCySOAP V0.1"; public static final String SOAP_HANDLER_VERSION = "YaCySOAP V0.1";
private serverLog theLogger;
/* =============================================================== /* ===============================================================
* Constants needed to set some SOAP properties * Constants needed to set some SOAP properties
* =============================================================== */ * =============================================================== */

4
source/yacy.java
Unescape View File

@ -67,8 +67,6 @@ import de.anomic.data.translator;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpc; import de.anomic.http.httpc;
import de.anomic.http.httpd; import de.anomic.http.httpd;
import de.anomic.http.httpdFileHandler;
import de.anomic.http.httpdProxyHandler;
import de.anomic.index.indexContainer; import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry; import de.anomic.index.indexRWIEntry;
import de.anomic.index.indexURLEntry; import de.anomic.index.indexURLEntry;
@ -319,7 +317,7 @@ public final class yacy {
// start main threads // start main threads
final String port = sb.getConfig("port", "8080"); final String port = sb.getConfig("port", "8080");
try { try {
final httpd protocolHandler = new httpd(sb, new httpdFileHandler(sb), new httpdProxyHandler(sb)); final httpd protocolHandler = new httpd(sb);
final serverCore server = new serverCore( final serverCore server = new serverCore(
timeout /*control socket timeout in milliseconds*/, timeout /*control socket timeout in milliseconds*/,
true /* block attacks (wrong protocol) */, true /* block attacks (wrong protocol) */,

Write Preview
Loading…
Cancel
Save