diff --git a/build.xml b/build.xml index cd06372ad..3adaade9a 100644 --- a/build.xml +++ b/build.xml @@ -258,11 +258,11 @@ - + - + diff --git a/libx/PDFBox-0.7.1.License b/libx/PDFBox-0.7.2.License similarity index 100% rename from libx/PDFBox-0.7.1.License rename to libx/PDFBox-0.7.2.License diff --git a/libx/PDFBox-0.7.1.jar b/libx/PDFBox-0.7.2.jar similarity index 89% rename from libx/PDFBox-0.7.1.jar rename to libx/PDFBox-0.7.2.jar index 8ac850ade..6d4b513fc 100644 Binary files a/libx/PDFBox-0.7.1.jar and b/libx/PDFBox-0.7.2.jar differ diff --git a/libx/jsch-0.1.19.jar b/libx/jsch-0.1.19.jar deleted file mode 100644 index f601660ef..000000000 Binary files a/libx/jsch-0.1.19.jar and /dev/null differ diff --git a/libx/jsch-0.1.19.License b/libx/jsch-0.1.21.License similarity index 100% rename from libx/jsch-0.1.19.License rename to libx/jsch-0.1.21.License diff --git a/libx/jsch-0.1.21.jar b/libx/jsch-0.1.21.jar new file mode 100644 index 000000000..0ae4e6f7b Binary files /dev/null and b/libx/jsch-0.1.21.jar differ diff --git a/source/de/anomic/http/httpChunkedInputStream.java b/source/de/anomic/http/httpChunkedInputStream.java new file mode 100644 index 000000000..91f538db1 --- /dev/null +++ b/source/de/anomic/http/httpChunkedInputStream.java @@ -0,0 +1,255 @@ +//httpChunkedInputStream.java +//----------------------- +//(C) by Michael Peter Christen; mc@anomic.de +//first published on http://www.anomic.de +//Frankfurt, Germany, 2004 +// +// This file is contributed by Martin Thelian +// last major change: $LastChangedDate$ by $LastChangedBy$ +// Revision: $LastChangedRevision$ +// +//This program is free software; you can redistribute it and/or modify +//it under the terms of the GNU General Public License as published by +//the Free Software Foundation; either version 2 of the License, or +//(at your option) any later version. +// +//This program is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. +// +//You should have received a copy of the GNU General Public License +//along with this program; if not, write to the Free Software +//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +//Using this software in any meaning (reading, learning, copying, compiling, +//running) means that you agree that the Author(s) is (are) not responsible +//for cost, loss of data or any harm that may be caused directly or indirectly +//by usage of this softare or this documentation. The usage of this software +//is on your own risk. The installation and usage (starting/running) of this +//software may allow other people or application to access your computer and +//any attached devices and is highly dependent on the configuration of the +//software which must be done by the user of the software; the author(s) is +//(are) also not responsible for proper configuration and usage of the +//software, even if provoked by documentation provided together with +//the software. +// +//Any changes to this file according to the GPL as documented in the file +//gpl.txt aside this file in the shipment you received can be done to the +//lines that follows this copyright notice here, but changes must not be +//done inside the copyright notive above. A re-distribution must contain +//the intact and unchanged copyright notice. +//Contributions and changes to the program code must be marked as such. + +package de.anomic.http; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +/** + * Some parts of this class code was copied from Apache httpclient Project. + * @author theli + */ +public class httpChunkedInputStream extends InputStream { + + private static final int READ_CHUNK_STATE_NORMAL = 0; + private static final int READ_CHUNK_STATE_CR_READ = 1; + private static final int READ_CHUNK_STATE_IN_EXT_CHUNK = 2; + private static final int READ_CHUNK_STATE_FINISHED = -1; + + private static final char CR = '\r'; + private static final char LF = '\n'; + + private InputStream inputStream; + private int currPos; + private int currChunkSize; + private httpHeader httpTrailer; + + private boolean beginningOfStream = true; + private boolean isEOF = false; + private boolean isClosed = false; + + + public httpChunkedInputStream(InputStream in) throws IOException { + + if (in == null)throw new IllegalArgumentException("InputStream must not be null"); + + this.inputStream = in; + this.currPos = 0; + } + + public int read() throws IOException { + + if (this.isClosed) throw new IOException("Inputstream already closed."); + if (this.isEOF) return -1; + + if (this.currPos >= this.currChunkSize) { + readNextChunk(); + if (this.isEOF) return -1; + } + this.currPos++; + return this.inputStream.read(); + } + + + public int read (byte[] b, int off, int len) throws IOException { + + if (b == null) throw new IllegalArgumentException("bytearry parameter must not be null"); + + if (this.isClosed) throw new IOException("Inputstream already closed."); + if (this.isEOF) return -1; + + if (this.currPos >= this.currChunkSize) { + readNextChunk(); + if (this.isEOF) return -1; + } + len = Math.min(len, this.currChunkSize - this.currPos); + int count = this.inputStream.read(b, off, len); + this.currPos += count; + return count; + } + + public int read (byte[] b) throws IOException { + if (b == null) throw new IllegalArgumentException("bytearry parameter must not be null"); + return read(b, 0, b.length); + } + + /** + * Read the CRLF terminator. + * @throws IOException If an IO error occurs. + */ + private void readCRLF() throws IOException { + int cr = this.inputStream.read(); + int lf = this.inputStream.read(); + if ((cr != CR) || (lf != LF)) { + throw new IOException("Malformed chunk. CRLF expected but '" + cr + lf + "' found"); + } + } + + + private void readNextChunk() throws IOException { + if (!this.beginningOfStream) readCRLF(); + + this.currChunkSize = readChunkFromStream(this.inputStream); + this.beginningOfStream = false; + this.currPos = 0; + if (this.currChunkSize == 0) { + this.isEOF = true; + readTrailer(); + } + } + + + private void readTrailer() throws IOException { + BufferedReader reader = null; + ByteArrayOutputStream bout = null; + try { + bout = new ByteArrayOutputStream(); + do { + int ch; + while ((ch = this.inputStream.read()) >= 0) { + bout.write(ch); + if (ch == LF) { + break; + } + } + if (bout.size() <= 2) break; + } while(true); + + ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray()); + reader = new BufferedReader(new InputStreamReader(bin)); + this.httpTrailer = httpHeader.readHttpHeader(reader); + } finally { + if (reader != null) try {reader.close();}catch(Exception e){} + if (bout != null) try {bout.close();}catch(Exception e){} + } + } + + public httpHeader getTrailer() { + return this.httpTrailer; + } + + private static int readChunkFromStream(final InputStream in) + throws IOException { + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int state = 0; + while (state != READ_CHUNK_STATE_FINISHED) { + int b = in.read(); + if (b == -1) throw new IOException("Malformed chunk. Unexpected end"); + + switch (state) { + case 0: + switch (b) { + case CR: + state = READ_CHUNK_STATE_CR_READ; + break; + case '\"': + case ';': + case ' ': + state = READ_CHUNK_STATE_IN_EXT_CHUNK; + break; + default: + baos.write(b); + } + break; + + case 1: + if (b == LF) { + state = READ_CHUNK_STATE_FINISHED; + } else { + // this was not CRLF + throw new IOException("Malformed chunk. Unexpected enf of chunk. MIssing CR character."); + } + break; + + case 2: + switch (b) { + case CR: + state = READ_CHUNK_STATE_CR_READ; + break; + default: + break; + } + break; + default: throw new RuntimeException("Malformed chunk. Illegal state."); + } + } + + + int result; + try { + result = Integer.parseInt(baos.toString().trim(), 16); + } catch (NumberFormatException e) { + throw new IOException ("Malformed chunk. Bad chunk size: " + baos.toString()); + } + return result; + } + + public void close() throws IOException { + if (!this.isClosed) { + try { + if (!this.isEOF) { + exhaustInputStream(this); + } + } finally { + this.isEOF = true; + this.isClosed = true; + } + } + } + + + static void exhaustInputStream(InputStream inStream) throws IOException { + byte buffer[] = new byte[1024]; + while (inStream.read(buffer) >= 0) { + ; + } + } +} + + diff --git a/source/de/anomic/http/httpHeader.java b/source/de/anomic/http/httpHeader.java index 13105f036..4dd750389 100644 --- a/source/de/anomic/http/httpHeader.java +++ b/source/de/anomic/http/httpHeader.java @@ -3,7 +3,9 @@ // (C) by Michael Peter Christen; mc@anomic.de // first published on http://www.anomic.de // Frankfurt, Germany, 2004 -// last major change: 29.04.2004 +// +// last major change: $LastChangedDate$ by $LastChangedBy$ +// Revision: $LastChangedRevision$ // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -57,6 +59,9 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; import java.text.Collator; import java.text.SimpleDateFormat; import java.util.Date; @@ -64,10 +69,13 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Locale; import java.util.Map; +import java.util.Properties; import java.util.TimeZone; import java.util.TreeMap; +import de.anomic.server.serverCore; import de.anomic.server.logging.serverLog; +import de.anomic.yacy.yacyCore; public final class httpHeader extends TreeMap implements Map { @@ -176,6 +184,31 @@ public final class httpHeader extends TreeMap implements Map { http1_1.put("504","Gateway Time-out"); http1_1.put("505","HTTP Version not supported"); } + + /* PROPERTIES: General properties */ + public static final String CONNECTION_PROP_HTTP_VER = "HTTP"; + public static final String CONNECTION_PROP_HOST = "HOST"; + public static final String CONNECTION_PROP_METHOD = "METHOD"; + public static final String CONNECTION_PROP_PATH = "PATH"; + public static final String CONNECTION_PROP_EXT = "EXT"; + public static final String CONNECTION_PROP_URL = "URL"; + public static final String CONNECTION_PROP_ARGS = "ARGS"; + public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP"; + public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT"; + public static final String CONNECTION_PROP_KEEP_ALIVE_COUNT = "KEEP-ALIVE_COUNT"; + public static final String CONNECTION_PROP_REQUESTLINE = "REQUESTLINE"; + public static final String CONNECTION_PROP_PREV_REQUESTLINE = "PREVREQUESTLINE"; + public static final String CONNECTION_PROP_REQUEST_START = "REQUEST_START"; + public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END"; + + /* PROPERTIES: Client -> Proxy */ + public static final String CONNECTION_PROP_CLIENT_REQUEST_HEADER = "CLIENT_REQUEST_HEADER"; + + /* PROPERTIES: Proxy -> Client */ + public static final String CONNECTION_PROP_PROXY_RESPOND_CODE = "PROXY_RESPOND_CODE"; + public static final String CONNECTION_PROP_PROXY_RESPOND_STATUS = "PROXY_RESPOND_STATUS"; + public static final String CONNECTION_PROP_PROXY_RESPOND_HEADER = "PROXY_RESPOND_HEADER"; + public static final String CONNECTION_PROP_PROXY_RESPOND_SIZE = "PROXY_REQUEST_SIZE"; private final HashMap reverseMappingCache; @@ -310,7 +343,6 @@ public final class httpHeader extends TreeMap implements Map { private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); private static SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'"); private static SimpleDateFormat EMLFormatter = new SimpleDateFormat("dd MMM yyyy HH:mm:ss", Locale.US); - public static Date parseHTTPDate(String s) { if ((s == null) || (s.length() < 9)) return new Date(); @@ -386,17 +418,309 @@ public final class httpHeader extends TreeMap implements Map { return ((containsKey(httpHeader.CONTENT_ENCODING)) && (((String) get(httpHeader.CONTENT_ENCODING)).toUpperCase().startsWith("GZIP"))); } - /* - public static void main(String[] args) { - Collator c; - c = Collator.getInstance(Locale.US); c.setStrength(Collator.PRIMARY); - System.out.println("PRIMARY: compare(abc, ABC) = " + c.compare("abc", "ABC")); - c = Collator.getInstance(Locale.US); c.setStrength(Collator.SECONDARY); - System.out.println("SECONDARY: compare(abc, ABC) = " + c.compare("abc", "ABC")); - c = Collator.getInstance(Locale.US); c.setStrength(Collator.TERTIARY); - System.out.println("TERTIARY: compare(abc, ABC) = " + c.compare("abc", "ABC")); - c = Collator.getInstance(Locale.US); c.setStrength(Collator.IDENTICAL); - System.out.println("IDENTICAL: compare(abc, ABC) = " + c.compare("abc", "ABC")); - } - */ + + public static Object[] parseResponseLine(String respLine) { + + if ((respLine == null) || (respLine.length() == 0)) { + return new Object[]{"HTTP/1.0",new Integer(500),"status line parse error"}; + } + + int p = respLine.indexOf(" "); + if (p < 0) { + return new Object[]{"HTTP/1.0",new Integer(500),"status line parse error"}; + } + + String httpVer, status, statusText; + Integer statusCode; + + // the http version reported by the server + httpVer = respLine.substring(0,p); + + // Status of the request, e.g. "200 OK" + status = respLine.substring(p + 1).trim(); // the status code plus reason-phrase + + // splitting the status into statuscode and statustext + p = status.indexOf(" "); + try { + statusCode = Integer.valueOf((p < 0) ? status.trim() : status.substring(0,p).trim()); + statusText = (p < 0) ? "" : status.substring(p+1).trim(); + } catch (Exception e) { + statusCode = new Integer(500); + statusText = status; + } + + return new Object[]{httpVer,statusCode,statusText}; + } + + public static Properties parseRequestLine(String s, Properties prop, String virtualHost) { + int p = s.indexOf(" "); + if (p >= 0) { + String cmd = s.substring(0,p); + String args = s.substring(p+1); + return parseRequestLine(cmd,args, prop,virtualHost); + } else { + return prop; + } + } + + public static Properties parseRequestLine(String cmd, String args, Properties prop, String virtualHost) { + + // getting the last request line for debugging purposes + String prevRequestLine = prop.containsKey(httpHeader.CONNECTION_PROP_REQUESTLINE)? + prop.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE) : ""; + + // reset property from previous run + prop.clear(); + + // storing informations about the request + prop.setProperty(httpHeader.CONNECTION_PROP_METHOD, cmd); + prop.setProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,cmd + " " + args); + prop.setProperty(httpHeader.CONNECTION_PROP_PREV_REQUESTLINE,prevRequestLine); + + // this parses a whole URL + if (args.length() == 0) { + prop.setProperty(httpHeader.CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(httpHeader.CONNECTION_PROP_PATH, "/"); + prop.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); + prop.setProperty(httpHeader.CONNECTION_PROP_EXT, ""); + return prop; + } + + // store the version propery "HTTP" and cut the query at both ends + int sep = args.indexOf(" "); + if (sep >= 0) { + // HTTP version is given + prop.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, args.substring(sep + 1).trim()); + args = args.substring(0, sep).trim(); // cut off HTTP version mark + } else { + // HTTP version is not given, it will be treated as ver 0.9 + prop.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); + } + + // properties of the query are stored with the prefix "&" + // additionally, the values URL and ARGC are computed + + String argsString = ""; + sep = args.indexOf("?"); + if (sep >= 0) { + // there are values attached to the query string + argsString = args.substring(sep + 1); // cut haed from tail of query + args = args.substring(0, sep); + } + prop.setProperty(httpHeader.CONNECTION_PROP_URL, args); // store URL + //System.out.println("HTTPD: ARGS=" + argsString); + if (argsString.length() != 0) prop.setProperty(httpHeader.CONNECTION_PROP_ARGS, argsString); // store arguments in original form + + // find out file extension + sep = args.lastIndexOf("."); + if (sep >= 0) { + if (args.indexOf("?", sep + 1) >= sep) + prop.setProperty(httpHeader.CONNECTION_PROP_EXT, args.substring(sep + 1, args.indexOf("?", sep + 1)).toLowerCase()); + else if (args.indexOf("#", sep + 1) >= sep) + prop.setProperty(httpHeader.CONNECTION_PROP_EXT, args.substring(sep + 1, args.indexOf("#", sep + 1)).toLowerCase()); + else + prop.setProperty(httpHeader.CONNECTION_PROP_EXT, args.substring(sep + 1).toLowerCase()); + } else { + prop.setProperty(httpHeader.CONNECTION_PROP_EXT, ""); + } + + // finally find host string + if (args.toUpperCase().startsWith("HTTP://")) { + // a host was given. extract it and set path + args = args.substring(7); + sep = args.indexOf("/"); + if (sep < 0) { + // this is a malformed url, something like + // http://index.html + // we are lazy and guess that it means + // /index.html + // which is a localhost access to the file servlet + prop.setProperty(httpHeader.CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(httpHeader.CONNECTION_PROP_PATH, "/" + args); + } else { + // THIS IS THE "GOOD" CASE + // a perfect formulated url + prop.setProperty(httpHeader.CONNECTION_PROP_HOST, args.substring(0, sep)); + prop.setProperty(httpHeader.CONNECTION_PROP_PATH, args.substring(sep)); // yes, including beginning "/" + } + } else { + // no host in url. set path + if (args.startsWith("/")) { + // thats also fine, its a perfect localhost access + // in this case, we simulate a + // http://localhost/s + // access by setting a virtual host + prop.setProperty(httpHeader.CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(httpHeader.CONNECTION_PROP_PATH, args); + } else { + // the client 'forgot' to set a leading '/' + // this is the same case as above, with some lazyness + prop.setProperty(httpHeader.CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(httpHeader.CONNECTION_PROP_PATH, "/" + args); + } + } + return prop; + } + + /** + * Reading http headers from a reader class and building up a httpHeader object + * @param reader the {@link BufferedReader} that is used to read the http header lines + * @return a {@link httpHeader}-Object containing all parsed headers + * @throws IOException + */ + public static httpHeader readHttpHeader(BufferedReader reader) throws IOException { + // reading all request headers + httpHeader httpHeader = new httpHeader(httpd.reverseMappingCache); + int p; + String line; + while ((line = reader.readLine()) != null) { + if (line.length() == 0) break; + if ((p = line.indexOf(":")) >= 0) { + // store a property + httpHeader.add(line.substring(0, p).trim(), line.substring(p + 1).trim()); + } + } + return httpHeader; + } + + public static httpHeader readHeader(Properties prop, serverCore.Session theSession) throws IOException { + + // reading all headers + httpHeader header = new httpHeader(httpd.reverseMappingCache); + int p; + String line; + while ((line = theSession.readLineAsString()) != null) { + if (line.length() == 0) break; // this seperates the header of the HTTP request from the body + // parse the header line: a property seperated with the ':' sign + if ((p = line.indexOf(":")) >= 0) { + // store a property + header.add(line.substring(0, p).trim(), line.substring(p + 1).trim()); + } + } + + /* + * doing some header validation here ... + */ + String httpVersion = prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); + if (httpVersion.equals("HTTP/1.1") && !header.containsKey(httpHeader.HOST)) { + // the HTTP/1.1 specification requires that an HTTP/1.1 server must reject any + // HTTP/1.1 message that does not contain a Host header. + httpd.sendRespondError(prop,theSession.out,0,400,null,null,null); + throw new IOException("400 Bad request"); + } + + return header; + } + + + public StringBuffer toHeaderString( + String httpVersion, + int httpStatusCode, + String httpStatusText) { + // creating a new buffer to store the header as string + StringBuffer theHeader = new StringBuffer(); + + // generating the header string + this.toHeaderString(httpVersion,httpStatusCode,httpStatusText,theHeader); + + // returning the result + return theHeader; + } + + + public void toHeaderString( + String httpVersion, + int httpStatusCode, + String httpStatusText, + StringBuffer theHeader) { + + if (theHeader == null) throw new IllegalArgumentException(); + + // setting the http version if it was not already set + if (httpVersion == null) httpVersion = "HTTP/1.0"; + + // setting the status text if it was not already set + if ((httpStatusText == null)||(httpStatusText.length()==0)) { + if (httpVersion.equals("HTTP/1.0") && httpHeader.http1_0.containsKey(Integer.toString(httpStatusCode))) + httpStatusText = (String) httpHeader.http1_0.get(Integer.toString(httpStatusCode)); + else if (httpVersion.equals("HTTP/1.1") && httpHeader.http1_1.containsKey(Integer.toString(httpStatusCode))) + httpStatusText = (String) httpHeader.http1_1.get(Integer.toString(httpStatusCode)); + else httpStatusText = "Unknown"; + } + + + // write status line + theHeader.append(httpVersion).append(" ") + .append(Integer.toString(httpStatusCode)).append(" ") + .append(httpStatusText).append("\r\n"); + + // write header + Iterator i = keySet().iterator(); + String key, value; + char tag; + int count; + while (i.hasNext()) { + key = (String) i.next(); + tag = key.charAt(0); + if ((tag != '*') && (tag != '#')) { // '#' in key is reserved for proxy attributes as artificial header values + count = keyCount(key); + for (int j = 0; j < count; j++) { + theHeader.append(key).append(": ").append((String) getSingle(key, j)).append("\r\n"); + } + } + } + // end header + theHeader.append("\r\n"); + } + + public static URL getRequestURL(Properties conProp) throws MalformedURLException { + String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); + String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH); // always starts with leading '/' + String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); // may be null if no args were given + String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer + + int port, pos; + if ((pos = host.indexOf(":")) < 0) { + port = 80; + } else { + port = Integer.parseInt(host.substring(pos + 1)); + host = host.substring(0, pos); + } + + URL url = new URL("http", host, port, (args == null) ? path : path + "?" + args); + return url; + } + + public static void handleTransparentProxySupport(httpHeader header, Properties prop, String virtualHost, boolean isTransparentProxy) { + // transparent proxy support is only available for http 1.0 and above connections + if (prop.getProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9").equals("HTTP/0.9")) return; + + // if the transparent proxy support was disabled, we have nothing todo here ... + if (!(isTransparentProxy && header.containsKey(HOST))) return; + + try { + String dstHost, dstHostSocket = (String) header.get(HOST); + + int idx = dstHostSocket.indexOf(":"); + dstHost = (idx != -1) ? dstHostSocket.substring(0,idx).trim() : dstHostSocket.trim(); + Integer dstPort = (idx != -1) ? Integer.valueOf(dstHostSocket.substring(idx+1)) : new Integer(80); + + if (dstPort.intValue() == 80) { + if (dstHost.endsWith(".yacy")) { + // if this peer is accessed via its yacy domain name we need to set the + // host property to virtualHost to redirect the request to the yacy server + if (dstHost.endsWith(yacyCore.seedDB.mySeed.getName()+".yacy")) { + prop.setProperty(CONNECTION_PROP_HOST,virtualHost); + } else { + prop.setProperty(CONNECTION_PROP_HOST,dstHostSocket); + } + } else { + InetAddress dstHostAddress = InetAddress.getByName(dstHost); + if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) { + prop.setProperty(CONNECTION_PROP_HOST,dstHostSocket); + } + } + } + } catch (Exception e) {} + } } \ No newline at end of file diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index 593b6f819..144b92700 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -1257,36 +1257,23 @@ do upload status = Integer.toString(statusCode) + " " + statusText; return; // in bad mood } - String buffer = new String(b); // this is the status response line - //System.out.println("#S#" + buffer); - int p = buffer.indexOf(" "); - if (p < 0) { - statusCode = 500; - statusText = "status line parse error"; - status = Integer.toString(statusCode) + " " + statusText; - // flush in anything that comes without parsing - while ((b != null) && (b.length != 0)) b = serverCore.receive(clientInput, readLineBuffer, terminalMaxLength, false); - return; // in bad mood - } - - // the http version reported by the server - this.httpVer = buffer.substring(0,p); - // Status of the request, e.g. "200 OK" - this.status = buffer.substring(p + 1).trim(); // the status code plus reason-phrase + // parsing the response status line + String buffer = new String(b); + Object[] responseInfo = httpHeader.parseResponseLine(buffer); + this.httpVer = (String) responseInfo[0]; + this.statusCode = ((Integer)responseInfo[1]).intValue(); + this.statusText = (String) responseInfo[2]; + this.status = this.statusCode + " " + this.statusText; - // splitting the status into statuscode and statustext - p = this.status.indexOf(" "); - try { - this.statusCode = Integer.parseInt((p < 0) ? this.status.trim() : this.status.substring(0,p).trim()); - this.statusText = (p < 0) ? "" : this.status.substring(p+1).trim(); - } catch (Exception e) { - this.statusCode = 500; - this.statusText = this.status; + if ((this.statusCode==500)&&(this.statusText.equals("status line parse error"))) { + // flush in anything that comes without parsing + while ((b != null) && (b.length != 0)) b = serverCore.receive(clientInput, readLineBuffer, terminalMaxLength, false); + return; // in bad mood } - + // check validity - if (this.status.startsWith("400")) { + if (this.statusCode == 400) { // bad request // flush in anything that comes without parsing while ((b = serverCore.receive(clientInput, readLineBuffer, terminalMaxLength, false)).length != 0) {} @@ -1307,7 +1294,7 @@ do upload responseHeader.put(key, (String) responseHeader.get(key) + " " + buffer.trim()); } else { // create new entry - p = buffer.indexOf(":"); + int p = buffer.indexOf(":"); if (p > 0) { responseHeader.add(buffer.substring(0, p).trim(), buffer.substring(p + 1).trim()); } else { diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index a6cd6839a..74b62bee0 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -81,35 +81,6 @@ import de.anomic.yacy.yacyCore; */ public final class httpd implements serverHandler { - /* PROPERTIES: General properties */ - public static final String CONNECTION_PROP_HTTP_VER = "HTTP"; - public static final String CONNECTION_PROP_HOST = "HOST"; - public static final String CONNECTION_PROP_METHOD = "METHOD"; - public static final String CONNECTION_PROP_PATH = "PATH"; - public static final String CONNECTION_PROP_EXT = "EXT"; - public static final String CONNECTION_PROP_URL = "URL"; - public static final String CONNECTION_PROP_ARGS = "ARGS"; - public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP"; - public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT"; - public static final String CONNECTION_PROP_KEEP_ALIVE_COUNT = "KEEP-ALIVE_COUNT"; - public static final String CONNECTION_PROP_REQUESTLINE = "REQUESTLINE"; - public static final String CONNECTION_PROP_PREV_REQUESTLINE = "PREVREQUESTLINE"; - public static final String CONNECTION_PROP_REQUEST_START = "REQUEST_START"; - public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END"; - - /* PROPERTIES: Client -> Proxy */ - public static final String CONNECTION_PROP_CLIENT_REQUEST_HEADER = "CLIENT_REQUEST_HEADER"; - - /* PROPERTIES: Proxy -> Server */ - - /* PROPERTIES: Server -> Proxy */ - - /* PROPERTIES: Proxy -> Client */ - public static final String CONNECTION_PROP_PROXY_RESPOND_CODE = "PROXY_RESPOND_CODE"; - public static final String CONNECTION_PROP_PROXY_RESPOND_STATUS = "PROXY_RESPOND_STATUS"; - public static final String CONNECTION_PROP_PROXY_RESPOND_HEADER = "PROXY_RESPOND_HEADER"; - public static final String CONNECTION_PROP_PROXY_RESPOND_SIZE = "PROXY_REQUEST_SIZE"; - /** * A hashset containing extensions that indicate content that should not be transported * using zipped content encoding @@ -124,7 +95,7 @@ public final class httpd implements serverHandler { public static final String copyright = "[ HTTP SERVER: AnomicHTTPD v" + vDATE + " by Michael Christen / www.anomic.de ]"; public static final String hline = "-------------------------------------------------------------------------------"; - private static HashMap reverseMappingCache = new HashMap(); + public static HashMap reverseMappingCache = new HashMap(); private httpdHandler proxyHandler = null; // a servlet that holds the proxy functions private httpdHandler fileHandler = null; // a servlet that holds the file serving functions private httpdHandler soapHandler = null; @@ -255,78 +226,6 @@ public final class httpd implements serverHandler { return "501 Exception occurred: " + e.getMessage(); } - /** - * reads a line from the input socket - * this function is provided by the server through a passed method on initialization - * @return the next requestline as string - */ - private String readLine() { - byte[] l = this.session.readLine(); - return (l == null) ? null: new String(l); - } - - private httpHeader readHeader() throws IOException { - - // reading all headers - httpHeader header = new httpHeader(reverseMappingCache); - int p; - String line; - while ((line = readLine()) != null) { - if (line.length() == 0) break; // this seperates the header of the HTTP request from the body - // parse the header line: a property seperated with the ':' sign - if ((p = line.indexOf(":")) >= 0) { - // store a property - header.add(line.substring(0, p).trim(), line.substring(p + 1).trim()); - } - } - - /* - * doing some header validation here ... - */ - String httpVersion = this.prop.getProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); - if (httpVersion.equals("HTTP/1.1") && !header.containsKey(httpHeader.HOST)) { - // the HTTP/1.1 specification requires that an HTTP/1.1 server must reject any - // HTTP/1.1 message that does not contain a Host header. - httpd.sendRespondError(this.prop,this.session.out,0,400,null,null,null); - throw new IOException("400 Bad request"); - } - - return header; - } - - private void handleTransparentProxySupport(httpHeader header) { - // transparent proxy support is only available for http 1.0 and above connections - if (this.prop.getProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9").equals("HTTP/0.9")) return; - - // if the transparent proxy support was disabled, we have nothing todo here ... - if (!(httpdProxyHandler.isTransparentProxy && header.containsKey(httpHeader.HOST))) return; - - try { - String dstHost, dstHostSocket = (String) header.get(httpHeader.HOST); - - int idx = dstHostSocket.indexOf(":"); - dstHost = (idx != -1) ? dstHostSocket.substring(0,idx).trim() : dstHostSocket.trim(); - Integer dstPort = (idx != -1) ? Integer.valueOf(dstHostSocket.substring(idx+1)) : new Integer(80); - - if (dstPort.intValue() == 80) { - if (dstHost.endsWith(".yacy")) { - // if this peer is accessed via its yacy domain name we need to set the - // host property to virtualHost to redirect the request to the yacy server - if (dstHost.endsWith(yacyCore.seedDB.mySeed.getName()+".yacy")) { - this.prop.setProperty(httpd.CONNECTION_PROP_HOST,virtualHost); - } else { - this.prop.setProperty(httpd.CONNECTION_PROP_HOST,dstHostSocket); - } - } else { - InetAddress dstHostAddress = InetAddress.getByName(dstHost); - if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) { - this.prop.setProperty(httpd.CONNECTION_PROP_HOST,dstHostSocket); - } - } - } - } catch (Exception e) {} - } - /** * This funciton is used to determine if a persistent connection was requested by the * client. @@ -336,12 +235,12 @@ public final class httpd implements serverHandler { private boolean handlePersistentConnection(httpHeader header) { if (!keepAliveSupport) { - this.prop.put(CONNECTION_PROP_PERSISTENT,"close"); + this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); return false; } // getting the http version that is used by the client - String httpVersion = this.prop.getProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); + String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); // managing keep-alive: in HTTP/0.9 and HTTP/1.0 every connection is closed // afterwards. In HTTP/1.1 (and above, in the future?) connections are @@ -356,17 +255,17 @@ public final class httpd implements serverHandler { // if the request does not contain a content-length we have to close the connection // independently of the value of the connection header if (persistent && - this.prop.getProperty(httpd.CONNECTION_PROP_METHOD).equals(httpHeader.METHOD_POST) && + this.prop.getProperty(httpHeader.CONNECTION_PROP_METHOD).equals(httpHeader.METHOD_POST) && !header.containsKey(httpHeader.CONTENT_LENGTH)) - this.prop.put(CONNECTION_PROP_PERSISTENT,"close"); - else this.prop.put(CONNECTION_PROP_PERSISTENT,persistent?"keep-alive":"close"); + this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); + else this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,persistent?"keep-alive":"close"); return persistent; } private boolean handleServerAuthentication(httpHeader header) throws IOException { // getting the http version that is used by the client - String httpVersion = this.prop.getProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); + String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); // reading the authentication settings from switchboard if (this.serverAccountBase64MD5 == null) @@ -430,8 +329,8 @@ public final class httpd implements serverHandler { args = ""; } - parseQuery(unknownCommand, args); - String httpVersion = this.prop.getProperty(httpd.CONNECTION_PROP_HTTP_VER,"HTTP/0.9"); + parseRequestLine(unknownCommand, args); + String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER,"HTTP/0.9"); sendRespondError(this.prop,this.session.out,0,501,null,unknownCommand + " method not implemented",null); return serverCore.TERMINATE_CONNECTION; @@ -456,27 +355,27 @@ public final class httpd implements serverHandler { public Boolean GET(String arg) throws IOException { try { // parsing the http request line - parseQuery(httpHeader.METHOD_GET,arg); + parseRequestLine(httpHeader.METHOD_GET,arg); // we now know the HTTP version. depending on that, we read the header - String httpVersion = this.prop.getProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); + String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); httpHeader header = (httpVersion.equals("HTTP/0.9")) ? new httpHeader(reverseMappingCache) - : readHeader(); + : httpHeader.readHeader(this.prop,this.session); // handling transparent proxy support - this.handleTransparentProxySupport(header); + header.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy); // determines if the connection should be kept alive handlePersistentConnection(header); - if (this.prop.getProperty(CONNECTION_PROP_HOST).equals(virtualHost)) { + if (this.prop.getProperty(httpHeader.CONNECTION_PROP_HOST).equals(virtualHost)) { // pass to server if (this.allowServer) { /* * Handling SOAP Requests here ... */ - if (this.prop.containsKey(CONNECTION_PROP_PATH) && this.prop.getProperty(CONNECTION_PROP_PATH).startsWith("/soap")) { + if (this.prop.containsKey(httpHeader.CONNECTION_PROP_PATH) && this.prop.getProperty(httpHeader.CONNECTION_PROP_PATH).startsWith("/soap")) { if (soapHandler == null) { try { Class soapHandlerClass = Class.forName("de.anomic.soap.httpdSoapHandler"); @@ -523,7 +422,7 @@ public final class httpd implements serverHandler { } } - return this.prop.getProperty(CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; + return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; } catch (Exception e) { logUnexpectedError(e); return serverCore.TERMINATE_CONNECTION; @@ -552,16 +451,16 @@ public final class httpd implements serverHandler { public Boolean HEAD(String arg) throws IOException { try { - parseQuery(httpHeader.METHOD_HEAD,arg); + parseRequestLine(httpHeader.METHOD_HEAD,arg); // we now know the HTTP version. depending on that, we read the header httpHeader header; - String httpVersion = prop.getProperty("HTTP", "HTTP/0.9"); + String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); if (httpVersion.equals("HTTP/0.9")) header = new httpHeader(reverseMappingCache); - else header = readHeader(); + else header = httpHeader.readHeader(this.prop,this.session); // handle transparent proxy support - this.handleTransparentProxySupport(header); + header.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy); // determines if the connection should be kept alive boolean persistent = handlePersistentConnection(header); @@ -594,7 +493,7 @@ public final class httpd implements serverHandler { return serverCore.TERMINATE_CONNECTION; } } - return this.prop.getProperty(CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; + return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; } catch (Exception e) { logUnexpectedError(e); return serverCore.TERMINATE_CONNECTION; @@ -603,16 +502,16 @@ public final class httpd implements serverHandler { public Boolean POST(String arg) throws IOException { try { - parseQuery("POST",arg); + parseRequestLine(httpHeader.METHOD_POST,arg); // we now know the HTTP version. depending on that, we read the header httpHeader header; String httpVersion = prop.getProperty("HTTP", "HTTP/0.9"); if (httpVersion.equals("HTTP/0.9")) header = new httpHeader(reverseMappingCache); - else header = readHeader(); + else header = httpHeader.readHeader(this.prop,this.session); // handle transparent proxy support - this.handleTransparentProxySupport(header); + header.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy); // determines if the connection should be kept alive boolean persistent = handlePersistentConnection(header); @@ -676,7 +575,7 @@ public final class httpd implements serverHandler { } } //return serverCore.RESUME_CONNECTION; - return this.prop.getProperty(CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; + return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; } catch (Exception e) { logUnexpectedError(e); return serverCore.TERMINATE_CONNECTION; @@ -695,10 +594,10 @@ public final class httpd implements serverHandler { httpVersion = arg.substring(pos + 1); arg = arg.substring(0, pos); } - prop.setProperty(httpd.CONNECTION_PROP_HTTP_VER, httpVersion); + prop.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpVersion); // parse hostname and port - prop.setProperty(httpd.CONNECTION_PROP_HOST, arg); + prop.setProperty(httpHeader.CONNECTION_PROP_HOST, arg); pos = arg.indexOf(":"); int port = 443; if (pos >= 0) { @@ -707,14 +606,14 @@ public final class httpd implements serverHandler { } // setting other connection properties - prop.setProperty(httpd.CONNECTION_PROP_CLIENTIP, this.clientIP); - prop.setProperty(httpd.CONNECTION_PROP_METHOD, httpHeader.METHOD_CONNECT); - prop.setProperty(httpd.CONNECTION_PROP_PATH, "/"); - prop.setProperty(httpd.CONNECTION_PROP_EXT, ""); - prop.setProperty(httpd.CONNECTION_PROP_URL, ""); + prop.setProperty(httpHeader.CONNECTION_PROP_CLIENTIP, this.clientIP); + prop.setProperty(httpHeader.CONNECTION_PROP_METHOD, httpHeader.METHOD_CONNECT); + prop.setProperty(httpHeader.CONNECTION_PROP_PATH, "/"); + prop.setProperty(httpHeader.CONNECTION_PROP_EXT, ""); + prop.setProperty(httpHeader.CONNECTION_PROP_URL, ""); // parse remaining lines - httpHeader header = readHeader(); + httpHeader header = httpHeader.readHeader(this.prop,this.session); if (!(allowProxy)) { // not authorized through firewall blocking (ip does not match filter) @@ -744,111 +643,23 @@ public final class httpd implements serverHandler { return serverCore.TERMINATE_CONNECTION; } - - private final Properties parseQuery(String cmd, String s) { + private final void parseRequestLine(String cmd, String s) { - // getting the last request line for debugging purposes - String prevRequestLine = this.prop.containsKey(CONNECTION_PROP_REQUESTLINE)? - this.prop.getProperty(CONNECTION_PROP_REQUESTLINE) : ""; + // parsing the header + httpHeader.parseRequestLine(cmd,s,this.prop,virtualHost); - // reset property from previous run - this.prop.clear(); + // reseting the empty request counter this.emptyRequestCount = 0; - - // storing informations about the request - this.prop.setProperty(CONNECTION_PROP_METHOD, cmd); - this.prop.setProperty(CONNECTION_PROP_REQUESTLINE,cmd + " " + s); - this.prop.setProperty(CONNECTION_PROP_PREV_REQUESTLINE,prevRequestLine); - this.prop.setProperty(CONNECTION_PROP_CLIENTIP, this.clientIP); // counting the amount of received requests within this permanent conneciton - this.prop.setProperty(CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer.toString(++this.keepAliveRequestCount)); - - // this parses a whole URL - if (s.length() == 0) { - this.prop.setProperty(CONNECTION_PROP_HOST, virtualHost); - this.prop.setProperty(CONNECTION_PROP_PATH, "/"); - this.prop.setProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); - this.prop.setProperty(CONNECTION_PROP_EXT, ""); - return this.prop; - } - - // store the version propery "HTTP" and cut the query at both ends - int sep = s.indexOf(" "); - if (sep >= 0) { - // HTTP version is given - this.prop.setProperty(CONNECTION_PROP_HTTP_VER, s.substring(sep + 1).trim()); - s = s.substring(0, sep).trim(); // cut off HTTP version mark - } else { - // HTTP version is not given, it will be treated as ver 0.9 - this.prop.setProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); - } - - // properties of the query are stored with the prefix "&" - // additionally, the values URL and ARGC are computed - - String argsString = ""; - sep = s.indexOf("?"); - if (sep >= 0) { - // there are values attached to the query string - argsString = s.substring(sep + 1); // cut haed from tail of query - s = s.substring(0, sep); - } - this.prop.setProperty(CONNECTION_PROP_URL, s); // store URL - //System.out.println("HTTPD: ARGS=" + argsString); - if (argsString.length() != 0) this.prop.setProperty(CONNECTION_PROP_ARGS, argsString); // store arguments in original form + this.prop.setProperty(httpHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer.toString(++this.keepAliveRequestCount)); - // find out file extension - sep = s.lastIndexOf("."); - if (sep >= 0) { - if (s.indexOf("?", sep + 1) >= sep) - this.prop.setProperty(CONNECTION_PROP_EXT, s.substring(sep + 1, s.indexOf("?", sep + 1)).toLowerCase()); - else if (s.indexOf("#", sep + 1) >= sep) - this.prop.setProperty(CONNECTION_PROP_EXT, s.substring(sep + 1, s.indexOf("#", sep + 1)).toLowerCase()); - else - this.prop.setProperty(CONNECTION_PROP_EXT, s.substring(sep + 1).toLowerCase()); - } else { - this.prop.setProperty(CONNECTION_PROP_EXT, ""); - } - - // finally find host string - if (s.toUpperCase().startsWith("HTTP://")) { - // a host was given. extract it and set path - s = s.substring(7); - sep = s.indexOf("/"); - if (sep < 0) { - // this is a malformed url, something like - // http://index.html - // we are lazy and guess that it means - // /index.html - // which is a localhost access to the file servlet - this.prop.setProperty(CONNECTION_PROP_HOST, virtualHost); - this.prop.setProperty(CONNECTION_PROP_PATH, "/" + s); - } else { - // THIS IS THE "GOOD" CASE - // a perfect formulated url - this.prop.setProperty(CONNECTION_PROP_HOST, s.substring(0, sep)); - this.prop.setProperty(CONNECTION_PROP_PATH, s.substring(sep)); // yes, including beginning "/" - } - } else { - // no host in url. set path - if (s.startsWith("/")) { - // thats also fine, its a perfect localhost access - // in this case, we simulate a - // http://localhost/s - // access by setting a virtual host - this.prop.setProperty(CONNECTION_PROP_HOST, virtualHost); - this.prop.setProperty(CONNECTION_PROP_PATH, s); - } else { - // the client 'forgot' to set a leading '/' - // this is the same case as above, with some lazyness - this.prop.setProperty(CONNECTION_PROP_HOST, virtualHost); - this.prop.setProperty(CONNECTION_PROP_PATH, "/" + s); - } - } - return this.prop; + // setting the client-IP + this.prop.setProperty(httpHeader.CONNECTION_PROP_CLIENTIP, this.clientIP); } + + // some static methods that needs to be used from any CGI // and also by the httpdFileHandler @@ -1109,7 +920,7 @@ public final class httpd implements serverHandler { ByteArrayOutputStream o = null; try { // setting the proper http status message - String httpVersion = conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER,"HTTP/1.1"); + String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER,"HTTP/1.1"); if ((httpStatusText == null)||(httpStatusText.length()==0)) { if (httpVersion.equals("HTTP/1.0") && httpHeader.http1_0.containsKey(Integer.toString(httpStatusCode))) httpStatusText = (String) httpHeader.http1_0.get(Integer.toString(httpStatusCode)); @@ -1119,10 +930,10 @@ public final class httpd implements serverHandler { } // generating the desired request url - String host = conProp.getProperty(httpd.CONNECTION_PROP_HOST); - String path = conProp.getProperty(httpd.CONNECTION_PROP_PATH,"/"); - String args = conProp.getProperty(httpd.CONNECTION_PROP_ARGS); - String method = conProp.getProperty(httpd.CONNECTION_PROP_METHOD); + String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); + String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH,"/"); + String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); + String method = conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD); int port = 80, pos = host.indexOf(":"); if (pos != -1) { @@ -1150,7 +961,7 @@ public final class httpd implements serverHandler { tp.put("errorMessageType", errorcase); tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText); - tp.put("requestMethod", conProp.getProperty(httpd.CONNECTION_PROP_METHOD)); + tp.put("requestMethod", conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD)); tp.put("requestURL", urlString); tp.put("errorMessageType_detailedErrorMsg",(detailedErrorMsg != null) ? detailedErrorMsg : ""); @@ -1270,7 +1081,7 @@ public final class httpd implements serverHandler { if (respond == null) throw new NullPointerException("The outputstream must not be null."); if (conProp == null) throw new NullPointerException("The connection property structure must not be null."); - if (httpVersion == null) httpVersion = conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER,"HTTP/1.1"); + if (httpVersion == null) httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER,"HTTP/1.1"); try { if ((httpStatusText == null)||(httpStatusText.length()==0)) { @@ -1286,21 +1097,21 @@ public final class httpd implements serverHandler { header.put(httpHeader.DATE, httpc.dateString(httpc.nowDate())); if (!header.containsKey(httpHeader.CONTENT_TYPE)) header.put(httpHeader.CONTENT_TYPE, "text/html"); // fix this - if (!header.containsKey(httpHeader.CONNECTION) && conProp.containsKey(CONNECTION_PROP_PERSISTENT)) - header.put(httpHeader.CONNECTION, conProp.getProperty(CONNECTION_PROP_PERSISTENT)); - if (!header.containsKey(httpHeader.PROXY_CONNECTION) && conProp.containsKey(CONNECTION_PROP_PERSISTENT)) - header.put(httpHeader.PROXY_CONNECTION, conProp.getProperty(CONNECTION_PROP_PERSISTENT)); + if (!header.containsKey(httpHeader.CONNECTION) && conProp.containsKey(httpHeader.CONNECTION_PROP_PERSISTENT)) + header.put(httpHeader.CONNECTION, conProp.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT)); + if (!header.containsKey(httpHeader.PROXY_CONNECTION) && conProp.containsKey(httpHeader.CONNECTION_PROP_PERSISTENT)) + header.put(httpHeader.PROXY_CONNECTION, conProp.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT)); - if (conProp.containsKey(CONNECTION_PROP_PERSISTENT) && - conProp.getProperty(CONNECTION_PROP_PERSISTENT).equals("keep-alive") && + if (conProp.containsKey(httpHeader.CONNECTION_PROP_PERSISTENT) && + conProp.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") && !header.containsKey(httpHeader.TRANSFER_ENCODING) && !header.containsKey(httpHeader.CONTENT_LENGTH)) header.put(httpHeader.CONTENT_LENGTH, "0"); // adding some yacy specific headers - header.put(httpHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT,conProp.getProperty(CONNECTION_PROP_KEEP_ALIVE_COUNT)); - header.put(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE,conProp.getProperty(CONNECTION_PROP_REQUESTLINE)); - header.put(httpHeader.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(CONNECTION_PROP_PREV_REQUESTLINE)); + header.put(httpHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT,conProp.getProperty(httpHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT)); + header.put(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE)); + header.put(httpHeader.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(httpHeader.CONNECTION_PROP_PREV_REQUESTLINE)); StringBuffer headerStringBuffer = new StringBuffer(560); @@ -1334,8 +1145,8 @@ public final class httpd implements serverHandler { respond.write(headerStringBuffer.toString().getBytes()); respond.flush(); - conProp.put(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER,header); - conProp.put(httpd.CONNECTION_PROP_PROXY_RESPOND_STATUS,Integer.toString(httpStatusCode)); + conProp.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER,header); + conProp.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS,Integer.toString(httpStatusCode)); } catch (Exception e) { // any interruption may be caused be network error or because the user has closed // the windows during transmission. We simply pass it as IOException diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index 75906eebc..432f3687e 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -245,10 +245,10 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http this.connectionProperties = conProp; // getting some connection properties - String method = conProp.getProperty(httpd.CONNECTION_PROP_METHOD); - String path = conProp.getProperty(httpd.CONNECTION_PROP_PATH); - String argsString = conProp.getProperty(httpd.CONNECTION_PROP_ARGS); // is null if no args were given - String httpVersion= conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER); + String method = conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD); + String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH); + String argsString = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); // is null if no args were given + String httpVersion= conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); String url = "http://" + requestHeader.get(httpHeader.HOST,"localhost") + path; // check hack attacks in path @@ -302,7 +302,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http (adminAccountBase64MD5.length() != 0) && (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(authorization.trim().substring(6))))) { // remove brute-force flag - serverCore.bfHost.remove(conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP)); + serverCore.bfHost.remove(conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP)); } // parse arguments @@ -597,10 +597,10 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http errorMessage.append("\nSession: ").append(Thread.currentThread().getName()) .append("\nQuery: ").append(path) - .append("\nClient: ").append(conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP,"unknown")) + .append("\nClient: ").append(conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP,"unknown")) .append("\nReason: ").append(e.toString()); - if (!conProp.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { + if (!conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { // sending back an error message to the client // if we have not already send an http header httpd.sendRespondError(conProp,out, 4, httpStatusCode, httpStatusText, errorMessage.toString(),errorExc); @@ -629,7 +629,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http private void forceConnectionClose() { if (this.connectionProperties != null) { - this.connectionProperties.setProperty(httpd.CONNECTION_PROP_PERSISTENT,"close"); + this.connectionProperties.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); } } diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 28e2b1bde..3ee2ff179 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -4,8 +4,9 @@ // (C) by Michael Peter Christen; mc@anomic.de // first published on http://www.anomic.de // Frankfurt, Germany, 2004 -//last major change: $LastChangedDate$ by $LastChangedBy$ -//Revision: $LastChangedRevision$ +// +// last major change: $LastChangedDate$ by $LastChangedBy$ +// Revision: $LastChangedRevision$ // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -305,17 +306,17 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt try { // remembering the starting time of the request Date requestDate = new Date(); // remember the time... - this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); + this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); switchboard.proxyLastAccess = System.currentTimeMillis(); // using an ByteCount OutputStream to count the send bytes (needed for the logfile) - respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpd.CONNECTION_PROP_REQUESTLINE).length() + 2); + respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2); - String host = conProp.getProperty(httpd.CONNECTION_PROP_HOST); - String path = conProp.getProperty(httpd.CONNECTION_PROP_PATH); // always starts with leading '/' - String args = conProp.getProperty(httpd.CONNECTION_PROP_ARGS); // may be null if no args were given - String ip = conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer + String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); + String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH); // always starts with leading '/' + String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); // may be null if no args were given + String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer int port, pos; if ((pos = host.indexOf(":")) < 0) { @@ -334,7 +335,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt URL url = null; try { - url = new URL("http", host, port, (args == null) ? path : path + "?" + args); + url = httpHeader.getRequestURL(conProp); } catch (MalformedURLException e) { String errorMsg = "ERROR: internal error with url generation: host=" + host + ", port=" + port + ", path=" + path + ", args=" + args; @@ -364,7 +365,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt } // setting the X-Forwarded-For Header - requestHeader.put(httpHeader.X_FORWARDED_FOR,conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP)); + requestHeader.put(httpHeader.X_FORWARDED_FOR,conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP)); // decide wether to use a cache entry or connect to the network File cacheFile = cacheManager.getCachePath(url); @@ -418,7 +419,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt String exTxt = e.getMessage(); if ((exTxt!=null)&&(exTxt.startsWith("Socket closed"))) { this.forceConnectionClose(); - } else if (!conProp.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { + } else if (!conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { String errorMsg = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage(); httpd.sendRespondError(conProp,respond,4,501,null,errorMsg,e); this.theLogger.logSevere(errorMsg); @@ -432,8 +433,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt try { respond.flush(); } catch (Exception e) {} if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish(); - this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis())); - this.connectionProperties.put(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount())); + this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis())); + this.connectionProperties.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount())); this.logProxyAccess(); } } @@ -448,11 +449,11 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt httpc.response res = null; try { - String host = conProp.getProperty(httpd.CONNECTION_PROP_HOST); - String path = conProp.getProperty(httpd.CONNECTION_PROP_PATH); // always starts with leading '/' - String args = conProp.getProperty(httpd.CONNECTION_PROP_ARGS); // may be null if no args were given - String ip = conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer - String httpVer = conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER); // the ip from the connecting peer + String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); + String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH); // always starts with leading '/' + String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); // may be null if no args were given + String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer + String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); // the ip from the connecting peer int port, pos; if ((pos = host.indexOf(":")) < 0) { @@ -482,7 +483,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt // send request res = remote.GET(remotePath, requestHeader); - conProp.put(httpd.CONNECTION_PROP_CLIENT_REQUEST_HEADER,requestHeader); + conProp.put(httpHeader.CONNECTION_PROP_CLIENT_REQUEST_HEADER,requestHeader); // determine if it's an internal error of the httpc if (res.responseHeader.size() == 0) { @@ -500,7 +501,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt res.responseHeader.put(httpHeader.CONTENT_LENGTH,"0"); } else { if (httpVer.equals("HTTP/0.9") || httpVer.equals("HTTP/1.0")) { - conProp.setProperty(httpd.CONNECTION_PROP_PERSISTENT,"close"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); } else { chunkedOut = new httpChunkedOutputStream(respond); } @@ -520,11 +521,11 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt // delete the cache sizeBeforeDelete = cacheFile.length(); cacheManager.deleteFile(url); - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS"); } // reserver cache entry - Date requestDate = new Date(((Long)conProp.get(httpd.CONNECTION_PROP_REQUEST_START)).longValue()); + Date requestDate = new Date(((Long)conProp.get(httpHeader.CONNECTION_PROP_REQUEST_START)).longValue()); plasmaHTCache.Entry cacheEntry = cacheManager.newEntry( requestDate, 0, @@ -585,19 +586,19 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt //cacheEntry.status = plasmaHTCache.CACHE_FILL; // it's an insert cacheEntry.cacheArray = cacheArray; cacheManager.push(cacheEntry); - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); } else if (sizeBeforeDelete == cacheArray.length) { // before we came here we deleted a cache entry cacheArray = null; //cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_BAD; //cacheManager.push(cacheEntry); // unnecessary update - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REF_FAIL_HIT"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REF_FAIL_HIT"); } else { // before we came here we deleted a cache entry //cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD; cacheEntry.cacheArray = cacheArray; cacheManager.push(cacheEntry); // necessary update, write response header to cache - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS"); } } else { // the file is too big to cache it in the ram, or the size is unknown @@ -611,17 +612,17 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt // totally fresh file //cacheEntry.status = plasmaHTCache.CACHE_FILL; // it's an insert cacheManager.push(cacheEntry); - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); } else if (sizeBeforeDelete == cacheFile.length()) { // before we came here we deleted a cache entry //cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_BAD; //cacheManager.push(cacheEntry); // unnecessary update - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REF_FAIL_HIT"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REF_FAIL_HIT"); } else { // before we came here we deleted a cache entry //cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD; cacheManager.push(cacheEntry); // necessary update, write response header to cache - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS"); } // beware! all these writings will not fill the cacheEntry.cacheArray // that means they are not available for the indexer (except they are scraped before) @@ -640,7 +641,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt //cacheEntry.status = plasmaHTCache.CACHE_STALE_NO_RELOAD; //cacheManager.push(cacheEntry); } - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); } if (gzippedOut != null) { @@ -670,7 +671,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt OutputStream respond ) throws IOException { - String httpVer = conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER); + String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); httpChunkedOutputStream chunkedOut = null; GZIPOutputStream gzippedOut = null; @@ -700,7 +701,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt // conditional request: freshness of cache for that condition was already // checked within shallUseCache(). Now send only a 304 response this.theLogger.logInfo("CACHE HIT/304 " + cacheFile.toString()); - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_HIT"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_HIT"); // setting the content length header to 0 cachedResponseHeader.put(httpHeader.CONTENT_LENGTH, Integer.toString(0)); @@ -711,7 +712,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt } else { // unconditional request: send content of cache this.theLogger.logInfo("CACHE HIT/203 " + cacheFile.toString()); - conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_HIT"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_HIT"); // setting the content header to the proper length cachedResponseHeader.put(httpHeader.CONTENT_LENGTH, Long.toString(cacheFile.length())); @@ -741,9 +742,9 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt } catch (Exception e) { // this happens if the client stops loading the file // we do nothing here - if (conProp.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { + if (conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { this.theLogger.logWarning("Error while trying to send cached message body."); - conProp.setProperty(httpd.CONNECTION_PROP_PERSISTENT,"close"); + conProp.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); } else { httpd.sendRespondError(conProp,respond,4,503,"socket error: " + e.getMessage(),"socket error: " + e.getMessage(), e); } @@ -785,7 +786,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt private void forceConnectionClose() { if (this.connectionProperties != null) { - this.connectionProperties.setProperty(httpd.CONNECTION_PROP_PERSISTENT,"close"); + this.connectionProperties.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); } } @@ -798,17 +799,17 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt try { // remembering the starting time of the request Date requestDate = new Date(); // remember the time... - this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); + this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); switchboard.proxyLastAccess = System.currentTimeMillis(); // using an ByteCount OutputStream to count the send bytes - respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpd.CONNECTION_PROP_REQUESTLINE).length() + 2); + respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2); - String host = conProp.getProperty(httpd.CONNECTION_PROP_HOST); - String path = conProp.getProperty(httpd.CONNECTION_PROP_PATH); - String args = conProp.getProperty(httpd.CONNECTION_PROP_ARGS); - String httpVer = conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER); + String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); + String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH); + String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); + String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); switchboard.proxyLastAccess = System.currentTimeMillis(); @@ -846,7 +847,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt } // setting the X-Forwarded-For Header - requestHeader.put(httpHeader.X_FORWARDED_FOR,conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP)); + requestHeader.put(httpHeader.X_FORWARDED_FOR,conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP)); // resolve yacy and yacyh domains String yAddress = yacyCore.seedDB.resolveYacyAddress(host); @@ -894,17 +895,17 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt try { // remembering the starting time of the request Date requestDate = new Date(); // remember the time... - this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); + this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); switchboard.proxyLastAccess = System.currentTimeMillis(); // using an ByteCount OutputStream to count the send bytes - respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpd.CONNECTION_PROP_REQUESTLINE).length() + 2); + respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2); - String host = conProp.getProperty(httpd.CONNECTION_PROP_HOST); - String path = conProp.getProperty(httpd.CONNECTION_PROP_PATH); - String args = conProp.getProperty(httpd.CONNECTION_PROP_ARGS); // may be null if no args were given - String httpVer = conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER); + String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); + String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH); + String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); // may be null if no args were given + String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); int port, pos; if ((pos = host.indexOf(":")) < 0) { @@ -931,7 +932,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt } // setting the X-Forwarded-For Header - requestHeader.put(httpHeader.X_FORWARDED_FOR,conProp.getProperty(httpd.CONNECTION_PROP_CLIENTIP)); + requestHeader.put(httpHeader.X_FORWARDED_FOR,conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP)); // resolve yacy and yacyh domains String yAddress = yacyCore.seedDB.resolveYacyAddress(host); @@ -1004,8 +1005,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt respond.flush(); if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish(); - this.connectionProperties.put(httpd.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis())); - this.connectionProperties.put(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount())); + this.connectionProperties.put(httpHeader.CONNECTION_PROP_REQUEST_END,new Long(System.currentTimeMillis())); + this.connectionProperties.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE,new Long(((httpdByteCountOutputStream)respond).getCount())); this.logProxyAccess(); } } @@ -1014,8 +1015,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt this.connectionProperties = conProp; switchboard.proxyLastAccess = System.currentTimeMillis(); - String host = conProp.getProperty(httpd.CONNECTION_PROP_HOST); - String httpVersion = conProp.getProperty(httpd.CONNECTION_PROP_HTTP_VER); + String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); + String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); int timeout = Integer.parseInt(switchboard.getConfig("clientTimeout", "10000")); int port, pos; @@ -1252,17 +1253,17 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt } // sending back an error message to the client - if (!conProp.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { + if (!conProp.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { httpd.sendRespondError(conProp,respond,4,httpStatusCode,httpStatusText,errorMessage,errorExc); } else { if (unknownError) { this.theLogger.logFine("Error while processing request '" + - conProp.getProperty(httpd.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" + + conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" + "\n" + Thread.currentThread().getName() + "\n" + errorMessage,e); } else { this.theLogger.logFine("Error while processing request '" + - conProp.getProperty(httpd.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" + + conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" + "\n" + Thread.currentThread().getName() + "\n" + errorMessage); } @@ -1315,8 +1316,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt this.logMessage.append(' '); // Elapsed time - Long requestStart = (Long) this.connectionProperties.get(httpd.CONNECTION_PROP_REQUEST_START); - Long requestEnd = (Long) this.connectionProperties.get(httpd.CONNECTION_PROP_REQUEST_END); + Long requestStart = (Long) this.connectionProperties.get(httpHeader.CONNECTION_PROP_REQUEST_START); + Long requestEnd = (Long) this.connectionProperties.get(httpHeader.CONNECTION_PROP_REQUEST_END); String elapsed = Long.toString(requestEnd.longValue()-requestStart.longValue()); for (int i=0; i<6-elapsed.length(); i++) this.logMessage.append(' '); @@ -1324,31 +1325,31 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt this.logMessage.append(' '); // Remote Host - String clientIP = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_CLIENTIP); + String clientIP = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); this.logMessage.append(clientIP); this.logMessage.append(' '); // Code/Status - String respondStatus = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_STATUS); - String respondCode = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"UNKNOWN"); + String respondStatus = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS); + String respondCode = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"UNKNOWN"); this.logMessage.append(respondCode); this.logMessage.append("/"); this.logMessage.append(respondStatus); this.logMessage.append(' '); // Bytes - Long bytes = (Long) this.connectionProperties.get(httpd.CONNECTION_PROP_PROXY_RESPOND_SIZE); + Long bytes = (Long) this.connectionProperties.get(httpHeader.CONNECTION_PROP_PROXY_RESPOND_SIZE); this.logMessage.append(bytes.toString()); this.logMessage.append(' '); // Method - String requestMethod = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_METHOD); + String requestMethod = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_METHOD); this.logMessage.append(requestMethod); this.logMessage.append(' '); // URL - String requestURL = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_URL); - String requestArgs = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_ARGS); + String requestURL = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_URL); + String requestArgs = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_ARGS); this.logMessage.append(requestURL); if (requestArgs != null) { this.logMessage.append("?") @@ -1361,15 +1362,15 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt this.logMessage.append(' '); // Peerstatus/Peerhost - String host = this.connectionProperties.getProperty(httpd.CONNECTION_PROP_HOST); + String host = this.connectionProperties.getProperty(httpHeader.CONNECTION_PROP_HOST); this.logMessage.append("DIRECT/"); this.logMessage.append(host); this.logMessage.append(' '); // Type String mime = "-"; - if (this.connectionProperties.containsKey(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { - httpHeader proxyRespondHeader = (httpHeader) this.connectionProperties.get(httpd.CONNECTION_PROP_PROXY_RESPOND_HEADER); + if (this.connectionProperties.containsKey(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER)) { + httpHeader proxyRespondHeader = (httpHeader) this.connectionProperties.get(httpHeader.CONNECTION_PROP_PROXY_RESPOND_HEADER); mime = proxyRespondHeader.mime(); if (mime.indexOf(";") != -1) { mime = mime.substring(0,mime.indexOf(";")); diff --git a/source/de/anomic/icap/icapHeader.java b/source/de/anomic/icap/icapHeader.java new file mode 100644 index 000000000..3e21f8e57 --- /dev/null +++ b/source/de/anomic/icap/icapHeader.java @@ -0,0 +1,303 @@ +//icapHeader.java +//----------------------- +//(C) by Michael Peter Christen; mc@anomic.de +//first published on http://www.anomic.de +//Frankfurt, Germany, 2004 +// +//This file is contributed by Martin Thelian +//last major change: $LastChangedDate$ by $LastChangedBy$ +//Revision: $LastChangedRevision$ +// +//This program is free software; you can redistribute it and/or modify +//it under the terms of the GNU General Public License as published by +//the Free Software Foundation; either version 2 of the License, or +//(at your option) any later version. +// +//This program is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. +// +//You should have received a copy of the GNU General Public License +//along with this program; if not, write to the Free Software +//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +//Using this software in any meaning (reading, learning, copying, compiling, +//running) means that you agree that the Author(s) is (are) not responsible +//for cost, loss of data or any harm that may be caused directly or indirectly +//by usage of this softare or this documentation. The usage of this software +//is on your own risk. The installation and usage (starting/running) of this +//software may allow other people or application to access your computer and +//any attached devices and is highly dependent on the configuration of the +//software which must be done by the user of the software; the author(s) is +//(are) also not responsible for proper configuration and usage of the +//software, even if provoked by documentation provided together with +//the software. +// +//Any changes to this file according to the GPL as documented in the file +//gpl.txt aside this file in the shipment you received can be done to the +//lines that follows this copyright notice here, but changes must not be +//done inside the copyright notive above. A re-distribution must contain +//the intact and unchanged copyright notice. +//Contributions and changes to the program code must be marked as such. + + +package de.anomic.icap; + +import java.io.IOException; +import java.text.Collator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; + +import de.anomic.server.serverCore; + +public class icapHeader extends TreeMap implements Map { + + /* ============================================================= + * Constants defining icap methods + * ============================================================= */ + public static final String METHOD_REQMOD = "REQMOD"; + public static final String METHOD_RESPMOD = "RESPMOD"; + public static final String METHOD_OPTIONS = "OPTIONS"; + + /* ============================================================= + * Constants defining http header names + * ============================================================= */ + public static final String HOST = "Host"; + public static final String USER_AGENT = "User-Agent"; + public static final String CONNECTION = "Connection"; + public static final String DATE = "Date"; + public static final String SERVER = "Server"; + public static final String ISTAG = "ISTAG"; + public static final String METHODS = "Methods"; + public static final String ALLOW = "Allow"; + public static final String ENCAPSULATED = "Encapsulated"; + public static final String MAX_CONNECTIONS = "Max-Connections"; + public static final String OPTIONS_TTL = "Options-TTL"; + public static final String SERVICE = "Service"; + public static final String SERVICE_ID = "Service-ID"; + public static final String PREVIEW = "Preview"; + public static final String TRANSFER_PREVIEW = "Transfer-Preview"; + public static final String TRANSFER_IGNORE = "Transfer-Ignore"; + public static final String TRANSFER_COMPLETE = "Transfer-Complete"; + + public static final String X_YACY_KEEP_ALIVE_REQUEST_COUNT = "X-Keep-Alive-Request-Count"; + + /* ============================================================= + * defining default icap status messages + * ============================================================= */ + public static final HashMap icap1_0 = new HashMap(); + static { + // (1yz) Informational codes + icap1_0.put("100","Continue after ICAP preview"); + + // (2yz) Success codes: + icap1_0.put("200","OK"); + icap1_0.put("204","No modifications needed"); + + // (4yz) Client error codes: + icap1_0.put("400","Bad request"); + icap1_0.put("404","ICAP Service not found"); + icap1_0.put("405","Method not allowed for service"); + icap1_0.put("408","Request timeout"); + + // (5yz) Server error codes: + icap1_0.put("500","Server error"); + icap1_0.put("501","Method not implemented"); + icap1_0.put("502","Bad Gateway"); + icap1_0.put("503","Service overloaded"); + icap1_0.put("505","ICAP version not supported by server"); + } + + /* PROPERTIES: General properties */ + public static final String CONNECTION_PROP_ICAP_VER = "ICAP"; + public static final String CONNECTION_PROP_HOST = "HOST"; + public static final String CONNECTION_PROP_PATH = "PATH"; + public static final String CONNECTION_PROP_EXT = "EXT"; + public static final String CONNECTION_PROP_METHOD = "METHOD"; + public static final String CONNECTION_PROP_REQUESTLINE = "REQUESTLINE"; + public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP"; + public static final String CONNECTION_PROP_URL = "URL"; + public static final String CONNECTION_PROP_ARGS = "ARGS"; + public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT"; + public static final String CONNECTION_PROP_KEEP_ALIVE_COUNT = "KEEP-ALIVE_COUNT"; + + private static final Collator insensitiveCollator = Collator.getInstance(Locale.US); + static { + insensitiveCollator.setStrength(Collator.SECONDARY); + insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION); + } + + public icapHeader() { + super(insensitiveCollator); + } + + public boolean allow(int statusCode) { + if (!super.containsKey("Allow")) return false; + + String allow = (String)get("Allow"); + return (allow.indexOf(Integer.toString(statusCode))!=-1); + } + + // to make the occurrence of multiple keys possible, we add them using a counter + public Object add(Object key, Object value) { + int c = keyCount((String) key); + if (c == 0) return put(key, value); else return put("*" + key + "-" + c, value); + } + + public int keyCount(String key) { + if (!(containsKey(key))) return 0; + int c = 1; + while (containsKey("*" + key + "-" + c)) c++; + return c; + } + + // a convenience method to access the map with fail-over defaults + public Object get(Object key, Object dflt) { + Object result = get(key); + if (result == null) return dflt; else return result; + } + + // return multiple results + public Object getSingle(Object key, int count) { + if (count == 0) return get(key, null); + return get("*" + key + "-" + count, null); + } + + public StringBuffer toHeaderString(String icapVersion, int icapStatusCode, String icapStatusText) { + + if ((icapStatusText == null)||(icapStatusText.length()==0)) { + if (icapVersion.equals("ICAP/1.0") && icapHeader.icap1_0.containsKey(Integer.toString(icapStatusCode))) + icapStatusText = (String) icapHeader.icap1_0.get(Integer.toString(icapStatusCode)); + } + + StringBuffer theHeader = new StringBuffer(); + + // write status line + theHeader.append(icapVersion).append(" ") + .append(Integer.toString(icapStatusCode)).append(" ") + .append(icapStatusText).append("\r\n"); + + // write header + Iterator i = keySet().iterator(); + String key, value; + char tag; + int count; + while (i.hasNext()) { + key = (String) i.next(); + tag = key.charAt(0); + if ((tag != '*') && (tag != '#')) { // '#' in key is reserved for proxy attributes as artificial header values + count = keyCount(key); + for (int j = 0; j < count; j++) { + theHeader.append(key).append(": ").append((String) getSingle(key, j)).append("\r\n"); + } + } + } + // end header + theHeader.append("\r\n"); + + + return theHeader; + } + + public static Properties parseRequestLine(String cmd, String s, Properties prop, String virtualHost) { + + // reset property from previous run + prop.clear(); + + // storing informations about the request + prop.setProperty(CONNECTION_PROP_METHOD, cmd); + prop.setProperty(CONNECTION_PROP_REQUESTLINE,cmd + " " + s); + + + // this parses a whole URL + if (s.length() == 0) { + prop.setProperty(CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(CONNECTION_PROP_PATH, "/"); + prop.setProperty(CONNECTION_PROP_ICAP_VER, "ICAP/1.0"); + prop.setProperty(CONNECTION_PROP_EXT, ""); + return prop; + } + + // store the version propery "ICAP" and cut the query at both ends + int sep = s.indexOf(" "); + if (sep >= 0) { + // ICAP version is given + prop.setProperty(CONNECTION_PROP_ICAP_VER, s.substring(sep + 1).trim()); + s = s.substring(0, sep).trim(); // cut off ICAP version mark + } else { + // ICAP version is not given, it will be treated as ver 0.9 + prop.setProperty(CONNECTION_PROP_ICAP_VER, "ICAP/1.0"); + } + + + String argsString = ""; + sep = s.indexOf("?"); + if (sep >= 0) { + // there are values attached to the query string + argsString = s.substring(sep + 1); // cut haed from tail of query + s = s.substring(0, sep); + } + prop.setProperty(CONNECTION_PROP_URL, s); // store URL + if (argsString.length() != 0) prop.setProperty(CONNECTION_PROP_ARGS, argsString); // store arguments in original form + + // finally find host string + if (s.toUpperCase().startsWith("ICAP://")) { + // a host was given. extract it and set path + s = s.substring(7); + sep = s.indexOf("/"); + if (sep < 0) { + // this is a malformed url, something like + // http://index.html + // we are lazy and guess that it means + // /index.html + // which is a localhost access to the file servlet + prop.setProperty(CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(CONNECTION_PROP_PATH, "/" + s); + } else { + // THIS IS THE "GOOD" CASE + // a perfect formulated url + prop.setProperty(CONNECTION_PROP_HOST, s.substring(0, sep)); + prop.setProperty(CONNECTION_PROP_PATH, s.substring(sep)); // yes, including beginning "/" + } + } else { + // no host in url. set path + if (s.startsWith("/")) { + // thats also fine, its a perfect localhost access + // in this case, we simulate a + // http://localhost/s + // access by setting a virtual host + prop.setProperty(CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(CONNECTION_PROP_PATH, s); + } else { + // the client 'forgot' to set a leading '/' + // this is the same case as above, with some lazyness + prop.setProperty(CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(CONNECTION_PROP_PATH, "/" + s); + } + } + return prop; + + } + + public static icapHeader readHeader(Properties prop, serverCore.Session theSession) throws IOException { + // reading all headers + icapHeader header = new icapHeader(); + int p; + String line; + while ((line = theSession.readLineAsString()) != null) { + if (line.length() == 0) break; // this seperates the header of the HTTP request from the body + // parse the header line: a property seperated with the ':' sign + if ((p = line.indexOf(":")) >= 0) { + // store a property + header.add(line.substring(0, p).trim(), line.substring(p + 1).trim()); + } + } + + return header; + } +} diff --git a/source/de/anomic/icap/icapd.java b/source/de/anomic/icap/icapd.java new file mode 100644 index 000000000..769b660b7 --- /dev/null +++ b/source/de/anomic/icap/icapd.java @@ -0,0 +1,449 @@ +//icapd.java +//----------------------- +//(C) by Michael Peter Christen; mc@anomic.de +//first published on http://www.anomic.de +//Frankfurt, Germany, 2004 +// +//This file is contributed by Martin Thelian +//last major change: $LastChangedDate$ by $LastChangedBy$ +//Revision: $LastChangedRevision$ +// +//This program is free software; you can redistribute it and/or modify +//it under the terms of the GNU General Public License as published by +//the Free Software Foundation; either version 2 of the License, or +//(at your option) any later version. +// +//This program is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. +// +//You should have received a copy of the GNU General Public License +//along with this program; if not, write to the Free Software +//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +//Using this software in any meaning (reading, learning, copying, compiling, +//running) means that you agree that the Author(s) is (are) not responsible +//for cost, loss of data or any harm that may be caused directly or indirectly +//by usage of this softare or this documentation. The usage of this software +//is on your own risk. The installation and usage (starting/running) of this +//software may allow other people or application to access your computer and +//any attached devices and is highly dependent on the configuration of the +//software which must be done by the user of the software; the author(s) is +//(are) also not responsible for proper configuration and usage of the +//software, even if provoked by documentation provided together with +//the software. +// +//Any changes to this file according to the GPL as documented in the file +//gpl.txt aside this file in the shipment you received can be done to the +//lines that follows this copyright notice here, but changes must not be +//done inside the copyright notive above. A re-distribution must contain +//the intact and unchanged copyright notice. +//Contributions and changes to the program code must be marked as such. + + +package de.anomic.icap; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.URL; +import java.util.Date; +import java.util.Properties; + +import de.anomic.http.httpChunkedInputStream; +import de.anomic.http.httpHeader; +import de.anomic.http.httpc; +import de.anomic.plasma.plasmaHTCache; +import de.anomic.plasma.plasmaParser; +import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.server.serverCore; +import de.anomic.server.serverFileUtils; +import de.anomic.server.serverHandler; +import de.anomic.server.logging.serverLog; +import de.anomic.server.serverCore.Session; + +/** + * @author theli + */ +public class icapd implements serverHandler { + + + private serverCore.Session session; // holds the session object of the calling class + + // the connection properties + private final Properties prop = new Properties(); + + // the address of the client + private InetAddress userAddress; + private String clientIP; + private int keepAliveRequestCount = 0; + + // needed for logging + private final serverLog log = new serverLog("ICAPD"); + + private static plasmaSwitchboard switchboard = null; + private static plasmaHTCache cacheManager = null; + private static String virtualHost = null; + private static boolean keepAliveSupport = true; + + + + public icapd() { + if (switchboard == null) { + switchboard = plasmaSwitchboard.getSwitchboard(); + cacheManager = switchboard.cacheManager; + virtualHost = switchboard.getConfig("fileHost","localhost"); + } + + } + + public Object clone(){ + return new icapd(); + } + + public void initSession(Session session) throws IOException { + this.session = session; + this.userAddress = session.userAddress; // client InetAddress + this.clientIP = this.userAddress.getHostAddress(); + if (this.userAddress.isAnyLocalAddress()) this.clientIP = "localhost"; + if (this.clientIP.equals("0:0:0:0:0:0:0:1")) this.clientIP = "localhost"; + if (this.clientIP.equals("127.0.0.1")) this.clientIP = "localhost"; + } + + public String greeting() { + // TODO Auto-generated method stub + return null; + } + + public String error(Throwable e) { + // TODO Auto-generated method stub + return null; + } + + public void reset() { + } + + public Boolean EMPTY(String arg) throws IOException { + // TODO Auto-generated method stub + return serverCore.TERMINATE_CONNECTION; + } + + public Boolean UNKNOWN(String requestLine) throws IOException { + // TODO Auto-generated method stub + return serverCore.TERMINATE_CONNECTION; + } + + public icapHeader getDefaultHeaders() { + icapHeader newHeaders = new icapHeader(); + + newHeaders.put(icapHeader.SERVER,"YaCy/" + switchboard.getConfig("vString","")); + newHeaders.put(icapHeader.DATE, httpc.dateString(httpc.nowDate())); + newHeaders.put(icapHeader.ISTAG, "\"" + switchboard.getConfig("vString","") + "\""); + + return newHeaders; + } + + public Boolean OPTIONS(String arg) throws IOException { + + BufferedInputStream in = new BufferedInputStream(this.session.in); + BufferedOutputStream out = new BufferedOutputStream(this.session.out); + + // parsing the http request line + parseRequestLine(icapHeader.METHOD_OPTIONS,arg); + + // reading the headers + icapHeader icapReqHeader = icapHeader.readHeader(this.prop,this.session); + + // determines if the connection should be kept alive + boolean persistent = handlePersistentConnection(icapReqHeader); + + // setting the icap response headers + icapHeader resHeader = getDefaultHeaders(); + resHeader.put(icapHeader.ALLOW,"204"); + resHeader.put(icapHeader.ENCAPSULATED,"null-body=0"); + resHeader.put(icapHeader.MAX_CONNECTIONS,"1000"); + resHeader.put(icapHeader.OPTIONS_TTL,"300"); + resHeader.put(icapHeader.SERVICE_ID, "???"); + resHeader.put(icapHeader.PREVIEW, "30"); + resHeader.put(icapHeader.TRANSFER_COMPLETE, "*"); + //resHeader.put(icapHeader.TRANSFER_PREVIEW, "*"); + if (!persistent) resHeader.put(icapHeader.CONNECTION, "close"); + + + // determining the requested service and call it or send back an error message + String reqService = this.prop.getProperty(icapHeader.CONNECTION_PROP_PATH,""); + if (reqService.equalsIgnoreCase("/resIndexing")) { + resHeader.put(icapHeader.SERVICE, "YaCy ICAP Indexing Service 1.0"); + resHeader.put(icapHeader.METHODS,icapHeader.METHOD_RESPMOD); + + String transferIgnoreList = plasmaParser.getMediaExtList(); + transferIgnoreList = transferIgnoreList.substring(1,transferIgnoreList.length()-1); + resHeader.put(icapHeader.TRANSFER_IGNORE, transferIgnoreList); + } else { + resHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0"); + } + + + StringBuffer headerStringBuffer = resHeader.toHeaderString("ICAP/1.0",200,null); + out.write(headerStringBuffer.toString().getBytes()); + out.flush(); + + return this.prop.getProperty(icapHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; + } + + public Boolean REQMOD(String arg) throws IOException { + return serverCore.TERMINATE_CONNECTION; + } + + public Boolean RESPMOD(String arg) throws IOException { + try { + InputStream in = this.session.in; + OutputStream out = this.session.out; + + // parsing the icap request line + parseRequestLine(icapHeader.METHOD_RESPMOD,arg); + + // reading the icap request header + icapHeader icapReqHeader = icapHeader.readHeader(this.prop,this.session); + + // determines if the connection should be kept alive + handlePersistentConnection(icapReqHeader); + + // determining the requested service and call it or send back an error message + String reqService = (String) this.prop.getProperty(icapHeader.CONNECTION_PROP_PATH,""); + if (reqService.equalsIgnoreCase("/resIndexing")) { + indexingService(icapReqHeader,in,out); + } else { + icapHeader icapResHeader = getDefaultHeaders(); + icapResHeader.put(icapHeader.ENCAPSULATED,icapReqHeader.get(icapHeader.ENCAPSULATED)); + icapResHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0"); + // icapResHeader.put(icapHeader.CONNECTION, "close"); + + StringBuffer headerStringBuffer = icapResHeader.toHeaderString("ICAP/1.0",404,null); + out.write(headerStringBuffer.toString().getBytes()); + out.flush(); + } + + + + } catch (Exception e) { + e.printStackTrace(); + } finally { + + } + return this.prop.getProperty(icapHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; + } + + private void blacklistService(icapHeader reqHeader, InputStream in, OutputStream out) { + try { + + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void indexingService(icapHeader reqHeader, InputStream in, OutputStream out) { + try { + + /* ========================================================================= + * Reading the various message parts into buffers + * ========================================================================= */ + ByteArrayInputStream reqHdrStream = null, resHdrStream = null, resBodyStream = null; + String[] encapsulated = ((String) reqHeader.get(icapHeader.ENCAPSULATED)).split(","); + int prevLength = 0, currLength=0; + for (int i=0; i < encapsulated.length; i++) { + // reading the request header + if (encapsulated[i].indexOf("req-hdr")>=0) { + prevLength = currLength; + currLength = Integer.parseInt(encapsulated[i+1].split("=")[1]); + + byte[] buffer = new byte[currLength-prevLength]; + in.read(buffer, 0, buffer.length); + + reqHdrStream = new ByteArrayInputStream(buffer); + + // reading the response header + } else if (encapsulated[i].indexOf("res-hdr")>=0) { + prevLength = currLength; + currLength = Integer.parseInt(encapsulated[i+1].split("=")[1]); + + byte[] buffer = new byte[currLength-prevLength]; + in.read(buffer, 0, buffer.length); + + resHdrStream = new ByteArrayInputStream(buffer); + + // reading the response body + } else if (encapsulated[i].indexOf("res-body")>=0) { + httpChunkedInputStream chunkedIn = new httpChunkedInputStream(in); + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + int l = 0,len = 0; + byte[] buffer = new byte[256]; + while ((l = chunkedIn.read(buffer)) >= 0) { + len += l; + bout.write(buffer,0,l); + } + resBodyStream = new ByteArrayInputStream(bout.toByteArray()); + } + } + + /* ========================================================================= + * sending back the icap status + * ========================================================================= */ + icapHeader icapResHeader = getDefaultHeaders(); + if (reqHeader.allow(204)) { + icapResHeader.put(icapHeader.ENCAPSULATED,reqHeader.get(icapHeader.ENCAPSULATED)); + icapResHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0"); + // resHeader.put(icapHeader.CONNECTION, "close"); + + StringBuffer headerStringBuffer = icapResHeader.toHeaderString("ICAP/1.0",204,null); + out.write(headerStringBuffer.toString().getBytes()); + out.flush(); + } else { + icapResHeader.put(icapHeader.ENCAPSULATED,reqHeader.get(icapHeader.ENCAPSULATED)); + icapResHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0"); + // icapResHeader.put(icapHeader.CONNECTION, "close"); + + StringBuffer headerStringBuffer = icapResHeader.toHeaderString("ICAP/1.0",503,null); + out.write(headerStringBuffer.toString().getBytes()); + out.flush(); + } + + /* ========================================================================= + * Parsing request data + * ========================================================================= */ + // reading the requestline + BufferedReader reader = new BufferedReader(new InputStreamReader(reqHdrStream)); + String httpRequestLine = reader.readLine(); + + // parsing the requestline + Properties httpReqProps = new Properties(); + httpHeader.parseRequestLine(httpRequestLine,httpReqProps,virtualHost); + + if (!httpReqProps.getProperty(httpHeader.CONNECTION_PROP_METHOD).equals(httpHeader.METHOD_GET)) { + this.log.logInfo("Wrong http request method for indexing:" + + "\nRequest Method: " + httpReqProps.getProperty(httpHeader.CONNECTION_PROP_METHOD) + + "\nRequest Line: " + httpRequestLine); + reader.close(); + reqHdrStream.close(); + return; + } + + // reading all request headers + httpHeader httpReqHeader = httpHeader.readHttpHeader(reader); + reader.close(); + reqHdrStream.close(); + + // handle transparent proxy support: this function call is needed to set the host property properly + httpHeader.handleTransparentProxySupport(httpReqHeader,httpReqProps,virtualHost,true); + + // getting the request URL + URL httpRequestURL = httpHeader.getRequestURL(httpReqProps); + + /* ========================================================================= + * Parsing response data + * ========================================================================= */ + // getting the response status + reader = new BufferedReader(new InputStreamReader(resHdrStream)); + String httpRespStatusLine = reader.readLine(); + + Object[] httpRespStatus = httpHeader.parseResponseLine(httpRespStatusLine); + + if (!(httpRespStatus[1].equals(new Integer(200)) || httpRespStatus[1].equals(new Integer(203)))) { + this.log.logInfo("Wrong status code for indexing:" + + "\nStatus Code: " + httpRespStatus[1] + + "\nRequest Line: " + httpRequestLine + + "\nResponse Line: " + httpRespStatusLine); + reader.close(); + resHdrStream.close(); + return; + } + + // reading all response headers + httpHeader httpResHeader = httpHeader.readHttpHeader(reader); + reader.close(); + resHdrStream.close(); + + if ((!(plasmaParser.supportedMimeTypesContains(httpResHeader.mime()))) && + (!(plasmaParser.supportedFileExt(httpRequestURL)))) { + this.log.logInfo("Wrong mimeType or fileExtension for indexing:" + + "\nMimeType: " + httpResHeader.mime() + + "\nRequest Line:" + httpRequestLine); + return ; + } + + + /* ========================================================================= + * Prepare data for indexing + * ========================================================================= */ + + // generating a htcache entry object + plasmaHTCache.Entry cacheEntry = cacheManager.newEntry( + new Date(), + 0, + httpRequestURL, + "", + httpReqHeader, + httpRespStatusLine, + httpResHeader, + null, + switchboard.defaultProxyProfile + ); + + // getting the filename/path to store the response body + File cacheFile = cacheManager.getCachePath(httpRequestURL); + + // if the file already exits we delete it + if (cacheFile.isFile()) { + cacheManager.deleteFile(httpRequestURL); + } + // we write the new cache entry to file system directly + cacheFile.getParentFile().mkdirs(); + + // copy the response body into the file + serverFileUtils.copy(resBodyStream,cacheFile); + resBodyStream.close(); resBodyStream = null; + + // indexing the response + cacheManager.push(cacheEntry); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private final void parseRequestLine(String cmd, String s) { + // parsing the requestlin + icapHeader.parseRequestLine(cmd,s, this.prop,virtualHost); + + // adding the client ip prop + this.prop.setProperty(icapHeader.CONNECTION_PROP_CLIENTIP, this.clientIP); + + // counting the amount of received requests within this permanent conneciton + this.prop.setProperty(icapHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer.toString(++this.keepAliveRequestCount)); + } + + private boolean handlePersistentConnection(icapHeader header) { + + if (!keepAliveSupport) { + this.prop.put(icapHeader.CONNECTION_PROP_PERSISTENT,"close"); + return false; + } + + boolean persistent = true; + if (((String)header.get(icapHeader.CONNECTION, "keep-alive")).toLowerCase().equals("close")) { + persistent = false; + } + + this.prop.put(icapHeader.CONNECTION_PROP_PERSISTENT,persistent?"keep-alive":"close"); + return persistent; + } + +} diff --git a/source/de/anomic/plasma/parser/pdf/build.xml b/source/de/anomic/plasma/parser/pdf/build.xml index 36b855786..cf752cb3c 100644 --- a/source/de/anomic/plasma/parser/pdf/build.xml +++ b/source/de/anomic/plasma/parser/pdf/build.xml @@ -16,10 +16,7 @@ - - - - + @@ -28,7 +25,7 @@ - + diff --git a/source/de/anomic/plasma/parser/pdf/pdfParser.java b/source/de/anomic/plasma/parser/pdf/pdfParser.java index 445be777c..70db05b45 100644 --- a/source/de/anomic/plasma/parser/pdf/pdfParser.java +++ b/source/de/anomic/plasma/parser/pdf/pdfParser.java @@ -43,7 +43,6 @@ package de.anomic.plasma.parser.pdf; -import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.io.OutputStreamWriter; import java.net.URL; @@ -74,8 +73,7 @@ public class pdfParser extends AbstractParser implements Parser { * @see Parser#getLibxDependences() */ private static final String[] LIBX_DEPENDENCIES = new String[] { - "PDFBox-0.7.1.jar", - "log4j-1.2.9.jar" + "PDFBox-0.7.2.jar" }; public pdfParser() { diff --git a/source/de/anomic/plasma/plasmaCondenser.java b/source/de/anomic/plasma/plasmaCondenser.java index 338ef1ef1..3cb24fcaa 100644 --- a/source/de/anomic/plasma/plasmaCondenser.java +++ b/source/de/anomic/plasma/plasmaCondenser.java @@ -66,7 +66,7 @@ import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.htmlFilter.htmlFilterOutputStream; import de.anomic.kelondro.kelondroMSetTools; -public class plasmaCondenser { +public final class plasmaCondenser { private final static int numlength = 5; diff --git a/source/de/anomic/plasma/plasmaParser.java b/source/de/anomic/plasma/plasmaParser.java index 407aa9185..55314f7ad 100644 --- a/source/de/anomic/plasma/plasmaParser.java +++ b/source/de/anomic/plasma/plasmaParser.java @@ -237,6 +237,12 @@ public final class plasmaParser { } } + public static String getMediaExtList() { + synchronized (mediaExtSet) { + return mediaExtSet.toString(); + } + } + public static void initSupportedRealtimeFileExt(List supportedRealtimeFileExtList) { synchronized (supportedRealtimeFileExt) { supportedRealtimeFileExt.clear(); @@ -446,12 +452,16 @@ public final class plasmaParser { // testing if all needed libx libraries are available String[] neededLibx = ((Parser)theParser).getLibxDependences(); + StringBuffer neededLibxBuf = new StringBuffer(); if (neededLibx != null) { for (int libxId=0; libxId < neededLibx.length; libxId++) { if (javaClassPath.indexOf(neededLibx[libxId]) == -1) { throw new ParserException("Missing dependency detected: '" + neededLibx[libxId] + "'."); } + neededLibxBuf.append(neededLibx[libxId]) + .append(","); } + if (neededLibxBuf.length()>0) neededLibxBuf.deleteCharAt(neededLibxBuf.length()-1); } // loading the list of mime-types that are supported by this parser class @@ -460,7 +470,8 @@ public final class plasmaParser { while (mimeTypeIterator.hasNext()) { String mimeType = (String) mimeTypeIterator.next(); availableParserList.put(mimeType,fullClassName); - serverLog.logInfo("PARSER", "Found functional parser for mimeType '" + mimeType + "'."); + serverLog.logInfo("PARSER", "Found functional parser for mimeType '" + mimeType + "'." + + ((neededLibxBuf.length()>0)?" Dependencies: " + neededLibxBuf.toString():"")); } } catch (Exception e) { /* we can ignore this for the moment */ diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 18c96eb81..88f85ca66 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -304,6 +304,18 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser sbQueue = new plasmaSwitchboardQueue(this.cacheManager, urlPool.loadedURL, new File(plasmaPath, "switchboardQueue1.stack"), 10, profiles); indexingTasksInProcess = new HashMap(); + // going through the sbQueue Entries and registering all content files as in use + int count = 0; + ArrayList sbQueueEntries = this.sbQueue.list(); + for (int i=0; i 0)) writeLine((String) result); - } + Object result; +// // send greeting +// Object result = commandObj.greeting(); +// if (result != null) { +// if ((result instanceof String) && (((String) result).length() > 0)) writeLine((String) result); +// } // start dialog byte[] requestBytes = null; boolean terminate = false; - int pos; - String cmd; - String tmp; + String reqCmd; + String reqProtocol = "HTTP"; Object[] stringParameter = new String[1]; - while ((this.in != null) && ((requestBytes = readLine()) != null)) { - this.commandCounter++; + while ((this.in != null) && ((requestBytes = readLine()) != null)) { this.setName("Session_" + this.userAddress.getHostAddress() + ":" + this.controlSocket.getPort() + "#" + commandCounter); this.request = new String(requestBytes); @@ -904,24 +903,50 @@ public final class serverCore extends serverAbstractThread implements serverThre // of the commandObject if (this.request.trim().length() == 0) this.request = "EMPTY"; - pos = this.request.indexOf(' '); + // getting the rest of the request parameters + int pos = this.request.indexOf(' '); if (pos < 0) { - cmd = this.request.trim().toUpperCase(); + reqCmd = this.request.trim().toUpperCase(); stringParameter[0] = ""; } else { - cmd = this.request.substring(0, pos).trim().toUpperCase(); + reqCmd = this.request.substring(0, pos).trim().toUpperCase(); stringParameter[0] = this.request.substring(pos).trim(); } + // now we need to initialize the session + if (this.commandCounter == 0) { + // first we need to determine the proper protocol handler + if (this.request.indexOf("ICAP") >= 0) reqProtocol = "ICAP"; + else reqProtocol = "HTTP"; + + // next we need to get the proper protocol handler + if (reqProtocol.equals("ICAP")) { + this.commandObj = new icapd(); + } else { +// if ((this.commandObj != null) && +// (this.commandObj.getClass().getName().equals(serverCore.this.handlerPrototype.getClass().getName()))) { +// this.commandObj.reset(); +// } else { +// this.commandObj = (serverHandler) serverCore.this.handlerPrototype.clone(); +// } + + this.commandObj = (serverHandler) serverCore.this.handlerPrototype.clone(); + } + + // initializing the session + this.commandObj.initSession(this); + } + this.commandCounter++; + // setting the socket timeout for reading of the request content this.controlSocket.setSoTimeout(this.socketTimeout); // exec command and return value - Object commandMethod = this.commandObjMethodCache.get(cmd); + Object commandMethod = this.commandObjMethodCache.get(reqProtocol + "_" + reqCmd); if (commandMethod == null) { try { - commandMethod = this.commandObj.getClass().getMethod(cmd, stringType); - this.commandObjMethodCache.put(cmd,commandMethod); + commandMethod = this.commandObj.getClass().getMethod(reqCmd, stringType); + this.commandObjMethodCache.put(reqProtocol + "_" + reqCmd,commandMethod); } catch (NoSuchMethodException noMethod) { commandMethod = this.commandObj.getClass().getMethod("UNKNOWN", stringType); stringParameter[0] = this.request.trim(); @@ -947,7 +972,7 @@ public final class serverCore extends serverAbstractThread implements serverThre } writeLine((String) result); } else if (result instanceof InputStream) { - tmp = send(out, (InputStream) result); + String tmp = send(out, (InputStream) result); if ((tmp.length() > 4) && (tmp.toUpperCase().startsWith("PASS"))) { log(true, "PASS ********"); } else { @@ -1014,6 +1039,7 @@ public final class serverCore extends serverAbstractThread implements serverThre if ((b = pbis.read()) != lf) if (b >= 0) pbis.unread(b); // we push back the byte } + if ((readLineBuffer.length()==0)&&(b == -1)) return null; return readLineBuffer.toByteArray(); } catch (ClosedByInterruptException e) { if (logerr) serverLog.logSevere("SERVER", "receive interrupted - timeout"); diff --git a/source/de/anomic/yacy/seedUpload/yacySeedUploadFtp.java b/source/de/anomic/yacy/seedUpload/yacySeedUploadFtp.java index d70fb9264..aad0e7438 100644 --- a/source/de/anomic/yacy/seedUpload/yacySeedUploadFtp.java +++ b/source/de/anomic/yacy/seedUpload/yacySeedUploadFtp.java @@ -64,6 +64,8 @@ public class yacySeedUploadFtp implements yacySeedUploader { if (sb == null) throw new NullPointerException("Reference to serverSwitch must not be null."); if (seedDB == null) throw new NullPointerException("Reference to seedDB must not be null."); if ((seedFile == null)||(!seedFile.exists())) throw new Exception("Seed file does not exist."); + if (!seedFile.isFile()) throw new Exception("Seed file is not a file."); + if (!seedFile.canRead()) throw new Exception("Seed file is not readable."); String seedFTPServer = sb.getConfig(CONFIG_FTP_SERVER,null); String seedFTPAccount = sb.getConfig(CONFIG_FTP_ACCOUNT,null); diff --git a/source/de/anomic/yacy/seedUpload/yacySeedUploadScp.xml b/source/de/anomic/yacy/seedUpload/yacySeedUploadScp.xml index b1eb6de86..449107c1e 100644 --- a/source/de/anomic/yacy/seedUpload/yacySeedUploadScp.xml +++ b/source/de/anomic/yacy/seedUpload/yacySeedUploadScp.xml @@ -17,7 +17,7 @@ - + @@ -26,7 +26,7 @@ - +