*) Replacing PDFBox 0.7.1 lib with newer version 0.7.2
*) Refactoring of classes httpd/httpc/httpHeaders to
make many methods for httpHeader/Requestline parsing
reusable for new icap implementation
*) adding chunked input stream support
- needed by new icap implementation
- needed by future httpc HTTP/1.1 support
*) httpd.java
- moving all connection property contants to class httpHeader
- moving readHeader function to class httpHeader
- moving parseQuery function to class httpHeader
- moving handleTransparentProxy function to class httpHeader
*) httpHeader.java
- adding new fuction to parse the http response line
- adding new function to converte http headers to a string that
can be send to the client
- adding a function that generates a proper url using all parsed
connection properties
*) ICAP Support
- yacy now supports handling of icap response modification requests
- this feature can be used by other icap enabled proxies to contact
yacy as icap server, and to handover the downloaded content to yacy.logging
for indexing
- functionality was successfully tested with squid 2.5Stable 10 + icap patch
- further icap services e.g. URL filtering based on yacy's blacklists are possible
*) plasmaSwitchboard.java
- htcache entries that are still needed for indexing are now properly registered
as in use after system restart
- extended logging: log message now shows parsing and indexing time for each sb. entry
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@757 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli
20 years ago
parent
6d1de8abfd
commit
b990dc1ad1
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,255 @@
|
||||
//httpChunkedInputStream.java
|
||||
//-----------------------
|
||||
//(C) by Michael Peter Christen; mc@anomic.de
|
||||
//first published on http://www.anomic.de
|
||||
//Frankfurt, Germany, 2004
|
||||
//
|
||||
// This file is contributed by Martin Thelian
|
||||
// last major change: $LastChangedDate$ by $LastChangedBy$
|
||||
// Revision: $LastChangedRevision$
|
||||
//
|
||||
//This program is free software; you can redistribute it and/or modify
|
||||
//it under the terms of the GNU General Public License as published by
|
||||
//the Free Software Foundation; either version 2 of the License, or
|
||||
//(at your option) any later version.
|
||||
//
|
||||
//This program is distributed in the hope that it will be useful,
|
||||
//but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
//GNU General Public License for more details.
|
||||
//
|
||||
//You should have received a copy of the GNU General Public License
|
||||
//along with this program; if not, write to the Free Software
|
||||
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
//
|
||||
//Using this software in any meaning (reading, learning, copying, compiling,
|
||||
//running) means that you agree that the Author(s) is (are) not responsible
|
||||
//for cost, loss of data or any harm that may be caused directly or indirectly
|
||||
//by usage of this softare or this documentation. The usage of this software
|
||||
//is on your own risk. The installation and usage (starting/running) of this
|
||||
//software may allow other people or application to access your computer and
|
||||
//any attached devices and is highly dependent on the configuration of the
|
||||
//software which must be done by the user of the software; the author(s) is
|
||||
//(are) also not responsible for proper configuration and usage of the
|
||||
//software, even if provoked by documentation provided together with
|
||||
//the software.
|
||||
//
|
||||
//Any changes to this file according to the GPL as documented in the file
|
||||
//gpl.txt aside this file in the shipment you received can be done to the
|
||||
//lines that follows this copyright notice here, but changes must not be
|
||||
//done inside the copyright notive above. A re-distribution must contain
|
||||
//the intact and unchanged copyright notice.
|
||||
//Contributions and changes to the program code must be marked as such.
|
||||
|
||||
package de.anomic.http;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
|
||||
/**
|
||||
* Some parts of this class code was copied from <a href="http://www.devdaily.com/java/jwarehouse/commons-httpclient-2.0/src/java/org/apache/commons/httpclient/ChunkedInputStream.shtml">Apache httpclient Project.</a>
|
||||
* @author theli
|
||||
*/
|
||||
public class httpChunkedInputStream extends InputStream {
|
||||
|
||||
private static final int READ_CHUNK_STATE_NORMAL = 0;
|
||||
private static final int READ_CHUNK_STATE_CR_READ = 1;
|
||||
private static final int READ_CHUNK_STATE_IN_EXT_CHUNK = 2;
|
||||
private static final int READ_CHUNK_STATE_FINISHED = -1;
|
||||
|
||||
private static final char CR = '\r';
|
||||
private static final char LF = '\n';
|
||||
|
||||
private InputStream inputStream;
|
||||
private int currPos;
|
||||
private int currChunkSize;
|
||||
private httpHeader httpTrailer;
|
||||
|
||||
private boolean beginningOfStream = true;
|
||||
private boolean isEOF = false;
|
||||
private boolean isClosed = false;
|
||||
|
||||
|
||||
public httpChunkedInputStream(InputStream in) throws IOException {
|
||||
|
||||
if (in == null)throw new IllegalArgumentException("InputStream must not be null");
|
||||
|
||||
this.inputStream = in;
|
||||
this.currPos = 0;
|
||||
}
|
||||
|
||||
public int read() throws IOException {
|
||||
|
||||
if (this.isClosed) throw new IOException("Inputstream already closed.");
|
||||
if (this.isEOF) return -1;
|
||||
|
||||
if (this.currPos >= this.currChunkSize) {
|
||||
readNextChunk();
|
||||
if (this.isEOF) return -1;
|
||||
}
|
||||
this.currPos++;
|
||||
return this.inputStream.read();
|
||||
}
|
||||
|
||||
|
||||
public int read (byte[] b, int off, int len) throws IOException {
|
||||
|
||||
if (b == null) throw new IllegalArgumentException("bytearry parameter must not be null");
|
||||
|
||||
if (this.isClosed) throw new IOException("Inputstream already closed.");
|
||||
if (this.isEOF) return -1;
|
||||
|
||||
if (this.currPos >= this.currChunkSize) {
|
||||
readNextChunk();
|
||||
if (this.isEOF) return -1;
|
||||
}
|
||||
len = Math.min(len, this.currChunkSize - this.currPos);
|
||||
int count = this.inputStream.read(b, off, len);
|
||||
this.currPos += count;
|
||||
return count;
|
||||
}
|
||||
|
||||
public int read (byte[] b) throws IOException {
|
||||
if (b == null) throw new IllegalArgumentException("bytearry parameter must not be null");
|
||||
return read(b, 0, b.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the CRLF terminator.
|
||||
* @throws IOException If an IO error occurs.
|
||||
*/
|
||||
private void readCRLF() throws IOException {
|
||||
int cr = this.inputStream.read();
|
||||
int lf = this.inputStream.read();
|
||||
if ((cr != CR) || (lf != LF)) {
|
||||
throw new IOException("Malformed chunk. CRLF expected but '" + cr + lf + "' found");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void readNextChunk() throws IOException {
|
||||
if (!this.beginningOfStream) readCRLF();
|
||||
|
||||
this.currChunkSize = readChunkFromStream(this.inputStream);
|
||||
this.beginningOfStream = false;
|
||||
this.currPos = 0;
|
||||
if (this.currChunkSize == 0) {
|
||||
this.isEOF = true;
|
||||
readTrailer();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void readTrailer() throws IOException {
|
||||
BufferedReader reader = null;
|
||||
ByteArrayOutputStream bout = null;
|
||||
try {
|
||||
bout = new ByteArrayOutputStream();
|
||||
do {
|
||||
int ch;
|
||||
while ((ch = this.inputStream.read()) >= 0) {
|
||||
bout.write(ch);
|
||||
if (ch == LF) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (bout.size() <= 2) break;
|
||||
} while(true);
|
||||
|
||||
ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
|
||||
reader = new BufferedReader(new InputStreamReader(bin));
|
||||
this.httpTrailer = httpHeader.readHttpHeader(reader);
|
||||
} finally {
|
||||
if (reader != null) try {reader.close();}catch(Exception e){}
|
||||
if (bout != null) try {bout.close();}catch(Exception e){}
|
||||
}
|
||||
}
|
||||
|
||||
public httpHeader getTrailer() {
|
||||
return this.httpTrailer;
|
||||
}
|
||||
|
||||
private static int readChunkFromStream(final InputStream in)
|
||||
throws IOException {
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
int state = 0;
|
||||
while (state != READ_CHUNK_STATE_FINISHED) {
|
||||
int b = in.read();
|
||||
if (b == -1) throw new IOException("Malformed chunk. Unexpected end");
|
||||
|
||||
switch (state) {
|
||||
case 0:
|
||||
switch (b) {
|
||||
case CR:
|
||||
state = READ_CHUNK_STATE_CR_READ;
|
||||
break;
|
||||
case '\"':
|
||||
case ';':
|
||||
case ' ':
|
||||
state = READ_CHUNK_STATE_IN_EXT_CHUNK;
|
||||
break;
|
||||
default:
|
||||
baos.write(b);
|
||||
}
|
||||
break;
|
||||
|
||||
case 1:
|
||||
if (b == LF) {
|
||||
state = READ_CHUNK_STATE_FINISHED;
|
||||
} else {
|
||||
// this was not CRLF
|
||||
throw new IOException("Malformed chunk. Unexpected enf of chunk. MIssing CR character.");
|
||||
}
|
||||
break;
|
||||
|
||||
case 2:
|
||||
switch (b) {
|
||||
case CR:
|
||||
state = READ_CHUNK_STATE_CR_READ;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default: throw new RuntimeException("Malformed chunk. Illegal state.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int result;
|
||||
try {
|
||||
result = Integer.parseInt(baos.toString().trim(), 16);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IOException ("Malformed chunk. Bad chunk size: " + baos.toString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
if (!this.isClosed) {
|
||||
try {
|
||||
if (!this.isEOF) {
|
||||
exhaustInputStream(this);
|
||||
}
|
||||
} finally {
|
||||
this.isEOF = true;
|
||||
this.isClosed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void exhaustInputStream(InputStream inStream) throws IOException {
|
||||
byte buffer[] = new byte[1024];
|
||||
while (inStream.read(buffer) >= 0) {
|
||||
;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,303 @@
|
||||
//icapHeader.java
|
||||
//-----------------------
|
||||
//(C) by Michael Peter Christen; mc@anomic.de
|
||||
//first published on http://www.anomic.de
|
||||
//Frankfurt, Germany, 2004
|
||||
//
|
||||
//This file is contributed by Martin Thelian
|
||||
//last major change: $LastChangedDate$ by $LastChangedBy$
|
||||
//Revision: $LastChangedRevision$
|
||||
//
|
||||
//This program is free software; you can redistribute it and/or modify
|
||||
//it under the terms of the GNU General Public License as published by
|
||||
//the Free Software Foundation; either version 2 of the License, or
|
||||
//(at your option) any later version.
|
||||
//
|
||||
//This program is distributed in the hope that it will be useful,
|
||||
//but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
//GNU General Public License for more details.
|
||||
//
|
||||
//You should have received a copy of the GNU General Public License
|
||||
//along with this program; if not, write to the Free Software
|
||||
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
//
|
||||
//Using this software in any meaning (reading, learning, copying, compiling,
|
||||
//running) means that you agree that the Author(s) is (are) not responsible
|
||||
//for cost, loss of data or any harm that may be caused directly or indirectly
|
||||
//by usage of this softare or this documentation. The usage of this software
|
||||
//is on your own risk. The installation and usage (starting/running) of this
|
||||
//software may allow other people or application to access your computer and
|
||||
//any attached devices and is highly dependent on the configuration of the
|
||||
//software which must be done by the user of the software; the author(s) is
|
||||
//(are) also not responsible for proper configuration and usage of the
|
||||
//software, even if provoked by documentation provided together with
|
||||
//the software.
|
||||
//
|
||||
//Any changes to this file according to the GPL as documented in the file
|
||||
//gpl.txt aside this file in the shipment you received can be done to the
|
||||
//lines that follows this copyright notice here, but changes must not be
|
||||
//done inside the copyright notive above. A re-distribution must contain
|
||||
//the intact and unchanged copyright notice.
|
||||
//Contributions and changes to the program code must be marked as such.
|
||||
|
||||
|
||||
package de.anomic.icap;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.text.Collator;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.server.serverCore;
|
||||
|
||||
public class icapHeader extends TreeMap implements Map {
|
||||
|
||||
/* =============================================================
|
||||
* Constants defining icap methods
|
||||
* ============================================================= */
|
||||
public static final String METHOD_REQMOD = "REQMOD";
|
||||
public static final String METHOD_RESPMOD = "RESPMOD";
|
||||
public static final String METHOD_OPTIONS = "OPTIONS";
|
||||
|
||||
/* =============================================================
|
||||
* Constants defining http header names
|
||||
* ============================================================= */
|
||||
public static final String HOST = "Host";
|
||||
public static final String USER_AGENT = "User-Agent";
|
||||
public static final String CONNECTION = "Connection";
|
||||
public static final String DATE = "Date";
|
||||
public static final String SERVER = "Server";
|
||||
public static final String ISTAG = "ISTAG";
|
||||
public static final String METHODS = "Methods";
|
||||
public static final String ALLOW = "Allow";
|
||||
public static final String ENCAPSULATED = "Encapsulated";
|
||||
public static final String MAX_CONNECTIONS = "Max-Connections";
|
||||
public static final String OPTIONS_TTL = "Options-TTL";
|
||||
public static final String SERVICE = "Service";
|
||||
public static final String SERVICE_ID = "Service-ID";
|
||||
public static final String PREVIEW = "Preview";
|
||||
public static final String TRANSFER_PREVIEW = "Transfer-Preview";
|
||||
public static final String TRANSFER_IGNORE = "Transfer-Ignore";
|
||||
public static final String TRANSFER_COMPLETE = "Transfer-Complete";
|
||||
|
||||
public static final String X_YACY_KEEP_ALIVE_REQUEST_COUNT = "X-Keep-Alive-Request-Count";
|
||||
|
||||
/* =============================================================
|
||||
* defining default icap status messages
|
||||
* ============================================================= */
|
||||
public static final HashMap icap1_0 = new HashMap();
|
||||
static {
|
||||
// (1yz) Informational codes
|
||||
icap1_0.put("100","Continue after ICAP preview");
|
||||
|
||||
// (2yz) Success codes:
|
||||
icap1_0.put("200","OK");
|
||||
icap1_0.put("204","No modifications needed");
|
||||
|
||||
// (4yz) Client error codes:
|
||||
icap1_0.put("400","Bad request");
|
||||
icap1_0.put("404","ICAP Service not found");
|
||||
icap1_0.put("405","Method not allowed for service");
|
||||
icap1_0.put("408","Request timeout");
|
||||
|
||||
// (5yz) Server error codes:
|
||||
icap1_0.put("500","Server error");
|
||||
icap1_0.put("501","Method not implemented");
|
||||
icap1_0.put("502","Bad Gateway");
|
||||
icap1_0.put("503","Service overloaded");
|
||||
icap1_0.put("505","ICAP version not supported by server");
|
||||
}
|
||||
|
||||
/* PROPERTIES: General properties */
|
||||
public static final String CONNECTION_PROP_ICAP_VER = "ICAP";
|
||||
public static final String CONNECTION_PROP_HOST = "HOST";
|
||||
public static final String CONNECTION_PROP_PATH = "PATH";
|
||||
public static final String CONNECTION_PROP_EXT = "EXT";
|
||||
public static final String CONNECTION_PROP_METHOD = "METHOD";
|
||||
public static final String CONNECTION_PROP_REQUESTLINE = "REQUESTLINE";
|
||||
public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP";
|
||||
public static final String CONNECTION_PROP_URL = "URL";
|
||||
public static final String CONNECTION_PROP_ARGS = "ARGS";
|
||||
public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT";
|
||||
public static final String CONNECTION_PROP_KEEP_ALIVE_COUNT = "KEEP-ALIVE_COUNT";
|
||||
|
||||
private static final Collator insensitiveCollator = Collator.getInstance(Locale.US);
|
||||
static {
|
||||
insensitiveCollator.setStrength(Collator.SECONDARY);
|
||||
insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION);
|
||||
}
|
||||
|
||||
public icapHeader() {
|
||||
super(insensitiveCollator);
|
||||
}
|
||||
|
||||
public boolean allow(int statusCode) {
|
||||
if (!super.containsKey("Allow")) return false;
|
||||
|
||||
String allow = (String)get("Allow");
|
||||
return (allow.indexOf(Integer.toString(statusCode))!=-1);
|
||||
}
|
||||
|
||||
// to make the occurrence of multiple keys possible, we add them using a counter
|
||||
public Object add(Object key, Object value) {
|
||||
int c = keyCount((String) key);
|
||||
if (c == 0) return put(key, value); else return put("*" + key + "-" + c, value);
|
||||
}
|
||||
|
||||
public int keyCount(String key) {
|
||||
if (!(containsKey(key))) return 0;
|
||||
int c = 1;
|
||||
while (containsKey("*" + key + "-" + c)) c++;
|
||||
return c;
|
||||
}
|
||||
|
||||
// a convenience method to access the map with fail-over defaults
|
||||
public Object get(Object key, Object dflt) {
|
||||
Object result = get(key);
|
||||
if (result == null) return dflt; else return result;
|
||||
}
|
||||
|
||||
// return multiple results
|
||||
public Object getSingle(Object key, int count) {
|
||||
if (count == 0) return get(key, null);
|
||||
return get("*" + key + "-" + count, null);
|
||||
}
|
||||
|
||||
public StringBuffer toHeaderString(String icapVersion, int icapStatusCode, String icapStatusText) {
|
||||
|
||||
if ((icapStatusText == null)||(icapStatusText.length()==0)) {
|
||||
if (icapVersion.equals("ICAP/1.0") && icapHeader.icap1_0.containsKey(Integer.toString(icapStatusCode)))
|
||||
icapStatusText = (String) icapHeader.icap1_0.get(Integer.toString(icapStatusCode));
|
||||
}
|
||||
|
||||
StringBuffer theHeader = new StringBuffer();
|
||||
|
||||
// write status line
|
||||
theHeader.append(icapVersion).append(" ")
|
||||
.append(Integer.toString(icapStatusCode)).append(" ")
|
||||
.append(icapStatusText).append("\r\n");
|
||||
|
||||
// write header
|
||||
Iterator i = keySet().iterator();
|
||||
String key, value;
|
||||
char tag;
|
||||
int count;
|
||||
while (i.hasNext()) {
|
||||
key = (String) i.next();
|
||||
tag = key.charAt(0);
|
||||
if ((tag != '*') && (tag != '#')) { // '#' in key is reserved for proxy attributes as artificial header values
|
||||
count = keyCount(key);
|
||||
for (int j = 0; j < count; j++) {
|
||||
theHeader.append(key).append(": ").append((String) getSingle(key, j)).append("\r\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
// end header
|
||||
theHeader.append("\r\n");
|
||||
|
||||
|
||||
return theHeader;
|
||||
}
|
||||
|
||||
public static Properties parseRequestLine(String cmd, String s, Properties prop, String virtualHost) {
|
||||
|
||||
// reset property from previous run
|
||||
prop.clear();
|
||||
|
||||
// storing informations about the request
|
||||
prop.setProperty(CONNECTION_PROP_METHOD, cmd);
|
||||
prop.setProperty(CONNECTION_PROP_REQUESTLINE,cmd + " " + s);
|
||||
|
||||
|
||||
// this parses a whole URL
|
||||
if (s.length() == 0) {
|
||||
prop.setProperty(CONNECTION_PROP_HOST, virtualHost);
|
||||
prop.setProperty(CONNECTION_PROP_PATH, "/");
|
||||
prop.setProperty(CONNECTION_PROP_ICAP_VER, "ICAP/1.0");
|
||||
prop.setProperty(CONNECTION_PROP_EXT, "");
|
||||
return prop;
|
||||
}
|
||||
|
||||
// store the version propery "ICAP" and cut the query at both ends
|
||||
int sep = s.indexOf(" ");
|
||||
if (sep >= 0) {
|
||||
// ICAP version is given
|
||||
prop.setProperty(CONNECTION_PROP_ICAP_VER, s.substring(sep + 1).trim());
|
||||
s = s.substring(0, sep).trim(); // cut off ICAP version mark
|
||||
} else {
|
||||
// ICAP version is not given, it will be treated as ver 0.9
|
||||
prop.setProperty(CONNECTION_PROP_ICAP_VER, "ICAP/1.0");
|
||||
}
|
||||
|
||||
|
||||
String argsString = "";
|
||||
sep = s.indexOf("?");
|
||||
if (sep >= 0) {
|
||||
// there are values attached to the query string
|
||||
argsString = s.substring(sep + 1); // cut haed from tail of query
|
||||
s = s.substring(0, sep);
|
||||
}
|
||||
prop.setProperty(CONNECTION_PROP_URL, s); // store URL
|
||||
if (argsString.length() != 0) prop.setProperty(CONNECTION_PROP_ARGS, argsString); // store arguments in original form
|
||||
|
||||
// finally find host string
|
||||
if (s.toUpperCase().startsWith("ICAP://")) {
|
||||
// a host was given. extract it and set path
|
||||
s = s.substring(7);
|
||||
sep = s.indexOf("/");
|
||||
if (sep < 0) {
|
||||
// this is a malformed url, something like
|
||||
// http://index.html
|
||||
// we are lazy and guess that it means
|
||||
// /index.html
|
||||
// which is a localhost access to the file servlet
|
||||
prop.setProperty(CONNECTION_PROP_HOST, virtualHost);
|
||||
prop.setProperty(CONNECTION_PROP_PATH, "/" + s);
|
||||
} else {
|
||||
// THIS IS THE "GOOD" CASE
|
||||
// a perfect formulated url
|
||||
prop.setProperty(CONNECTION_PROP_HOST, s.substring(0, sep));
|
||||
prop.setProperty(CONNECTION_PROP_PATH, s.substring(sep)); // yes, including beginning "/"
|
||||
}
|
||||
} else {
|
||||
// no host in url. set path
|
||||
if (s.startsWith("/")) {
|
||||
// thats also fine, its a perfect localhost access
|
||||
// in this case, we simulate a
|
||||
// http://localhost/s
|
||||
// access by setting a virtual host
|
||||
prop.setProperty(CONNECTION_PROP_HOST, virtualHost);
|
||||
prop.setProperty(CONNECTION_PROP_PATH, s);
|
||||
} else {
|
||||
// the client 'forgot' to set a leading '/'
|
||||
// this is the same case as above, with some lazyness
|
||||
prop.setProperty(CONNECTION_PROP_HOST, virtualHost);
|
||||
prop.setProperty(CONNECTION_PROP_PATH, "/" + s);
|
||||
}
|
||||
}
|
||||
return prop;
|
||||
|
||||
}
|
||||
|
||||
public static icapHeader readHeader(Properties prop, serverCore.Session theSession) throws IOException {
|
||||
// reading all headers
|
||||
icapHeader header = new icapHeader();
|
||||
int p;
|
||||
String line;
|
||||
while ((line = theSession.readLineAsString()) != null) {
|
||||
if (line.length() == 0) break; // this seperates the header of the HTTP request from the body
|
||||
// parse the header line: a property seperated with the ':' sign
|
||||
if ((p = line.indexOf(":")) >= 0) {
|
||||
// store a property
|
||||
header.add(line.substring(0, p).trim(), line.substring(p + 1).trim());
|
||||
}
|
||||
}
|
||||
|
||||
return header;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,449 @@
|
||||
//icapd.java
|
||||
//-----------------------
|
||||
//(C) by Michael Peter Christen; mc@anomic.de
|
||||
//first published on http://www.anomic.de
|
||||
//Frankfurt, Germany, 2004
|
||||
//
|
||||
//This file is contributed by Martin Thelian
|
||||
//last major change: $LastChangedDate$ by $LastChangedBy$
|
||||
//Revision: $LastChangedRevision$
|
||||
//
|
||||
//This program is free software; you can redistribute it and/or modify
|
||||
//it under the terms of the GNU General Public License as published by
|
||||
//the Free Software Foundation; either version 2 of the License, or
|
||||
//(at your option) any later version.
|
||||
//
|
||||
//This program is distributed in the hope that it will be useful,
|
||||
//but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
//GNU General Public License for more details.
|
||||
//
|
||||
//You should have received a copy of the GNU General Public License
|
||||
//along with this program; if not, write to the Free Software
|
||||
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
//
|
||||
//Using this software in any meaning (reading, learning, copying, compiling,
|
||||
//running) means that you agree that the Author(s) is (are) not responsible
|
||||
//for cost, loss of data or any harm that may be caused directly or indirectly
|
||||
//by usage of this softare or this documentation. The usage of this software
|
||||
//is on your own risk. The installation and usage (starting/running) of this
|
||||
//software may allow other people or application to access your computer and
|
||||
//any attached devices and is highly dependent on the configuration of the
|
||||
//software which must be done by the user of the software; the author(s) is
|
||||
//(are) also not responsible for proper configuration and usage of the
|
||||
//software, even if provoked by documentation provided together with
|
||||
//the software.
|
||||
//
|
||||
//Any changes to this file according to the GPL as documented in the file
|
||||
//gpl.txt aside this file in the shipment you received can be done to the
|
||||
//lines that follows this copyright notice here, but changes must not be
|
||||
//done inside the copyright notive above. A re-distribution must contain
|
||||
//the intact and unchanged copyright notice.
|
||||
//Contributions and changes to the program code must be marked as such.
|
||||
|
||||
|
||||
package de.anomic.icap;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URL;
|
||||
import java.util.Date;
|
||||
import java.util.Properties;
|
||||
|
||||
import de.anomic.http.httpChunkedInputStream;
|
||||
import de.anomic.http.httpHeader;
|
||||
import de.anomic.http.httpc;
|
||||
import de.anomic.plasma.plasmaHTCache;
|
||||
import de.anomic.plasma.plasmaParser;
|
||||
import de.anomic.plasma.plasmaSwitchboard;
|
||||
import de.anomic.server.serverCore;
|
||||
import de.anomic.server.serverFileUtils;
|
||||
import de.anomic.server.serverHandler;
|
||||
import de.anomic.server.logging.serverLog;
|
||||
import de.anomic.server.serverCore.Session;
|
||||
|
||||
/**
|
||||
* @author theli
|
||||
*/
|
||||
public class icapd implements serverHandler {
|
||||
|
||||
|
||||
private serverCore.Session session; // holds the session object of the calling class
|
||||
|
||||
// the connection properties
|
||||
private final Properties prop = new Properties();
|
||||
|
||||
// the address of the client
|
||||
private InetAddress userAddress;
|
||||
private String clientIP;
|
||||
private int keepAliveRequestCount = 0;
|
||||
|
||||
// needed for logging
|
||||
private final serverLog log = new serverLog("ICAPD");
|
||||
|
||||
private static plasmaSwitchboard switchboard = null;
|
||||
private static plasmaHTCache cacheManager = null;
|
||||
private static String virtualHost = null;
|
||||
private static boolean keepAliveSupport = true;
|
||||
|
||||
|
||||
|
||||
public icapd() {
|
||||
if (switchboard == null) {
|
||||
switchboard = plasmaSwitchboard.getSwitchboard();
|
||||
cacheManager = switchboard.cacheManager;
|
||||
virtualHost = switchboard.getConfig("fileHost","localhost");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public Object clone(){
|
||||
return new icapd();
|
||||
}
|
||||
|
||||
public void initSession(Session session) throws IOException {
|
||||
this.session = session;
|
||||
this.userAddress = session.userAddress; // client InetAddress
|
||||
this.clientIP = this.userAddress.getHostAddress();
|
||||
if (this.userAddress.isAnyLocalAddress()) this.clientIP = "localhost";
|
||||
if (this.clientIP.equals("0:0:0:0:0:0:0:1")) this.clientIP = "localhost";
|
||||
if (this.clientIP.equals("127.0.0.1")) this.clientIP = "localhost";
|
||||
}
|
||||
|
||||
public String greeting() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
public String error(Throwable e) {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
}
|
||||
|
||||
public Boolean EMPTY(String arg) throws IOException {
|
||||
// TODO Auto-generated method stub
|
||||
return serverCore.TERMINATE_CONNECTION;
|
||||
}
|
||||
|
||||
public Boolean UNKNOWN(String requestLine) throws IOException {
|
||||
// TODO Auto-generated method stub
|
||||
return serverCore.TERMINATE_CONNECTION;
|
||||
}
|
||||
|
||||
public icapHeader getDefaultHeaders() {
|
||||
icapHeader newHeaders = new icapHeader();
|
||||
|
||||
newHeaders.put(icapHeader.SERVER,"YaCy/" + switchboard.getConfig("vString",""));
|
||||
newHeaders.put(icapHeader.DATE, httpc.dateString(httpc.nowDate()));
|
||||
newHeaders.put(icapHeader.ISTAG, "\"" + switchboard.getConfig("vString","") + "\"");
|
||||
|
||||
return newHeaders;
|
||||
}
|
||||
|
||||
public Boolean OPTIONS(String arg) throws IOException {
|
||||
|
||||
BufferedInputStream in = new BufferedInputStream(this.session.in);
|
||||
BufferedOutputStream out = new BufferedOutputStream(this.session.out);
|
||||
|
||||
// parsing the http request line
|
||||
parseRequestLine(icapHeader.METHOD_OPTIONS,arg);
|
||||
|
||||
// reading the headers
|
||||
icapHeader icapReqHeader = icapHeader.readHeader(this.prop,this.session);
|
||||
|
||||
// determines if the connection should be kept alive
|
||||
boolean persistent = handlePersistentConnection(icapReqHeader);
|
||||
|
||||
// setting the icap response headers
|
||||
icapHeader resHeader = getDefaultHeaders();
|
||||
resHeader.put(icapHeader.ALLOW,"204");
|
||||
resHeader.put(icapHeader.ENCAPSULATED,"null-body=0");
|
||||
resHeader.put(icapHeader.MAX_CONNECTIONS,"1000");
|
||||
resHeader.put(icapHeader.OPTIONS_TTL,"300");
|
||||
resHeader.put(icapHeader.SERVICE_ID, "???");
|
||||
resHeader.put(icapHeader.PREVIEW, "30");
|
||||
resHeader.put(icapHeader.TRANSFER_COMPLETE, "*");
|
||||
//resHeader.put(icapHeader.TRANSFER_PREVIEW, "*");
|
||||
if (!persistent) resHeader.put(icapHeader.CONNECTION, "close");
|
||||
|
||||
|
||||
// determining the requested service and call it or send back an error message
|
||||
String reqService = this.prop.getProperty(icapHeader.CONNECTION_PROP_PATH,"");
|
||||
if (reqService.equalsIgnoreCase("/resIndexing")) {
|
||||
resHeader.put(icapHeader.SERVICE, "YaCy ICAP Indexing Service 1.0");
|
||||
resHeader.put(icapHeader.METHODS,icapHeader.METHOD_RESPMOD);
|
||||
|
||||
String transferIgnoreList = plasmaParser.getMediaExtList();
|
||||
transferIgnoreList = transferIgnoreList.substring(1,transferIgnoreList.length()-1);
|
||||
resHeader.put(icapHeader.TRANSFER_IGNORE, transferIgnoreList);
|
||||
} else {
|
||||
resHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0");
|
||||
}
|
||||
|
||||
|
||||
StringBuffer headerStringBuffer = resHeader.toHeaderString("ICAP/1.0",200,null);
|
||||
out.write(headerStringBuffer.toString().getBytes());
|
||||
out.flush();
|
||||
|
||||
return this.prop.getProperty(icapHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
|
||||
}
|
||||
|
||||
public Boolean REQMOD(String arg) throws IOException {
|
||||
return serverCore.TERMINATE_CONNECTION;
|
||||
}
|
||||
|
||||
public Boolean RESPMOD(String arg) throws IOException {
|
||||
try {
|
||||
InputStream in = this.session.in;
|
||||
OutputStream out = this.session.out;
|
||||
|
||||
// parsing the icap request line
|
||||
parseRequestLine(icapHeader.METHOD_RESPMOD,arg);
|
||||
|
||||
// reading the icap request header
|
||||
icapHeader icapReqHeader = icapHeader.readHeader(this.prop,this.session);
|
||||
|
||||
// determines if the connection should be kept alive
|
||||
handlePersistentConnection(icapReqHeader);
|
||||
|
||||
// determining the requested service and call it or send back an error message
|
||||
String reqService = (String) this.prop.getProperty(icapHeader.CONNECTION_PROP_PATH,"");
|
||||
if (reqService.equalsIgnoreCase("/resIndexing")) {
|
||||
indexingService(icapReqHeader,in,out);
|
||||
} else {
|
||||
icapHeader icapResHeader = getDefaultHeaders();
|
||||
icapResHeader.put(icapHeader.ENCAPSULATED,icapReqHeader.get(icapHeader.ENCAPSULATED));
|
||||
icapResHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0");
|
||||
// icapResHeader.put(icapHeader.CONNECTION, "close");
|
||||
|
||||
StringBuffer headerStringBuffer = icapResHeader.toHeaderString("ICAP/1.0",404,null);
|
||||
out.write(headerStringBuffer.toString().getBytes());
|
||||
out.flush();
|
||||
}
|
||||
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
|
||||
}
|
||||
return this.prop.getProperty(icapHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
|
||||
}
|
||||
|
||||
private void blacklistService(icapHeader reqHeader, InputStream in, OutputStream out) {
|
||||
try {
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private void indexingService(icapHeader reqHeader, InputStream in, OutputStream out) {
|
||||
try {
|
||||
|
||||
/* =========================================================================
|
||||
* Reading the various message parts into buffers
|
||||
* ========================================================================= */
|
||||
ByteArrayInputStream reqHdrStream = null, resHdrStream = null, resBodyStream = null;
|
||||
String[] encapsulated = ((String) reqHeader.get(icapHeader.ENCAPSULATED)).split(",");
|
||||
int prevLength = 0, currLength=0;
|
||||
for (int i=0; i < encapsulated.length; i++) {
|
||||
// reading the request header
|
||||
if (encapsulated[i].indexOf("req-hdr")>=0) {
|
||||
prevLength = currLength;
|
||||
currLength = Integer.parseInt(encapsulated[i+1].split("=")[1]);
|
||||
|
||||
byte[] buffer = new byte[currLength-prevLength];
|
||||
in.read(buffer, 0, buffer.length);
|
||||
|
||||
reqHdrStream = new ByteArrayInputStream(buffer);
|
||||
|
||||
// reading the response header
|
||||
} else if (encapsulated[i].indexOf("res-hdr")>=0) {
|
||||
prevLength = currLength;
|
||||
currLength = Integer.parseInt(encapsulated[i+1].split("=")[1]);
|
||||
|
||||
byte[] buffer = new byte[currLength-prevLength];
|
||||
in.read(buffer, 0, buffer.length);
|
||||
|
||||
resHdrStream = new ByteArrayInputStream(buffer);
|
||||
|
||||
// reading the response body
|
||||
} else if (encapsulated[i].indexOf("res-body")>=0) {
|
||||
httpChunkedInputStream chunkedIn = new httpChunkedInputStream(in);
|
||||
ByteArrayOutputStream bout = new ByteArrayOutputStream();
|
||||
int l = 0,len = 0;
|
||||
byte[] buffer = new byte[256];
|
||||
while ((l = chunkedIn.read(buffer)) >= 0) {
|
||||
len += l;
|
||||
bout.write(buffer,0,l);
|
||||
}
|
||||
resBodyStream = new ByteArrayInputStream(bout.toByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* sending back the icap status
|
||||
* ========================================================================= */
|
||||
icapHeader icapResHeader = getDefaultHeaders();
|
||||
if (reqHeader.allow(204)) {
|
||||
icapResHeader.put(icapHeader.ENCAPSULATED,reqHeader.get(icapHeader.ENCAPSULATED));
|
||||
icapResHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0");
|
||||
// resHeader.put(icapHeader.CONNECTION, "close");
|
||||
|
||||
StringBuffer headerStringBuffer = icapResHeader.toHeaderString("ICAP/1.0",204,null);
|
||||
out.write(headerStringBuffer.toString().getBytes());
|
||||
out.flush();
|
||||
} else {
|
||||
icapResHeader.put(icapHeader.ENCAPSULATED,reqHeader.get(icapHeader.ENCAPSULATED));
|
||||
icapResHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0");
|
||||
// icapResHeader.put(icapHeader.CONNECTION, "close");
|
||||
|
||||
StringBuffer headerStringBuffer = icapResHeader.toHeaderString("ICAP/1.0",503,null);
|
||||
out.write(headerStringBuffer.toString().getBytes());
|
||||
out.flush();
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* Parsing request data
|
||||
* ========================================================================= */
|
||||
// reading the requestline
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(reqHdrStream));
|
||||
String httpRequestLine = reader.readLine();
|
||||
|
||||
// parsing the requestline
|
||||
Properties httpReqProps = new Properties();
|
||||
httpHeader.parseRequestLine(httpRequestLine,httpReqProps,virtualHost);
|
||||
|
||||
if (!httpReqProps.getProperty(httpHeader.CONNECTION_PROP_METHOD).equals(httpHeader.METHOD_GET)) {
|
||||
this.log.logInfo("Wrong http request method for indexing:" +
|
||||
"\nRequest Method: " + httpReqProps.getProperty(httpHeader.CONNECTION_PROP_METHOD) +
|
||||
"\nRequest Line: " + httpRequestLine);
|
||||
reader.close();
|
||||
reqHdrStream.close();
|
||||
return;
|
||||
}
|
||||
|
||||
// reading all request headers
|
||||
httpHeader httpReqHeader = httpHeader.readHttpHeader(reader);
|
||||
reader.close();
|
||||
reqHdrStream.close();
|
||||
|
||||
// handle transparent proxy support: this function call is needed to set the host property properly
|
||||
httpHeader.handleTransparentProxySupport(httpReqHeader,httpReqProps,virtualHost,true);
|
||||
|
||||
// getting the request URL
|
||||
URL httpRequestURL = httpHeader.getRequestURL(httpReqProps);
|
||||
|
||||
/* =========================================================================
|
||||
* Parsing response data
|
||||
* ========================================================================= */
|
||||
// getting the response status
|
||||
reader = new BufferedReader(new InputStreamReader(resHdrStream));
|
||||
String httpRespStatusLine = reader.readLine();
|
||||
|
||||
Object[] httpRespStatus = httpHeader.parseResponseLine(httpRespStatusLine);
|
||||
|
||||
if (!(httpRespStatus[1].equals(new Integer(200)) || httpRespStatus[1].equals(new Integer(203)))) {
|
||||
this.log.logInfo("Wrong status code for indexing:" +
|
||||
"\nStatus Code: " + httpRespStatus[1] +
|
||||
"\nRequest Line: " + httpRequestLine +
|
||||
"\nResponse Line: " + httpRespStatusLine);
|
||||
reader.close();
|
||||
resHdrStream.close();
|
||||
return;
|
||||
}
|
||||
|
||||
// reading all response headers
|
||||
httpHeader httpResHeader = httpHeader.readHttpHeader(reader);
|
||||
reader.close();
|
||||
resHdrStream.close();
|
||||
|
||||
if ((!(plasmaParser.supportedMimeTypesContains(httpResHeader.mime()))) &&
|
||||
(!(plasmaParser.supportedFileExt(httpRequestURL)))) {
|
||||
this.log.logInfo("Wrong mimeType or fileExtension for indexing:" +
|
||||
"\nMimeType: " + httpResHeader.mime() +
|
||||
"\nRequest Line:" + httpRequestLine);
|
||||
return ;
|
||||
}
|
||||
|
||||
|
||||
/* =========================================================================
|
||||
* Prepare data for indexing
|
||||
* ========================================================================= */
|
||||
|
||||
// generating a htcache entry object
|
||||
plasmaHTCache.Entry cacheEntry = cacheManager.newEntry(
|
||||
new Date(),
|
||||
0,
|
||||
httpRequestURL,
|
||||
"",
|
||||
httpReqHeader,
|
||||
httpRespStatusLine,
|
||||
httpResHeader,
|
||||
null,
|
||||
switchboard.defaultProxyProfile
|
||||
);
|
||||
|
||||
// getting the filename/path to store the response body
|
||||
File cacheFile = cacheManager.getCachePath(httpRequestURL);
|
||||
|
||||
// if the file already exits we delete it
|
||||
if (cacheFile.isFile()) {
|
||||
cacheManager.deleteFile(httpRequestURL);
|
||||
}
|
||||
// we write the new cache entry to file system directly
|
||||
cacheFile.getParentFile().mkdirs();
|
||||
|
||||
// copy the response body into the file
|
||||
serverFileUtils.copy(resBodyStream,cacheFile);
|
||||
resBodyStream.close(); resBodyStream = null;
|
||||
|
||||
// indexing the response
|
||||
cacheManager.push(cacheEntry);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private final void parseRequestLine(String cmd, String s) {
|
||||
// parsing the requestlin
|
||||
icapHeader.parseRequestLine(cmd,s, this.prop,virtualHost);
|
||||
|
||||
// adding the client ip prop
|
||||
this.prop.setProperty(icapHeader.CONNECTION_PROP_CLIENTIP, this.clientIP);
|
||||
|
||||
// counting the amount of received requests within this permanent conneciton
|
||||
this.prop.setProperty(icapHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer.toString(++this.keepAliveRequestCount));
|
||||
}
|
||||
|
||||
private boolean handlePersistentConnection(icapHeader header) {
|
||||
|
||||
if (!keepAliveSupport) {
|
||||
this.prop.put(icapHeader.CONNECTION_PROP_PERSISTENT,"close");
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean persistent = true;
|
||||
if (((String)header.get(icapHeader.CONNECTION, "keep-alive")).toLowerCase().equals("close")) {
|
||||
persistent = false;
|
||||
}
|
||||
|
||||
this.prop.put(icapHeader.CONNECTION_PROP_PERSISTENT,persistent?"keep-alive":"close");
|
||||
return persistent;
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue