boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) Some performance improvements

Browse Source 
- many classes set to final
- implementation of a session-thread pool
- reusage of the server handler class (normally the httpd object)
  within the session thread
- implementation of a httpc object pool
- introduction of a linebuffer in httpd which can be reused
- reusing the properties table in the httpc
- added to apache libs (commons-collections, commons-pool) which 
  are needed for the object/thread pool implementation

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@26 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
(no author) 20 years ago
parent d5ff81c636
commit f39812da91
27 changed files with 1318 additions and 717 deletions
Show Stats Download Patch File Download Diff File

BIN
lib/commons-collections.jar
View File

Binary file not shown.

BIN
lib/commons-pool-1.2.jar
View File

Binary file not shown.

27
makerelease.sh
Unescape View File

@ -52,9 +52,12 @@ release='yacy_dev_v'$version'_'$datestr
target='RELEASE' target='RELEASE'
classes='classes' classes='classes'
source='source' source='source'
lib='lib'
doc='doc' doc='doc'
data='DATA' data='DATA'
mainclass='yacy.java' mainclass='yacy.java'
classpath='$classes:$lib'
mkdir $release mkdir $release
# clean up # clean up
@ -103,18 +106,18 @@ mv -f $source/$mainclass $source/$mainclass.orig
sed `echo 's/<<REPL_DATE>>/'$datestr'/'` $source/$mainclass.orig > $source/$mainclass.sed1 sed `echo 's/<<REPL_DATE>>/'$datestr'/'` $source/$mainclass.orig > $source/$mainclass.sed1
sed `echo 's/<<REPL_VERSION>>/'$version'/'` $source/$mainclass.sed1 > $source/$mainclass sed `echo 's/<<REPL_VERSION>>/'$version'/'` $source/$mainclass.sed1 > $source/$mainclass
rm $source/$mainclass.sed1 rm $source/$mainclass.sed1
#javac -classpath $classes -sourcepath $source -d $classes -g:none $source/httpd.java #javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/httpd.java
#javac -classpath $classes -sourcepath $source -d $classes -g:none $source/$mainclass #javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/$mainclass
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/net/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/net/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/server/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/server/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/http/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/http/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/kelondro/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/kelondro/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/data/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/data/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/plasma/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/plasma/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/yacy/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/yacy/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/$mainclass javac -classpath $classpath -sourcepath $source -d $classes -g $source/$mainclass
mv -f $source/$mainclass.orig $source/$mainclass mv -f $source/$mainclass.orig $source/$mainclass
# compile server pages # compile server pages

17
source/de/anomic/htmlFilter/htmlFilterOutputStream.java
Unescape View File

@ -53,7 +53,7 @@ import java.net.*;
import java.util.*; import java.util.*;
import de.anomic.server.*; import de.anomic.server.*;
public class htmlFilterOutputStream extends OutputStream { public final class htmlFilterOutputStream extends OutputStream {
public static final byte lb = (byte) '<'; public static final byte lb = (byte) '<';
public static final byte rb = (byte) '>'; public static final byte rb = (byte) '>';
@ -321,14 +321,15 @@ public class htmlFilterOutputStream extends OutputStream {
} }
} else if (inScript) { } else if (inScript) {
buffer.append(b); buffer.append(b);
int bufferLength = buffer.length();
if ((b == rb) && (buffer.length() > 14) && if ((b == rb) && (buffer.length() > 14) &&
(buffer.byteAt(buffer.length() - 8) == (byte) '/') && (buffer.byteAt(bufferLength - 8) == (byte) '/') &&
(buffer.byteAt(buffer.length() - 7) == (byte) 's') && (buffer.byteAt(bufferLength - 7) == (byte) 's') &&
(buffer.byteAt(buffer.length() - 6) == (byte) 'c') && (buffer.byteAt(bufferLength - 6) == (byte) 'c') &&
(buffer.byteAt(buffer.length() - 5) == (byte) 'r') && (buffer.byteAt(bufferLength - 5) == (byte) 'r') &&
(buffer.byteAt(buffer.length() - 4) == (byte) 'i') && (buffer.byteAt(bufferLength - 4) == (byte) 'i') &&
(buffer.byteAt(buffer.length() - 3) == (byte) 'p') && (buffer.byteAt(bufferLength - 3) == (byte) 'p') &&
(buffer.byteAt(buffer.length() - 2) == (byte) 't')) { (buffer.byteAt(bufferLength - 2) == (byte) 't')) {
// script is at end // script is at end
inScript = false; inScript = false;
if (out != null) out.write(buffer.getBytes()); if (out != null) out.write(buffer.getBytes());

14
source/de/anomic/http/httpHeader.java
Unescape View File

@ -57,9 +57,9 @@ import java.util.*;
import java.text.*; import java.text.*;
import de.anomic.server.*; import de.anomic.server.*;
public class httpHeader extends TreeMap implements Map { public final class httpHeader extends TreeMap implements Map {
private HashMap reverseMappingCache; private final HashMap reverseMappingCache;
private static Collator insensitiveCollator = Collator.getInstance(Locale.US); private static Collator insensitiveCollator = Collator.getInstance(Locale.US);
static { static {
@ -111,16 +111,17 @@ public class httpHeader extends TreeMap implements Map {
// we override the put method to make use of the reverseMappingCache // we override the put method to make use of the reverseMappingCache
public Object put(Object key, Object value) { public Object put(Object key, Object value) {
String k = (String) key; String k = (String) key;
String upperK = k.toUpperCase();
if (reverseMappingCache == null) { if (reverseMappingCache == null) {
return super.put(k, value); return super.put(k, value);
} else { } else {
if (reverseMappingCache.containsKey(k.toUpperCase())) { if (reverseMappingCache.containsKey(upperK)) {
// we put in the value using the reverse mapping // we put in the value using the reverse mapping
return super.put(reverseMappingCache.get(k.toUpperCase()), value); return super.put(reverseMappingCache.get(upperK), value);
} else { } else {
// we put in without a cached key and store the key afterwards // we put in without a cached key and store the key afterwards
Object r = super.put(k, value); Object r = super.put(k, value);
reverseMappingCache.put(k.toUpperCase(), k); reverseMappingCache.put(upperK, k);
return r; return r;
} }
} }
@ -180,9 +181,8 @@ public class httpHeader extends TreeMap implements Map {
} catch (java.lang.NumberFormatException e) { } catch (java.lang.NumberFormatException e) {
//System.out.println("ERROR long version parse: " + e.getMessage() + " at position " + e.getErrorOffset()); //System.out.println("ERROR long version parse: " + e.getMessage() + " at position " + e.getErrorOffset());
serverLog.logError("HTTPC-header", "DATE ERROR (NumberFormat): " + s); serverLog.logError("HTTPC-header", "DATE ERROR (NumberFormat): " + s);
new Date(); return new Date();
} }
return new Date();
} }
private Date headerDate(String kind) { private Date headerDate(String kind) {

2
source/de/anomic/http/httpTemplate.java
Unescape View File

@ -46,7 +46,7 @@ import de.anomic.server.*;
import java.util.*; import java.util.*;
import java.io.*; import java.io.*;
public class httpTemplate { final class httpTemplate {
private static final byte hash = (byte)'#'; private static final byte hash = (byte)'#';
private static final byte[] hasha = {hash}; private static final byte[] hasha = {hash};

424
source/de/anomic/http/httpc.java
Unescape View File

@ -56,23 +56,29 @@ import java.lang.*;
import java.util.*; import java.util.*;
import java.util.zip.*; import java.util.zip.*;
import de.anomic.server.*; import de.anomic.server.*;
import de.anomic.server.serverCore.Session;
import de.anomic.server.serverCore.SessionFactory;
import de.anomic.server.serverCore.SessionPool;
import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.SSLSocketFactory;
public class httpc { import org.apache.commons.pool.impl.GenericObjectPool;
public final class httpc {
// statics // statics
private static final String vDATE = "20040602"; private static final String vDATE = "20040602";
private static String userAgent; private static String userAgent;
public static String systemOST; public static String systemOST;
private static final int terminalMaxLength = 30000; private static final int terminalMaxLength = 30000;
private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); private static final TimeZone GMTTimeZone = TimeZone.getTimeZone("PST");
// --- The GMT standard date format used in the HTTP protocol // --- The GMT standard date format used in the HTTP protocol
private static SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US); private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US);
private static SimpleDateFormat EMLFormatter = new SimpleDateFormat("dd MMM yyyy HH:mm:ss", Locale.US); private static final SimpleDateFormat EMLFormatter = new SimpleDateFormat("dd MMM yyyy HH:mm:ss", Locale.US);
private static SimpleDateFormat ShortFormatter = new SimpleDateFormat("yyyyMMddHHmmss"); private static final SimpleDateFormat ShortFormatter = new SimpleDateFormat("yyyyMMddHHmmss");
//Mo 06 Sep 2004 23:32 //Mo 06 Sep 2004 23:32
private static HashMap reverseMappingCache = new HashMap(); private static final HashMap reverseMappingCache = new HashMap();
// class variables // class variables
private Socket socket = null; // client socket for commands private Socket socket = null; // client socket for commands
@ -89,15 +95,83 @@ public class httpc {
private String requestPath = null; private String requestPath = null;
// the dns cache // the dns cache
private static HashMap nameCacheHit = new HashMap(); private static final HashMap nameCacheHit = new HashMap();
//private static HashSet nameCacheMiss = new HashSet(); //private static HashSet nameCacheMiss = new HashSet();
static { static {
// set time-out of InetAddress.getByName cache ttl // set time-out of InetAddress.getByName cache ttl
java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); java.security.Security.setProperty("networkaddress.cache.ttl" , "60");
java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0"); java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0");
} }
private static final httpcPool theHttpcPool;
static {
// implementation of session thread pool
GenericObjectPool.Config config = new GenericObjectPool.Config();
// The maximum number of active connections that can be allocated from pool at the same time,
// 0 for no limit
config.maxActive = 150;
// The maximum number of idle connections connections in the pool
// 0 = no limit.
config.maxIdle = 75;
config.minIdle = 10;
config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
config.minEvictableIdleTimeMillis = 30000;
theHttpcPool = new httpcPool(new httpcFactory(),config);
}
private static final ByteArrayOutputStream readLineBuffer = new ByteArrayOutputStream();
public static httpc getInstance(String server, int port, int timeout, boolean ssl,
String remoteProxyHost, int remoteProxyPort) throws IOException {
try {
// fetching a new httpc from the object pool
httpc newHttpc = (httpc) httpc.theHttpcPool.borrowObject();
// initialize it
newHttpc.init(server,port,timeout,ssl,remoteProxyHost, remoteProxyPort);
return newHttpc;
} catch (Exception e) {
throw new IOException("Unable to initialize a new httpc. " + e.getMessage());
}
}
public static httpc getInstance(String server, int port, int timeout, boolean ssl) throws IOException {
try {
// fetching a new httpc from the object pool
httpc newHttpc = (httpc) httpc.theHttpcPool.borrowObject();
// initialize it
newHttpc.init(server,port,timeout,ssl);
return newHttpc;
} catch (Exception e) {
throw new IOException("Unable to initialize a new httpc. " + e.getMessage());
}
}
public static void returnInstance(httpc theHttpc) {
try {
theHttpc.reset();
httpc.theHttpcPool.returnObject(theHttpc);
} catch (Exception e) {
// we could ignore this error
}
}
protected void finalize() throws Throwable {
System.err.println("Httpc object was not returned to object pool.");
this.reset();
httpc.theHttpcPool.invalidateObject(this);
}
public static String dnsResolve(String host) { public static String dnsResolve(String host) {
// looks for the ip of host <host> and returns ip number as string // looks for the ip of host <host> and returns ip number as string
String ip = (String) nameCacheHit.get(host); String ip = (String) nameCacheHit.get(host);
@ -134,45 +208,72 @@ public class httpc {
return false; return false;
} }
} }
void reset() {
try {
if (this.clientInput != null) {
this.clientInput.close();
this.clientInput = null;
}
if (this.clientOutput != null) {
this.clientOutput.close();
this.clientOutput = null;
}
if (this.socket != null) {
this.socket.close();
this.socket = null;
}
this.host = null;
this.timeout = 0;
this.handle = 0;
this.remoteProxyUse = false;
this.savedRemoteHost = null;
this.requestPath = null;
} catch (Exception e) {
// we could ignore this ...
}
}
// http client // http client
public httpc(String server, int port, int timeout, boolean ssl, void init(String server, int port, int timeout, boolean ssl,
String remoteProxyHost, int remoteProxyPort) throws IOException { String remoteProxyHost, int remoteProxyPort) throws IOException {
this(remoteProxyHost, remoteProxyPort, timeout, ssl); this.init(remoteProxyHost, remoteProxyPort, timeout, ssl);
this.remoteProxyUse = true; this.remoteProxyUse = true;
this.savedRemoteHost = server + ((port == 80) ? "" : (":" + port)); this.savedRemoteHost = server + ((port == 80) ? "" : (":" + port));
} }
public httpc(String server, int port, int timeout, boolean ssl) throws IOException { void init(String server, int port, int timeout, boolean ssl) throws IOException {
handle = System.currentTimeMillis(); handle = System.currentTimeMillis();
//serverLog.logDebug("HTTPC", handle + " initialized"); //serverLog.logDebug("HTTPC", handle + " initialized");
this.remoteProxyUse = false; this.remoteProxyUse = false;
this.timeout = timeout; this.timeout = timeout;
this.savedRemoteHost = server; this.savedRemoteHost = server;
try { try {
this.host = server + ((port == 80) ? "" : (":" + port)); this.host = server + ((port == 80) ? "" : (":" + port));
String hostip; String hostip;
if ((server.equals("localhost")) || (server.equals("127.0.0.1")) || (server.startsWith("192.168.")) || (server.startsWith("10."))) { if ((server.equals("localhost")) || (server.equals("127.0.0.1")) || (server.startsWith("192.168.")) || (server.startsWith("10."))) {
hostip = server; hostip = server;
} else { } else {
hostip = dnsResolve(server); hostip = dnsResolve(server);
if (hostip == null) throw new UnknownHostException(server); if (hostip == null) throw new UnknownHostException(server);
} }
if (ssl) if (ssl)
socket = SSLSocketFactory.getDefault().createSocket(hostip, port); socket = SSLSocketFactory.getDefault().createSocket(hostip, port);
else else
socket = new Socket(hostip, port); socket = new Socket(hostip, port);
socket.setSoTimeout(timeout); // waiting time for write socket.setSoTimeout(timeout); // waiting time for write
//socket.setSoLinger(true, timeout); // waiting time for read //socket.setSoLinger(true, timeout); // waiting time for read
socket.setKeepAlive(true); // socket.setKeepAlive(true); //
clientInput = new PushbackInputStream(socket.getInputStream()); clientInput = new PushbackInputStream(socket.getInputStream());
clientOutput = socket.getOutputStream(); clientOutput = socket.getOutputStream();
// if we reached this point, we should have a connection // if we reached this point, we should have a connection
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
throw new IOException("unknown host: " + server); throw new IOException("unknown host: " + server);
} }
} }
// provide HTTP date handling static methods // provide HTTP date handling static methods
@ -230,7 +331,7 @@ public class httpc {
} }
// reads in the http header, right now, right here // reads in the http header, right now, right here
byte[] b = serverCore.receive(clientInput, timeout, terminalMaxLength, false); byte[] b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false);
if (b == null) { if (b == null) {
// the server has meanwhile disconnected // the server has meanwhile disconnected
status = "503 server has closed connection"; status = "503 server has closed connection";
@ -242,7 +343,7 @@ public class httpc {
if (p < 0) { if (p < 0) {
status = "500 status line parse error"; status = "500 status line parse error";
// flush in anything that comes without parsing // flush in anything that comes without parsing
while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)).length != 0) {} while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)).length != 0) {}
return; // in bad mood return; // in bad mood
} }
// we have a status // we have a status
@ -252,14 +353,14 @@ public class httpc {
if (status.startsWith("400")) { if (status.startsWith("400")) {
// bad request // bad request
// flush in anything that comes without parsing // flush in anything that comes without parsing
while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)).length != 0) {} while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)).length != 0) {}
return; // in bad mood return; // in bad mood
} }
// at this point we should have a valid response. read in the header properties // at this point we should have a valid response. read in the header properties
String key = ""; String key = "";
String value = ""; String value = "";
while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)) != null) { while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)) != null) {
if (b.length == 0) break; if (b.length == 0) break;
buffer = new String(b); buffer = new String(b);
//System.out.println("#H#" + buffer); // debug //System.out.println("#H#" + buffer); // debug
@ -727,24 +828,35 @@ do upload
String user, String password, boolean ssl, String user, String password, boolean ssl,
String proxyHost, int proxyPort, String proxyHost, int proxyPort,
httpHeader requestHeader) throws IOException { httpHeader requestHeader) throws IOException {
if (requestHeader == null) requestHeader = new httpHeader(); if (requestHeader == null) requestHeader = new httpHeader();
if ((user != null) && (password != null) && (user.length() != 0)) { if ((user != null) && (password != null) && (user.length() != 0)) {
requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password));
} }
httpc con;
if ((proxyHost == null) || (proxyPort == 0)) httpc con = null;
con = new httpc(host, port, timeout, ssl); try {
else
con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort); if ((proxyHost == null) || (proxyPort == 0)) {
httpc.response res = con.GET(path, null); con = httpc.getInstance(host, port, timeout, ssl);
if (res.status.startsWith("2")) { } else {
ByteArrayOutputStream bos = new ByteArrayOutputStream(); con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort);
res.writeContent(bos, null); }
con.close();
return bos.toByteArray(); httpc.response res = con.GET(path, null);
} else { if (res.status.startsWith("2")) {
return res.status.getBytes(); ByteArrayOutputStream bos = new ByteArrayOutputStream();
} res.writeContent(bos, null);
con.close();
return bos.toByteArray();
} else {
return res.status.getBytes();
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
} }
public static byte[] singleGET(URL u, int timeout, public static byte[] singleGET(URL u, int timeout,
@ -773,25 +885,35 @@ do upload
String user, String password, boolean ssl, String user, String password, boolean ssl,
String proxyHost, int proxyPort, String proxyHost, int proxyPort,
httpHeader requestHeader, serverObjects props) throws IOException { httpHeader requestHeader, serverObjects props) throws IOException {
if (requestHeader == null) requestHeader = new httpHeader();
if ((user != null) && (password != null) && (user.length() != 0)) { if (requestHeader == null) requestHeader = new httpHeader();
requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); if ((user != null) && (password != null) && (user.length() != 0)) {
} requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password));
httpc con; }
if ((proxyHost == null) || (proxyPort == 0))
con = new httpc(host, port, timeout, ssl); httpc con = null;
else try {
con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort); if ((proxyHost == null) || (proxyPort == 0))
httpc.response res = con.POST(path, null, props, null); con = httpc.getInstance(host, port, timeout, ssl);
//System.out.println("response=" + res.toString()); else
if (res.status.startsWith("2")) { con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort);
ByteArrayOutputStream bos = new ByteArrayOutputStream(); httpc.response res = con.POST(path, null, props, null);
res.writeContent(bos, null);
con.close(); //System.out.println("response=" + res.toString());
return bos.toByteArray(); if (res.status.startsWith("2")) {
} else { ByteArrayOutputStream bos = new ByteArrayOutputStream();
return res.status.getBytes(); res.writeContent(bos, null);
} con.close();
return bos.toByteArray();
} else {
return res.status.getBytes();
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
} }
public static byte[] singlePOST(URL u, int timeout, public static byte[] singlePOST(URL u, int timeout,
@ -831,33 +953,41 @@ do upload
} }
public static httpHeader whead(URL url, int timeout, String user, String password, String proxyHost, int proxyPort) throws IOException { public static httpHeader whead(URL url, int timeout, String user, String password, String proxyHost, int proxyPort) throws IOException {
// generate request header // generate request header
httpHeader requestHeader = new httpHeader(); httpHeader requestHeader = new httpHeader();
if ((user != null) && (password != null) && (user.length() != 0)) { if ((user != null) && (password != null) && (user.length() != 0)) {
requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password));
} }
// parse query // parse query
int port = url.getPort(); int port = url.getPort();
boolean ssl = url.getProtocol().equals("https"); boolean ssl = url.getProtocol().equals("https");
if (port < 0) port = (ssl) ? 443 : 80; if (port < 0) port = (ssl) ? 443 : 80;
String path = url.getPath(); String path = url.getPath();
String query = url.getQuery(); String query = url.getQuery();
if ((query != null) && (query.length() > 0)) path = path + "?" + query; if ((query != null) && (query.length() > 0)) path = path + "?" + query;
String host = url.getHost(); String host = url.getHost();
// start connection // start connection
httpc con; httpc con = null;
if ((proxyHost == null) || (proxyPort == 0)) try {
con = new httpc(host, port, timeout, ssl); if ((proxyHost == null) || (proxyPort == 0))
else con = httpc.getInstance(host, port, timeout, ssl);
con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort); else con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort);
httpc.response res = con.HEAD(path, requestHeader);
if (res.status.startsWith("2")) { httpc.response res = con.HEAD(path, requestHeader);
// success if (res.status.startsWith("2")) {
return res.responseHeader; // success
} else { return res.responseHeader;
// fail } else {
return res.responseHeader; // fail
} return res.responseHeader;
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
} }
/* /*
@ -929,6 +1059,9 @@ do upload
Enumeration i = text.elements(); Enumeration i = text.elements();
while (i.hasMoreElements()) System.out.println((String) i.nextElement()); while (i.hasMoreElements()) System.out.println((String) i.nextElement());
} }
} }
@ -1027,3 +1160,90 @@ public class SSLSocketClientWithClientAuth {
} }
} }
*/ */
final class httpcFactory implements org.apache.commons.pool.PoolableObjectFactory {
public httpcFactory() {
super();
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#makeObject()
*/
public Object makeObject() throws Exception {
return new httpc();
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)
*/
public void destroyObject(Object obj) {
if (obj instanceof httpc) {
httpc theHttpc = (httpc) obj;
}
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object)
*/
public boolean validateObject(Object obj) {
if (obj instanceof httpc)
{
httpc theHttpc = (httpc) obj;
return true;
}
return true;
}
/**
* @param obj
*
*/
public void activateObject(Object obj) {
//log.debug(" activateObject...");
}
/**
* @param obj
*
*/
public void passivateObject(Object obj) {
//log.debug(" passivateObject..." + obj);
if (obj instanceof Session) {
httpc theHttpc = (httpc) obj;
}
}
}
final class httpcPool extends GenericObjectPool {
/**
* First constructor.
* @param objFactory
*/
public httpcPool(httpcFactory objFactory) {
super(objFactory);
this.setMaxIdle(75); // Maximum idle threads.
this.setMaxActive(150); // Maximum active threads.
this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs.
//this.setMaxWait(1000); // Wait 1 second till a thread is available
}
public httpcPool(httpcFactory objFactory,
GenericObjectPool.Config config) {
super(objFactory, config);
}
/**
* @see org.apache.commons.pool.impl.GenericObjectPool#borrowObject()
*/
public Object borrowObject() throws Exception {
return super.borrowObject();
}
/**
* @see org.apache.commons.pool.impl.GenericObjectPool#returnObject(java.lang.Object)
*/
public void returnObject(Object obj) throws Exception {
super.returnObject(obj);
}
}

176
source/de/anomic/http/httpd.java
Unescape View File

@ -53,9 +53,15 @@ import java.io.*;
import java.net.*; import java.net.*;
import java.util.*; import java.util.*;
import java.text.*; import java.text.*;
import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.server.*; import de.anomic.server.*;
import de.anomic.server.serverCore.Session;
import de.anomic.server.serverCore.SessionFactory;
import de.anomic.server.serverCore.SessionPool;
public class httpd implements serverHandler { public final class httpd implements serverHandler {
// static objects // static objects
public static final String vDATE = "<<REPL>>"; public static final String vDATE = "<<REPL>>";
@ -77,15 +83,19 @@ public class httpd implements serverHandler {
private String proxyAccountBase64MD5; private String proxyAccountBase64MD5;
private String serverAccountBase64MD5; private String serverAccountBase64MD5;
private String clientIP; private String clientIP;
// the connection properties
private static final Properties prop = new Properties();
// class methods // class methods
public httpd(serverSwitch s, httpdHandler fileHandler, httpdHandler proxyHandler) { public httpd(serverSwitch s, httpdHandler fileHandler, httpdHandler proxyHandler) {
// handler info // handler info
this.switchboard = s; httpd.switchboard = s;
this.fileHandler = fileHandler; httpd.fileHandler = fileHandler;
this.proxyHandler = proxyHandler; httpd.proxyHandler = proxyHandler;
this.virtualHost = switchboard.getConfig("fileHost","localhost"); httpd.virtualHost = switchboard.getConfig("fileHost","localhost");
// authentication: by default none // authentication: by default none
this.proxyAccountBase64MD5 = null; this.proxyAccountBase64MD5 = null;
@ -93,99 +103,109 @@ public class httpd implements serverHandler {
this.clientIP = null; this.clientIP = null;
} }
public void reset() {
this.session = null;
this.userAddress = null;
this.allowProxy = false;
this.allowServer = false;
this.proxyAccountBase64MD5 = null;
this.serverAccountBase64MD5 = null;
this.clientIP = null;
}
// must be called at least once, but can be called again to re-use the object. // must be called at least once, but can be called again to re-use the object.
public void initSession(serverCore.Session session) throws IOException { public void initSession(serverCore.Session session) throws IOException {
this.session = session; this.session = session;
this.userAddress = session.userAddress; // client InetAddress this.userAddress = session.userAddress; // client InetAddress
this.clientIP = userAddress.getHostAddress(); this.clientIP = userAddress.getHostAddress();
if (this.userAddress.isAnyLocalAddress()) this.clientIP = "localhost"; if (this.userAddress.isAnyLocalAddress()) this.clientIP = "localhost";
if (this.clientIP.equals("0:0:0:0:0:0:0:1")) this.clientIP = "localhost"; if (this.clientIP.equals("0:0:0:0:0:0:0:1")) this.clientIP = "localhost";
if (this.clientIP.equals("127.0.0.1")) this.clientIP = "localhost"; if (this.clientIP.equals("127.0.0.1")) this.clientIP = "localhost";
String proxyClient = switchboard.getConfig("proxyClient", "*"); String proxyClient = switchboard.getConfig("proxyClient", "*");
String serverClient = switchboard.getConfig("serverClient", "*"); String serverClient = switchboard.getConfig("serverClient", "*");
this.allowProxy = (proxyClient.equals("*")) ? true : match(clientIP, proxyClient); this.allowProxy = (proxyClient.equals("*")) ? true : match(clientIP, proxyClient);
this.allowServer = (serverClient.equals("*")) ? true : match(clientIP, serverClient); this.allowServer = (serverClient.equals("*")) ? true : match(clientIP, serverClient);
// check if we want to allow this socket to connect us // check if we want to allow this socket to connect us
if (!((allowProxy) || (allowServer))) { if (!((allowProxy) || (allowServer))) {
throw new IOException("CONNECTION FROM " + clientIP + " FORBIDDEN"); throw new IOException("CONNECTION FROM " + clientIP + " FORBIDDEN");
} }
proxyAccountBase64MD5 = null; proxyAccountBase64MD5 = null;
serverAccountBase64MD5 = null; serverAccountBase64MD5 = null;
} }
private static boolean match(String key, String latch) { private static boolean match(String key, String latch) {
// the latch is a comma-separated list of patterns // the latch is a comma-separated list of patterns
// each pattern may contain one wildcard-character '*' which matches anything // each pattern may contain one wildcard-character '*' which matches anything
StringTokenizer st = new StringTokenizer(latch,","); StringTokenizer st = new StringTokenizer(latch,",");
String pattern; String pattern;
int pos; int pos;
while (st.hasMoreTokens()) { while (st.hasMoreTokens()) {
pattern = st.nextToken(); pattern = st.nextToken();
pos = pattern.indexOf("*"); pos = pattern.indexOf("*");
if (pos < 0) { if (pos < 0) {
// no wild card: exact match // no wild card: exact match
if (key.equals(pattern)) return true; if (key.equals(pattern)) return true;
} else { } else {
// wild card: match left and right side of pattern // wild card: match left and right side of pattern
if ((key.startsWith(pattern.substring(0, pos))) && if ((key.startsWith(pattern.substring(0, pos))) &&
(key.endsWith(pattern.substring(pos + 1)))) return true; (key.endsWith(pattern.substring(pos + 1)))) return true;
} }
} }
return false; return false;
} }
public String greeting() { // OBLIGATORIC FUNCTION public String greeting() { // OBLIGATORIC FUNCTION
// a response line upon connection is send to client // a response line upon connection is send to client
// if no response line is wanted, return "" or null // if no response line is wanted, return "" or null
return null; return null;
} }
public String error(Throwable e) { // OBLIGATORIC FUNCTION public String error(Throwable e) { // OBLIGATORIC FUNCTION
// return string in case of any error that occurs during communication // return string in case of any error that occurs during communication
// is always (but not only) called if an IO-dependent exception occurrs. // is always (but not only) called if an IO-dependent exception occurrs.
e.printStackTrace(); e.printStackTrace();
return "501 Exception occurred: " + e.getMessage(); return "501 Exception occurred: " + e.getMessage();
} }
private String readLine() { private String readLine() {
// reads a line from the input socket // reads a line from the input socket
// this function is provided by the server through a passed method on initialization // this function is provided by the server through a passed method on initialization
byte[] l = this.session.readLine(); byte[] l = this.session.readLine();
if (l == null) return null; else return new String(l); if (l == null) return null; else return new String(l);
} }
private httpHeader readHeader() { private httpHeader readHeader() {
httpHeader header = new httpHeader(reverseMappingCache); httpHeader header = new httpHeader(reverseMappingCache);
int p; int p;
String line; String line;
String key; String key;
String value; String value;
while ((line = readLine()) != null) { while ((line = readLine()) != null) {
if (line.length() == 0) break; // this seperates the header of the HTTP request from the body if (line.length() == 0) break; // this seperates the header of the HTTP request from the body
//System.out.println("***" + line); // debug //System.out.println("***" + line); // debug
// parse the header line: a property seperated with the ':' sign // parse the header line: a property seperated with the ':' sign
p = line.indexOf(":"); p = line.indexOf(":");
if (p >= 0) { if (p >= 0) {
// store a property // store a property
key = line.substring(0, p).trim(); key = line.substring(0, p).trim();
value = (String) header.get(key); value = (String) header.get(key);
// check if the header occurred already // check if the header occurred already
if (value == null) { if (value == null) {
// create new entry // create new entry
header.put(key, line.substring(p + 1).trim()); header.put(key, line.substring(p + 1).trim());
} else { } else {
// value can occur double times, attach with '#' - separator // value can occur double times, attach with '#' - separator
header.put(key, value + "#" + line.substring(p + 1).trim()); header.put(key, value + "#" + line.substring(p + 1).trim());
} }
} }
} }
return header; return header;
} }
public Boolean GET(String arg) throws IOException { public Boolean GET(String arg) throws IOException {
Properties prop = parseQuery(arg); parseQuery(prop, arg);
prop.setProperty("METHOD", "GET"); prop.setProperty("METHOD", "GET");
prop.setProperty("CLIENTIP", clientIP); prop.setProperty("CLIENTIP", clientIP);
@ -268,7 +288,7 @@ public class httpd implements serverHandler {
} }
public Boolean HEAD(String arg) throws IOException { public Boolean HEAD(String arg) throws IOException {
Properties prop = parseQuery(arg); parseQuery(prop,arg);
prop.setProperty("METHOD", "HEAD"); prop.setProperty("METHOD", "HEAD");
prop.setProperty("CLIENTIP", clientIP); prop.setProperty("CLIENTIP", clientIP);
@ -342,7 +362,7 @@ public class httpd implements serverHandler {
} }
public Boolean POST(String arg) throws IOException { public Boolean POST(String arg) throws IOException {
Properties prop = parseQuery(arg); parseQuery(prop, arg);
prop.setProperty("METHOD", "POST"); prop.setProperty("METHOD", "POST");
prop.setProperty("CLIENTIP", clientIP); prop.setProperty("CLIENTIP", clientIP);
@ -487,8 +507,13 @@ public class httpd implements serverHandler {
} }
private Properties parseQuery(String s) { private static final Properties parseQuery(Properties prop, String s) {
Properties prop = new Properties();
if (prop == null) {
prop = new Properties();
} else {
prop.clear();
}
// this parses a whole URL // this parses a whole URL
if (s.length() == 0) { if (s.length() == 0) {
@ -819,7 +844,6 @@ permission
if (pos < 0) return false; if (pos < 0) return false;
return whitelist.contains(mime.substring(0, pos)); return whitelist.contains(mime.substring(0, pos));
} }
} }
/* /*

4
source/de/anomic/http/httpdAbstractHandler.java
Unescape View File

@ -51,11 +51,9 @@
package de.anomic.http; package de.anomic.http;
import java.io.*;
import java.util.*;
import java.text.*; import java.text.*;
public abstract class httpdAbstractHandler { abstract class httpdAbstractHandler {
// static tools // static tools

1
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -76,7 +76,6 @@ package de.anomic.http;
import java.io.*; import java.io.*;
import java.util.*; import java.util.*;
import java.text.*;
import java.lang.reflect.*; import java.lang.reflect.*;
import de.anomic.server.*; import de.anomic.server.*;

176
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -62,16 +62,13 @@ package de.anomic.http;
import java.io.*; import java.io.*;
import java.net.*; import java.net.*;
import java.util.*; import java.util.*;
import java.text.*;
import de.anomic.htmlFilter.*; import de.anomic.htmlFilter.*;
import de.anomic.server.*; import de.anomic.server.*;
import de.anomic.tools.*;
import de.anomic.yacy.*; import de.anomic.yacy.*;
import de.anomic.http.*;
import de.anomic.plasma.*; import de.anomic.plasma.*;
public class httpdProxyHandler extends httpdAbstractHandler implements httpdHandler { public final class httpdProxyHandler extends httpdAbstractHandler implements httpdHandler {
// static variables // static variables
// can only be instantiated upon first instantiation of this class object // can only be instantiated upon first instantiation of this class object
@ -87,10 +84,10 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
public static int remoteProxyPort = -1; public static int remoteProxyPort = -1;
public static String remoteProxyNoProxy = ""; public static String remoteProxyNoProxy = "";
public static String[] remoteProxyNoProxyPatterns = null; public static String[] remoteProxyNoProxyPatterns = null;
private static HashSet remoteProxyAllowProxySet = new HashSet(); private static final HashSet remoteProxyAllowProxySet = new HashSet();
private static HashSet remoteProxyDisallowProxySet = new HashSet(); private static final HashSet remoteProxyDisallowProxySet = new HashSet();
private static htmlFilterTransformer transformer = null; private static htmlFilterTransformer transformer = null;
public static String userAgent = "yacy (" + httpc.systemOST +") yacy.net"; public static final String userAgent = "yacy (" + httpc.systemOST +") yacy.net";
private File htRootPath = null; private File htRootPath = null;
// class methods // class methods
@ -108,8 +105,6 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
} }
remoteProxyUse = switchboard.getConfig("remoteProxyUse","false").equals("true"); remoteProxyUse = switchboard.getConfig("remoteProxyUse","false").equals("true");
remoteProxyNoProxy = switchboard.getConfig("remoteProxyNoProxy",""); remoteProxyNoProxy = switchboard.getConfig("remoteProxyNoProxy","");
remoteProxyAllowProxySet = new HashSet();
remoteProxyDisallowProxySet = new HashSet();
remoteProxyNoProxyPatterns = remoteProxyNoProxy.split(","); remoteProxyNoProxyPatterns = remoteProxyNoProxy.split(",");
// set loglevel // set loglevel
@ -586,6 +581,8 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
respond.write(("]\r\n").getBytes()); respond.write(("]\r\n").getBytes());
} }
} catch (Exception ee) {} } catch (Exception ee) {}
} finally {
if (remote != null) httpc.returnInstance(remote);
} }
respond.flush(); respond.flush();
} }
@ -686,7 +683,10 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
e.printStackTrace(new PrintStream(respond)); e.printStackTrace(new PrintStream(respond));
respond.write(("]\r\n").getBytes()); respond.write(("]\r\n").getBytes());
} catch (Exception ee) {} } catch (Exception ee) {}
} } finally {
if (remote != null) httpc.returnInstance(remote);
}
respond.flush(); respond.flush();
} }
@ -740,71 +740,80 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
e.printStackTrace(new PrintStream(respond)); e.printStackTrace(new PrintStream(respond));
respond.write(("]\r\n").getBytes()); respond.write(("]\r\n").getBytes());
} catch (Exception ee) {} } catch (Exception ee) {}
} } finally {
if (remote != null) httpc.returnInstance(remote);
}
respond.flush(); respond.flush();
} }
public void doConnect(Properties conProp, de.anomic.http.httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) throws IOException { public void doConnect(Properties conProp, de.anomic.http.httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) throws IOException {
String host = conProp.getProperty("HOST"); String host = conProp.getProperty("HOST");
int port = Integer.parseInt(conProp.getProperty("PORT")); int port = Integer.parseInt(conProp.getProperty("PORT"));
String httpVersion = conProp.getProperty("HTTP"); String httpVersion = conProp.getProperty("HTTP");
int timeout = Integer.parseInt(switchboard.getConfig("clientTimeout", "10000")); int timeout = Integer.parseInt(switchboard.getConfig("clientTimeout", "10000"));
// possibly branch into PROXY-PROXY connection // possibly branch into PROXY-PROXY connection
if (remoteProxyUse) { if (remoteProxyUse) {
httpc remoteProxy = new httpc(host, port, timeout, false, remoteProxyHost, remoteProxyPort); httpc remoteProxy = null;
httpc.response response = remoteProxy.CONNECT(host, port, requestHeader); try {
response.print(); remoteProxy = httpc.getInstance(host, port, timeout, false, remoteProxyHost, remoteProxyPort);
if (response.success()) { httpc.response response = remoteProxy.CONNECT(host, port, requestHeader);
// replace connection details response.print();
host = remoteProxyHost; if (response.success()) {
port = remoteProxyPort; // replace connection details
// go on (see below) host = remoteProxyHost;
} else { port = remoteProxyPort;
// pass error response back to client // go on (see below)
respondHeader(clientOut, response.status, response.responseHeader); } else {
return; // pass error response back to client
} respondHeader(clientOut, response.status, response.responseHeader);
} return;
}
// try to establish connection to remote host } catch (Exception e) {
Socket sslSocket = new Socket(host, port); throw new IOException(e.getMessage());
sslSocket.setSoTimeout(timeout); // waiting time for write } finally {
sslSocket.setSoLinger(true, timeout); // waiting time for read if (remoteProxy != null) httpc.returnInstance(remoteProxy);
InputStream promiscuousIn = sslSocket.getInputStream(); }
OutputStream promiscuousOut = sslSocket.getOutputStream(); }
// now then we can return a success message // try to establish connection to remote host
clientOut.write((httpVersion + " 200 Connection established" + serverCore.crlfString + Socket sslSocket = new Socket(host, port);
"Proxy-agent: YACY" + serverCore.crlfString + sslSocket.setSoTimeout(timeout); // waiting time for write
serverCore.crlfString).getBytes()); sslSocket.setSoLinger(true, timeout); // waiting time for read
InputStream promiscuousIn = sslSocket.getInputStream();
log.logInfo("SSL CONNECTION TO " + host + ":" + port + " ESTABLISHED"); OutputStream promiscuousOut = sslSocket.getOutputStream();
// start stream passing with mediate processes // now then we can return a success message
try { clientOut.write((httpVersion + " 200 Connection established" + serverCore.crlfString +
Mediate cs = new Mediate(sslSocket, clientIn, promiscuousOut); "Proxy-agent: YACY" + serverCore.crlfString +
Mediate sc = new Mediate(sslSocket, promiscuousIn, clientOut); serverCore.crlfString).getBytes());
cs.start();
sc.start(); log.logInfo("SSL CONNECTION TO " + host + ":" + port + " ESTABLISHED");
while ((sslSocket != null) &&
(sslSocket.isBound()) && // start stream passing with mediate processes
(!(sslSocket.isClosed())) && try {
(sslSocket.isConnected()) && Mediate cs = new Mediate(sslSocket, clientIn, promiscuousOut);
((cs.isAlive()) || (sc.isAlive()))) { Mediate sc = new Mediate(sslSocket, promiscuousIn, clientOut);
// idle cs.start();
try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait a while sc.start();
} while ((sslSocket != null) &&
// set stop mode (sslSocket.isBound()) &&
cs.pleaseTerminate(); (!(sslSocket.isClosed())) &&
sc.pleaseTerminate(); (sslSocket.isConnected()) &&
// wake up thread ((cs.isAlive()) || (sc.isAlive()))) {
cs.interrupt(); // idle
sc.interrupt(); try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait a while
// ...hope they have terminated... }
} catch (IOException e) { // set stop mode
//System.out.println("promiscuous termination: " + e.getMessage()); cs.pleaseTerminate();
} sc.pleaseTerminate();
// wake up thread
cs.interrupt();
sc.interrupt();
// ...hope they have terminated...
} catch (IOException e) {
//System.out.println("promiscuous termination: " + e.getMessage());
}
} }
@ -873,9 +882,9 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
} }
// branch to server/proxy // branch to server/proxy
if (useProxy) { if (useProxy) {
return new httpc(server, port, timeout, false, remoteProxyHost, remoteProxyPort); return httpc.getInstance(server, port, timeout, false, remoteProxyHost, remoteProxyPort);
} else { } else {
return new httpc(server, port, timeout, false); return httpc.getInstance(server, port, timeout, false);
} }
} }
@ -895,15 +904,18 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
} }
private void respondHeader(OutputStream respond, String status, httpHeader header) throws IOException, SocketException { private void respondHeader(OutputStream respond, String status, httpHeader header) throws IOException, SocketException {
String s;
// prepare header // prepare header
//header.put("Server", "AnomicHTTPD (www.anomic.de)"); //header.put("Server", "AnomicHTTPD (www.anomic.de)");
if (!(header.containsKey("date"))) header.put("Date", httpc.dateString(httpc.nowDate())); if (!(header.containsKey("date"))) header.put("Date", httpc.dateString(httpc.nowDate()));
if (!(header.containsKey("content-type"))) header.put("Content-type", "text/html"); // fix this if (!(header.containsKey("content-type"))) header.put("Content-type", "text/html"); // fix this
StringBuffer headerStringBuffer = new StringBuffer();
// write status line // write status line
respond.write(("HTTP/1.1 " + status + "\r\n").getBytes()); headerStringBuffer.append("HTTP/1.1 ")
.append(status)
.append("\r\n");
//System.out.println("HEADER: PROXY TO CLIENT = " + header.toString()); // DEBUG //System.out.println("HEADER: PROXY TO CLIENT = " + header.toString()); // DEBUG
@ -920,17 +932,27 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
if (!(key.equals("Location"))) while ((pos = value.lastIndexOf("#")) >= 0) { if (!(key.equals("Location"))) while ((pos = value.lastIndexOf("#")) >= 0) {
// special handling is needed if a key appeared several times, which is valid. // special handling is needed if a key appeared several times, which is valid.
// all lines with same key are combined in one value, separated by a "#" // all lines with same key are combined in one value, separated by a "#"
respond.write((key + ": " + value.substring(pos + 1).trim() + "\r\n").getBytes()); headerStringBuffer
.append(key)
.append(": ")
.append(value.substring(pos + 1).trim())
.append("\r\n");
//System.out.println("#" + key + ": " + value.substring(pos + 1).trim()); //System.out.println("#" + key + ": " + value.substring(pos + 1).trim());
value = value.substring(0, pos).trim(); value = value.substring(0, pos).trim();
} }
respond.write((key + ": " + value + "\r\n").getBytes()); headerStringBuffer
.append(key)
.append(": ")
.append(value)
.append("\r\n");
//System.out.println("#" + key + ": " + value); //System.out.println("#" + key + ": " + value);
} }
} }
headerStringBuffer.append("\r\n");
// end header // end header
respond.write(("\r\n").getBytes()); respond.write(headerStringBuffer.toString().getBytes());
respond.flush(); respond.flush();
} }

4
source/de/anomic/http/httpdSwitchboard.java
Unescape View File

@ -44,9 +44,9 @@ import java.io.*;
import java.util.*; import java.util.*;
import de.anomic.server.*; import de.anomic.server.*;
public class httpdSwitchboard extends serverAbstractSwitch implements serverSwitch { public final class httpdSwitchboard extends serverAbstractSwitch implements serverSwitch {
private LinkedList cacheStack; private final LinkedList cacheStack;
public httpdSwitchboard(String rootPath, String initPath, String configPath) throws IOException { public httpdSwitchboard(String rootPath, String initPath, String configPath) throws IOException {
super(rootPath, initPath, configPath); super(rootPath, initPath, configPath);

38
source/de/anomic/plasma/plasmaCrawlLoader.java
Unescape View File

@ -50,7 +50,7 @@ import de.anomic.server.*;
import de.anomic.tools.*; import de.anomic.tools.*;
import de.anomic.htmlFilter.*; import de.anomic.htmlFilter.*;
public class plasmaCrawlLoader { public final class plasmaCrawlLoader {
private plasmaHTCache cacheManager; private plasmaHTCache cacheManager;
private int socketTimeout; private int socketTimeout;
@ -130,7 +130,7 @@ public class plasmaCrawlLoader {
return result; return result;
} }
public class Exec extends Thread { public final class Exec extends Thread {
public URL url; public URL url;
public String referer; public String referer;
@ -160,9 +160,8 @@ public class plasmaCrawlLoader {
private httpc newhttpc(String server, int port, boolean ssl) throws IOException { private httpc newhttpc(String server, int port, boolean ssl) throws IOException {
// a new httpc connection, combined with possible remote proxy // a new httpc connection, combined with possible remote proxy
if (remoteProxyUse) if (remoteProxyUse)
return new httpc(server, port, socketTimeout, ssl, remoteProxyHost, remoteProxyPort); return httpc.getInstance(server, port, socketTimeout, ssl, remoteProxyHost, remoteProxyPort);
else else return httpc.getInstance(server, port, socketTimeout, ssl);
return new httpc(server, port, socketTimeout, ssl);
} }
private void load(URL url, String referer, String initiator, int depth, plasmaCrawlProfile.entry profile) throws IOException { private void load(URL url, String referer, String initiator, int depth, plasmaCrawlProfile.entry profile) throws IOException {
@ -179,20 +178,21 @@ public class plasmaCrawlLoader {
if (referer.length() == 0) referer = "http://www.yacy.net/yacy/"; if (referer.length() == 0) referer = "http://www.yacy.net/yacy/";
// take a file from the net // take a file from the net
httpc remote = null;
try { try {
// create a request header // create a request header
httpHeader requestHeader = new httpHeader(); httpHeader requestHeader = new httpHeader();
requestHeader.put("User-Agent", httpdProxyHandler.userAgent); requestHeader.put("User-Agent", httpdProxyHandler.userAgent);
requestHeader.put("Referer", referer); requestHeader.put("Referer", referer);
requestHeader.put("Accept-Encoding", "gzip,deflate"); requestHeader.put("Accept-Encoding", "gzip,deflate");
//System.out.println("CRAWLER_REQUEST_HEADER=" + requestHeader.toString()); // DEBUG //System.out.println("CRAWLER_REQUEST_HEADER=" + requestHeader.toString()); // DEBUG
// open the connection // open the connection
httpc remote = newhttpc(host, port, ssl); remote = newhttpc(host, port, ssl);
// send request // send request
httpc.response res = remote.GET(path, requestHeader); httpc.response res = remote.GET(path, requestHeader);
if (res.status.startsWith("200")) { if (res.status.startsWith("200")) {
// the transfer is ok // the transfer is ok
@ -250,6 +250,8 @@ public class plasmaCrawlLoader {
// remote server was wrong. // remote server was wrong.
log.logError("CRAWLER LOADER ERROR2 with url=" + url.toString() + ": " + e.toString()); log.logError("CRAWLER LOADER ERROR2 with url=" + url.toString() + ": " + e.toString());
e.printStackTrace(); e.printStackTrace();
} finally {
if (remote != null) httpc.returnInstance(remote);
} }
} }

8
source/de/anomic/server/serverAbstractSwitch.java
Unescape View File

@ -47,12 +47,12 @@ import java.util.*;
public abstract class serverAbstractSwitch implements serverSwitch { public abstract class serverAbstractSwitch implements serverSwitch {
// configuration management // configuration management
private File configFile; private final File configFile;
private Hashtable configProps; private Hashtable configProps;
private String configComment; private final String configComment;
private Hashtable authorization; private final Hashtable authorization;
private String rootPath; private String rootPath;
private TreeMap workerThreads; private final TreeMap workerThreads;
public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException { public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException {
// we initialize the switchboard with a property file, // we initialize the switchboard with a property file,

41
source/de/anomic/server/serverAbstractThread.java
Unescape View File

@ -58,14 +58,14 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
private long threadBlockTimestamp = System.currentTimeMillis(); private long threadBlockTimestamp = System.currentTimeMillis();
private long idleCycles = 0, busyCycles = 0; private long idleCycles = 0, busyCycles = 0;
protected void announceThreadBlockApply() { protected final void announceThreadBlockApply() {
// shall only be used, if a thread blocks for an important reason // shall only be used, if a thread blocks for an important reason
// like a socket connect and must renew the timestamp to correct // like a socket connect and must renew the timestamp to correct
// statistics // statistics
this.threadBlockTimestamp = System.currentTimeMillis(); this.threadBlockTimestamp = System.currentTimeMillis();
} }
protected void announceThreadBlockRelease() { protected final void announceThreadBlockRelease() {
// shall only be used, if a thread blocks for an important reason // shall only be used, if a thread blocks for an important reason
// like a socket connect and must renew the timestamp to correct // like a socket connect and must renew the timestamp to correct
// statistics // statistics
@ -74,69 +74,69 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
this.busytime -= thisBlockTime; this.busytime -= thisBlockTime;
} }
protected void announceMoreExecTime(long millis) { protected final void announceMoreExecTime(long millis) {
this.busytime += millis; this.busytime += millis;
} }
protected void announceMoreSleepTime(long millis) { protected final void announceMoreSleepTime(long millis) {
this.idletime += millis; this.idletime += millis;
} }
public void setDescription(String shortText, String longText) { public final void setDescription(String shortText, String longText) {
// sets a visible description string // sets a visible description string
this.shortDescr = shortText; this.shortDescr = shortText;
this.longDescr = longText; this.longDescr = longText;
} }
public void setStartupSleep(long milliseconds) { public final void setStartupSleep(long milliseconds) {
// sets a sleep time before execution of the job-loop // sets a sleep time before execution of the job-loop
startup = milliseconds; startup = milliseconds;
} }
public void setIdleSleep(long milliseconds) { public final void setIdleSleep(long milliseconds) {
// sets a sleep time for pauses between two jobs // sets a sleep time for pauses between two jobs
idlePause = milliseconds; idlePause = milliseconds;
} }
public void setBusySleep(long milliseconds) { public final void setBusySleep(long milliseconds) {
// sets a sleep time for pauses between two jobs // sets a sleep time for pauses between two jobs
busyPause = milliseconds; busyPause = milliseconds;
} }
public String getShortDescription() { public final String getShortDescription() {
return this.shortDescr; return this.shortDescr;
} }
public String getLongDescription() { public final String getLongDescription() {
return this.longDescr; return this.longDescr;
} }
public long getIdleCycles() { public final long getIdleCycles() {
// returns the total number of cycles of job execution with idle-result // returns the total number of cycles of job execution with idle-result
return this.idleCycles; return this.idleCycles;
} }
public long getBusyCycles() { public final long getBusyCycles() {
// returns the total number of cycles of job execution with busy-result // returns the total number of cycles of job execution with busy-result
return this.busyCycles; return this.busyCycles;
} }
public long getBlockTime() { public final long getBlockTime() {
// returns the total time that this thread has been blocked so far // returns the total time that this thread has been blocked so far
return this.blockPause; return this.blockPause;
} }
public long getSleepTime() { public final long getSleepTime() {
// returns the total time that this thread has slept so far // returns the total time that this thread has slept so far
return this.idletime; return this.idletime;
} }
public long getExecTime() { public final long getExecTime() {
// returns the total time that this thread has worked so far // returns the total time that this thread has worked so far
return this.busytime; return this.busytime;
} }
public void setLog(serverLog log) { public final void setLog(serverLog log) {
// defines a log where process states can be written to // defines a log where process states can be written to
this.log = log; this.log = log;
} }
@ -145,12 +145,15 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
// after calling this method, the thread shall terminate // after calling this method, the thread shall terminate
this.running = false; this.running = false;
// wait for termination // wait for termination
if (waitFor) while (this.isAlive()) if (waitFor) {
try {this.sleep(100);} catch (InterruptedException e) {break;} // Busy waiting removed: while (this.isAlive()) try {this.sleep(100);} catch (InterruptedException e) {break;}
try { this.join(); } catch (InterruptedException e) {return;}
}
// If we reach this point, the process is closed // If we reach this point, the process is closed
} }
private void logError(String text) { private final void logError(String text) {
if (log == null) if (log == null)
serverLog.logError("THREAD-CONTROL", text); serverLog.logError("THREAD-CONTROL", text);
else else

15
source/de/anomic/server/serverByteBuffer.java
Unescape View File

@ -43,7 +43,7 @@ package de.anomic.server;
import java.io.*; import java.io.*;
import java.util.*; import java.util.*;
public class serverByteBuffer extends OutputStream { public final class serverByteBuffer extends OutputStream {
public static final byte singlequote = (byte) 39; public static final byte singlequote = (byte) 39;
public static final byte doublequote = (byte) 34; public static final byte doublequote = (byte) 34;
@ -94,13 +94,14 @@ public class serverByteBuffer extends OutputStream {
try { try {
FileInputStream fis = new FileInputStream(f); FileInputStream fis = new FileInputStream(f);
byte buf[] = new byte[512]; // byte buf[] = new byte[512];
int p = 0; // int p = 0;
int l; int l;
while ((l = fis.read(buf)) > 0) { // while ((l = fis.read(buf)) > 0) {
System.arraycopy(buf, 0, buffer, p, l); // System.arraycopy(buf, 0, buffer, p, l);
p += l; // p += l;
} l = fis.read(buffer);
// }
fis.close(); fis.close();
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
throw new IOException("File not found: " + f.toString() + "; " + e.getMessage()); throw new IOException("File not found: " + f.toString() + "; " + e.getMessage());

4
source/de/anomic/server/serverClassLoader.java
Unescape View File

@ -45,9 +45,9 @@ import java.io.*;
import java.util.*; import java.util.*;
import java.lang.reflect.*; import java.lang.reflect.*;
public class serverClassLoader extends ClassLoader { public final class serverClassLoader extends ClassLoader {
Hashtable classes; private final Hashtable classes;
public serverClassLoader() { public serverClassLoader() {
super(ClassLoader.getSystemClassLoader()); super(ClassLoader.getSystemClassLoader());

2
source/de/anomic/server/serverCodings.java
Unescape View File

@ -44,7 +44,7 @@ import java.io.*;
import java.security.*; import java.security.*;
public class serverCodings { public final class serverCodings {
// this provides encoding and decoding of long cardinals into a 6-bit - based number format // this provides encoding and decoding of long cardinals into a 6-bit - based number format
// expressed by a string. This is probably the most compact form to encode numbers as strings. // expressed by a string. This is probably the most compact form to encode numbers as strings.

1059
source/de/anomic/server/serverCore.java
View File

File diff suppressed because it is too large Load Diff

2
source/de/anomic/server/serverDate.java
Unescape View File

@ -47,7 +47,7 @@ import java.lang.*;
import java.util.*; import java.util.*;
import java.text.*; import java.text.*;
public class serverDate { public final class serverDate {
// statics // statics

2
source/de/anomic/server/serverFileUtils.java
Unescape View File

@ -42,7 +42,7 @@ package de.anomic.server;
import java.io.*; import java.io.*;
public class serverFileUtils { public final class serverFileUtils {
public static void copy(InputStream source, OutputStream dest) throws IOException { public static void copy(InputStream source, OutputStream dest) throws IOException {
byte[] buffer = new byte[4096]; byte[] buffer = new byte[4096];

4
source/de/anomic/server/serverHandler.java
Unescape View File

@ -110,4 +110,8 @@ public interface serverHandler {
// but only the necessary one for a newly initialized instance // but only the necessary one for a newly initialized instance
public Object clone(); public Object clone();
// Instead of using clone this function can be used to reset an existing
// handler prototype so that it can e reused
public void reset();
} }

2
source/de/anomic/server/serverInstantThread.java
Unescape View File

@ -3,7 +3,7 @@ package de.anomic.server;
import java.lang.reflect.*; import java.lang.reflect.*;
public class serverInstantThread extends serverAbstractThread implements serverThread { public final class serverInstantThread extends serverAbstractThread implements serverThread {
private Method jobExecMethod, jobCountMethod; private Method jobExecMethod, jobCountMethod;
private Object environment; private Object environment;

2
source/de/anomic/server/serverLog.java
Unescape View File

@ -43,7 +43,7 @@ package de.anomic.server;
import java.text.*; import java.text.*;
import java.util.*; import java.util.*;
public class serverLog { public final class serverLog {
// statics // statics
private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST");

2
source/de/anomic/server/serverObjects.java
Unescape View File

@ -60,7 +60,7 @@ package de.anomic.server;
import java.io.*; import java.io.*;
import java.util.*; import java.util.*;
public class serverObjects extends Hashtable implements Cloneable { public final class serverObjects extends Hashtable implements Cloneable {
public serverObjects() { public serverObjects() {
super(); super();

3
source/de/anomic/server/serverSystem.java
Unescape View File

@ -41,11 +41,10 @@
package de.anomic.server; package de.anomic.server;
import java.io.*; import java.io.*;
import java.net.*;
import java.util.*; import java.util.*;
import java.lang.reflect.*; import java.lang.reflect.*;
public class serverSystem { public final class serverSystem {
// constants for system identification // constants for system identification
public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.* public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.*

6
source/yacy.java
Unescape View File

@ -75,11 +75,11 @@ import de.anomic.server.*;
import de.anomic.yacy.*; import de.anomic.yacy.*;
//import de.anomic.http.*; //import de.anomic.http.*;
public class yacy { public final class yacy {
// static objects // static objects
private static final String vString = "0.361"; private static final String vString = "0.361";
private static final String vDATE = "@REPL_DATE@"; private static final String vDATE = "20050419";
private static final String copyright = "[ YACY Proxy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]"; private static final String copyright = "[ YACY Proxy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
private static final String hline = "-------------------------------------------------------------------------------"; private static final String hline = "-------------------------------------------------------------------------------";
@ -321,7 +321,7 @@ public class yacy {
httpHeader requestHeader = new httpHeader(); httpHeader requestHeader = new httpHeader();
requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify
try { try {
httpc con = new httpc("localhost", port, 10000, false); httpc con = httpc.getInstance("localhost", port, 10000, false);
httpc.response res = con.GET("Steering.html?shutdown=", requestHeader); httpc.response res = con.GET("Steering.html?shutdown=", requestHeader);
// read response // read response

Write Preview
Loading…
Cancel
Save