diff --git a/lib/commons-collections.jar b/lib/commons-collections.jar new file mode 100644 index 000000000..5cc4f9062 Binary files /dev/null and b/lib/commons-collections.jar differ diff --git a/lib/commons-pool-1.2.jar b/lib/commons-pool-1.2.jar new file mode 100644 index 000000000..4ba534c90 Binary files /dev/null and b/lib/commons-pool-1.2.jar differ diff --git a/makerelease.sh b/makerelease.sh index da93b5110..e847fbf97 100755 --- a/makerelease.sh +++ b/makerelease.sh @@ -52,9 +52,12 @@ release='yacy_dev_v'$version'_'$datestr target='RELEASE' classes='classes' source='source' +lib='lib' doc='doc' data='DATA' mainclass='yacy.java' +classpath='$classes:$lib' + mkdir $release # clean up @@ -103,18 +106,18 @@ mv -f $source/$mainclass $source/$mainclass.orig sed `echo 's/<>/'$datestr'/'` $source/$mainclass.orig > $source/$mainclass.sed1 sed `echo 's/<>/'$version'/'` $source/$mainclass.sed1 > $source/$mainclass rm $source/$mainclass.sed1 -#javac -classpath $classes -sourcepath $source -d $classes -g:none $source/httpd.java -#javac -classpath $classes -sourcepath $source -d $classes -g:none $source/$mainclass -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/net/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/server/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/http/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/kelondro/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/data/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/plasma/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/yacy/*.java -javac -classpath $classes -sourcepath $source -d $classes -g $source/$mainclass +#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/httpd.java +#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/$mainclass +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/net/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/server/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/http/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/kelondro/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/data/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/plasma/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/yacy/*.java +javac -classpath $classpath -sourcepath $source -d $classes -g $source/$mainclass mv -f $source/$mainclass.orig $source/$mainclass # compile server pages diff --git a/source/de/anomic/htmlFilter/htmlFilterOutputStream.java b/source/de/anomic/htmlFilter/htmlFilterOutputStream.java index 4550db46e..14c236efc 100644 --- a/source/de/anomic/htmlFilter/htmlFilterOutputStream.java +++ b/source/de/anomic/htmlFilter/htmlFilterOutputStream.java @@ -53,7 +53,7 @@ import java.net.*; import java.util.*; import de.anomic.server.*; -public class htmlFilterOutputStream extends OutputStream { +public final class htmlFilterOutputStream extends OutputStream { public static final byte lb = (byte) '<'; public static final byte rb = (byte) '>'; @@ -321,14 +321,15 @@ public class htmlFilterOutputStream extends OutputStream { } } else if (inScript) { buffer.append(b); + int bufferLength = buffer.length(); if ((b == rb) && (buffer.length() > 14) && - (buffer.byteAt(buffer.length() - 8) == (byte) '/') && - (buffer.byteAt(buffer.length() - 7) == (byte) 's') && - (buffer.byteAt(buffer.length() - 6) == (byte) 'c') && - (buffer.byteAt(buffer.length() - 5) == (byte) 'r') && - (buffer.byteAt(buffer.length() - 4) == (byte) 'i') && - (buffer.byteAt(buffer.length() - 3) == (byte) 'p') && - (buffer.byteAt(buffer.length() - 2) == (byte) 't')) { + (buffer.byteAt(bufferLength - 8) == (byte) '/') && + (buffer.byteAt(bufferLength - 7) == (byte) 's') && + (buffer.byteAt(bufferLength - 6) == (byte) 'c') && + (buffer.byteAt(bufferLength - 5) == (byte) 'r') && + (buffer.byteAt(bufferLength - 4) == (byte) 'i') && + (buffer.byteAt(bufferLength - 3) == (byte) 'p') && + (buffer.byteAt(bufferLength - 2) == (byte) 't')) { // script is at end inScript = false; if (out != null) out.write(buffer.getBytes()); diff --git a/source/de/anomic/http/httpHeader.java b/source/de/anomic/http/httpHeader.java index 5adc8ad87..c043d6cb3 100644 --- a/source/de/anomic/http/httpHeader.java +++ b/source/de/anomic/http/httpHeader.java @@ -57,9 +57,9 @@ import java.util.*; import java.text.*; import de.anomic.server.*; -public class httpHeader extends TreeMap implements Map { +public final class httpHeader extends TreeMap implements Map { - private HashMap reverseMappingCache; + private final HashMap reverseMappingCache; private static Collator insensitiveCollator = Collator.getInstance(Locale.US); static { @@ -111,16 +111,17 @@ public class httpHeader extends TreeMap implements Map { // we override the put method to make use of the reverseMappingCache public Object put(Object key, Object value) { String k = (String) key; + String upperK = k.toUpperCase(); if (reverseMappingCache == null) { return super.put(k, value); } else { - if (reverseMappingCache.containsKey(k.toUpperCase())) { + if (reverseMappingCache.containsKey(upperK)) { // we put in the value using the reverse mapping - return super.put(reverseMappingCache.get(k.toUpperCase()), value); + return super.put(reverseMappingCache.get(upperK), value); } else { // we put in without a cached key and store the key afterwards Object r = super.put(k, value); - reverseMappingCache.put(k.toUpperCase(), k); + reverseMappingCache.put(upperK, k); return r; } } @@ -180,9 +181,8 @@ public class httpHeader extends TreeMap implements Map { } catch (java.lang.NumberFormatException e) { //System.out.println("ERROR long version parse: " + e.getMessage() + " at position " + e.getErrorOffset()); serverLog.logError("HTTPC-header", "DATE ERROR (NumberFormat): " + s); - new Date(); + return new Date(); } - return new Date(); } private Date headerDate(String kind) { diff --git a/source/de/anomic/http/httpTemplate.java b/source/de/anomic/http/httpTemplate.java index 360a21cf9..84e98b178 100644 --- a/source/de/anomic/http/httpTemplate.java +++ b/source/de/anomic/http/httpTemplate.java @@ -46,7 +46,7 @@ import de.anomic.server.*; import java.util.*; import java.io.*; -public class httpTemplate { +final class httpTemplate { private static final byte hash = (byte)'#'; private static final byte[] hasha = {hash}; diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index a81f5db14..570b4487f 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -56,23 +56,29 @@ import java.lang.*; import java.util.*; import java.util.zip.*; import de.anomic.server.*; +import de.anomic.server.serverCore.Session; +import de.anomic.server.serverCore.SessionFactory; +import de.anomic.server.serverCore.SessionPool; + import javax.net.ssl.SSLSocketFactory; -public class httpc { +import org.apache.commons.pool.impl.GenericObjectPool; + +public final class httpc { // statics private static final String vDATE = "20040602"; private static String userAgent; public static String systemOST; private static final int terminalMaxLength = 30000; - private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); + private static final TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); // --- The GMT standard date format used in the HTTP protocol - private static SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US); - private static SimpleDateFormat EMLFormatter = new SimpleDateFormat("dd MMM yyyy HH:mm:ss", Locale.US); - private static SimpleDateFormat ShortFormatter = new SimpleDateFormat("yyyyMMddHHmmss"); + private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US); + private static final SimpleDateFormat EMLFormatter = new SimpleDateFormat("dd MMM yyyy HH:mm:ss", Locale.US); + private static final SimpleDateFormat ShortFormatter = new SimpleDateFormat("yyyyMMddHHmmss"); //Mo 06 Sep 2004 23:32 - private static HashMap reverseMappingCache = new HashMap(); + private static final HashMap reverseMappingCache = new HashMap(); // class variables private Socket socket = null; // client socket for commands @@ -89,15 +95,83 @@ public class httpc { private String requestPath = null; // the dns cache - private static HashMap nameCacheHit = new HashMap(); + private static final HashMap nameCacheHit = new HashMap(); //private static HashSet nameCacheMiss = new HashSet(); static { - // set time-out of InetAddress.getByName cache ttl - java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); + // set time-out of InetAddress.getByName cache ttl + java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0"); } + + private static final httpcPool theHttpcPool; + static { + // implementation of session thread pool + GenericObjectPool.Config config = new GenericObjectPool.Config(); + + // The maximum number of active connections that can be allocated from pool at the same time, + // 0 for no limit + config.maxActive = 150; + + // The maximum number of idle connections connections in the pool + // 0 = no limit. + config.maxIdle = 75; + config.minIdle = 10; + + config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; + config.minEvictableIdleTimeMillis = 30000; + + theHttpcPool = new httpcPool(new httpcFactory(),config); + } + + private static final ByteArrayOutputStream readLineBuffer = new ByteArrayOutputStream(); + + public static httpc getInstance(String server, int port, int timeout, boolean ssl, + String remoteProxyHost, int remoteProxyPort) throws IOException { + + try { + // fetching a new httpc from the object pool + httpc newHttpc = (httpc) httpc.theHttpcPool.borrowObject(); + + // initialize it + newHttpc.init(server,port,timeout,ssl,remoteProxyHost, remoteProxyPort); + return newHttpc; + + } catch (Exception e) { + throw new IOException("Unable to initialize a new httpc. " + e.getMessage()); + } + } + + public static httpc getInstance(String server, int port, int timeout, boolean ssl) throws IOException { + + try { + // fetching a new httpc from the object pool + httpc newHttpc = (httpc) httpc.theHttpcPool.borrowObject(); + + // initialize it + newHttpc.init(server,port,timeout,ssl); + return newHttpc; + + } catch (Exception e) { + throw new IOException("Unable to initialize a new httpc. " + e.getMessage()); + } + } + + public static void returnInstance(httpc theHttpc) { + try { + theHttpc.reset(); + httpc.theHttpcPool.returnObject(theHttpc); + } catch (Exception e) { + // we could ignore this error + } + } + protected void finalize() throws Throwable { + System.err.println("Httpc object was not returned to object pool."); + this.reset(); + httpc.theHttpcPool.invalidateObject(this); + } + public static String dnsResolve(String host) { // looks for the ip of host and returns ip number as string String ip = (String) nameCacheHit.get(host); @@ -134,45 +208,72 @@ public class httpc { return false; } } + + void reset() { + try { + if (this.clientInput != null) { + this.clientInput.close(); + this.clientInput = null; + } + if (this.clientOutput != null) { + this.clientOutput.close(); + this.clientOutput = null; + } + if (this.socket != null) { + this.socket.close(); + this.socket = null; + } + + this.host = null; + this.timeout = 0; + this.handle = 0; + + this.remoteProxyUse = false; + this.savedRemoteHost = null; + this.requestPath = null; + } catch (Exception e) { + // we could ignore this ... + } + } // http client - public httpc(String server, int port, int timeout, boolean ssl, + void init(String server, int port, int timeout, boolean ssl, String remoteProxyHost, int remoteProxyPort) throws IOException { - this(remoteProxyHost, remoteProxyPort, timeout, ssl); - this.remoteProxyUse = true; - this.savedRemoteHost = server + ((port == 80) ? "" : (":" + port)); + this.init(remoteProxyHost, remoteProxyPort, timeout, ssl); + this.remoteProxyUse = true; + this.savedRemoteHost = server + ((port == 80) ? "" : (":" + port)); } - public httpc(String server, int port, int timeout, boolean ssl) throws IOException { + void init(String server, int port, int timeout, boolean ssl) throws IOException { handle = System.currentTimeMillis(); //serverLog.logDebug("HTTPC", handle + " initialized"); - this.remoteProxyUse = false; - this.timeout = timeout; - this.savedRemoteHost = server; - try { - this.host = server + ((port == 80) ? "" : (":" + port)); - String hostip; - if ((server.equals("localhost")) || (server.equals("127.0.0.1")) || (server.startsWith("192.168.")) || (server.startsWith("10."))) { - hostip = server; - } else { - hostip = dnsResolve(server); - if (hostip == null) throw new UnknownHostException(server); - } - if (ssl) - socket = SSLSocketFactory.getDefault().createSocket(hostip, port); - else - socket = new Socket(hostip, port); - socket.setSoTimeout(timeout); // waiting time for write - //socket.setSoLinger(true, timeout); // waiting time for read - socket.setKeepAlive(true); // - clientInput = new PushbackInputStream(socket.getInputStream()); - clientOutput = socket.getOutputStream(); - // if we reached this point, we should have a connection - } catch (UnknownHostException e) { - throw new IOException("unknown host: " + server); - } + this.remoteProxyUse = false; + this.timeout = timeout; + this.savedRemoteHost = server; + try { + this.host = server + ((port == 80) ? "" : (":" + port)); + String hostip; + if ((server.equals("localhost")) || (server.equals("127.0.0.1")) || (server.startsWith("192.168.")) || (server.startsWith("10."))) { + hostip = server; + } else { + hostip = dnsResolve(server); + if (hostip == null) throw new UnknownHostException(server); + } + if (ssl) + socket = SSLSocketFactory.getDefault().createSocket(hostip, port); + else + socket = new Socket(hostip, port); + socket.setSoTimeout(timeout); // waiting time for write + //socket.setSoLinger(true, timeout); // waiting time for read + socket.setKeepAlive(true); // + clientInput = new PushbackInputStream(socket.getInputStream()); + clientOutput = socket.getOutputStream(); + // if we reached this point, we should have a connection + } catch (UnknownHostException e) { + throw new IOException("unknown host: " + server); + } } // provide HTTP date handling static methods @@ -230,7 +331,7 @@ public class httpc { } // reads in the http header, right now, right here - byte[] b = serverCore.receive(clientInput, timeout, terminalMaxLength, false); + byte[] b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false); if (b == null) { // the server has meanwhile disconnected status = "503 server has closed connection"; @@ -242,7 +343,7 @@ public class httpc { if (p < 0) { status = "500 status line parse error"; // flush in anything that comes without parsing - while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)).length != 0) {} + while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)).length != 0) {} return; // in bad mood } // we have a status @@ -252,14 +353,14 @@ public class httpc { if (status.startsWith("400")) { // bad request // flush in anything that comes without parsing - while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)).length != 0) {} + while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)).length != 0) {} return; // in bad mood } // at this point we should have a valid response. read in the header properties String key = ""; String value = ""; - while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)) != null) { + while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)) != null) { if (b.length == 0) break; buffer = new String(b); //System.out.println("#H#" + buffer); // debug @@ -727,24 +828,35 @@ do upload String user, String password, boolean ssl, String proxyHost, int proxyPort, httpHeader requestHeader) throws IOException { - if (requestHeader == null) requestHeader = new httpHeader(); - if ((user != null) && (password != null) && (user.length() != 0)) { - requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); - } - httpc con; - if ((proxyHost == null) || (proxyPort == 0)) - con = new httpc(host, port, timeout, ssl); - else - con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort); - httpc.response res = con.GET(path, null); - if (res.status.startsWith("2")) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - res.writeContent(bos, null); - con.close(); - return bos.toByteArray(); - } else { - return res.status.getBytes(); - } + if (requestHeader == null) requestHeader = new httpHeader(); + if ((user != null) && (password != null) && (user.length() != 0)) { + requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); + } + + httpc con = null; + try { + + if ((proxyHost == null) || (proxyPort == 0)) { + con = httpc.getInstance(host, port, timeout, ssl); + } else { + con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort); + } + + httpc.response res = con.GET(path, null); + if (res.status.startsWith("2")) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + res.writeContent(bos, null); + con.close(); + return bos.toByteArray(); + } else { + return res.status.getBytes(); + } + } catch (Exception e) { + throw new IOException(e.getMessage()); + } finally { + if (con != null) httpc.returnInstance(con); + } + } public static byte[] singleGET(URL u, int timeout, @@ -773,25 +885,35 @@ do upload String user, String password, boolean ssl, String proxyHost, int proxyPort, httpHeader requestHeader, serverObjects props) throws IOException { - if (requestHeader == null) requestHeader = new httpHeader(); - if ((user != null) && (password != null) && (user.length() != 0)) { - requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); - } - httpc con; - if ((proxyHost == null) || (proxyPort == 0)) - con = new httpc(host, port, timeout, ssl); - else - con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort); - httpc.response res = con.POST(path, null, props, null); - //System.out.println("response=" + res.toString()); - if (res.status.startsWith("2")) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - res.writeContent(bos, null); - con.close(); - return bos.toByteArray(); - } else { - return res.status.getBytes(); - } + + if (requestHeader == null) requestHeader = new httpHeader(); + if ((user != null) && (password != null) && (user.length() != 0)) { + requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); + } + + httpc con = null; + try { + if ((proxyHost == null) || (proxyPort == 0)) + con = httpc.getInstance(host, port, timeout, ssl); + else + con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort); + httpc.response res = con.POST(path, null, props, null); + + //System.out.println("response=" + res.toString()); + if (res.status.startsWith("2")) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + res.writeContent(bos, null); + con.close(); + return bos.toByteArray(); + } else { + return res.status.getBytes(); + } + } catch (Exception e) { + throw new IOException(e.getMessage()); + } finally { + if (con != null) httpc.returnInstance(con); + } + } public static byte[] singlePOST(URL u, int timeout, @@ -831,33 +953,41 @@ do upload } public static httpHeader whead(URL url, int timeout, String user, String password, String proxyHost, int proxyPort) throws IOException { - // generate request header + // generate request header httpHeader requestHeader = new httpHeader(); - if ((user != null) && (password != null) && (user.length() != 0)) { - requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); - } + if ((user != null) && (password != null) && (user.length() != 0)) { + requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password)); + } // parse query + int port = url.getPort(); boolean ssl = url.getProtocol().equals("https"); - if (port < 0) port = (ssl) ? 443 : 80; - String path = url.getPath(); - String query = url.getQuery(); - if ((query != null) && (query.length() > 0)) path = path + "?" + query; - String host = url.getHost(); + if (port < 0) port = (ssl) ? 443 : 80; + String path = url.getPath(); + String query = url.getQuery(); + if ((query != null) && (query.length() > 0)) path = path + "?" + query; + String host = url.getHost(); + // start connection - httpc con; - if ((proxyHost == null) || (proxyPort == 0)) - con = new httpc(host, port, timeout, ssl); - else - con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort); - httpc.response res = con.HEAD(path, requestHeader); - if (res.status.startsWith("2")) { - // success - return res.responseHeader; - } else { - // fail - return res.responseHeader; - } + httpc con = null; + try { + if ((proxyHost == null) || (proxyPort == 0)) + con = httpc.getInstance(host, port, timeout, ssl); + else con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort); + + httpc.response res = con.HEAD(path, requestHeader); + if (res.status.startsWith("2")) { + // success + return res.responseHeader; + } else { + // fail + return res.responseHeader; + } + } catch (Exception e) { + throw new IOException(e.getMessage()); + } finally { + if (con != null) httpc.returnInstance(con); + } } /* @@ -929,6 +1059,9 @@ do upload Enumeration i = text.elements(); while (i.hasMoreElements()) System.out.println((String) i.nextElement()); } + + + } @@ -1027,3 +1160,90 @@ public class SSLSocketClientWithClientAuth { } } */ + +final class httpcFactory implements org.apache.commons.pool.PoolableObjectFactory { + + public httpcFactory() { + super(); + } + + /** + * @see org.apache.commons.pool.PoolableObjectFactory#makeObject() + */ + public Object makeObject() throws Exception { + return new httpc(); + } + + /** + * @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object) + */ + public void destroyObject(Object obj) { + if (obj instanceof httpc) { + httpc theHttpc = (httpc) obj; + } + } + + /** + * @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object) + */ + public boolean validateObject(Object obj) { + if (obj instanceof httpc) + { + httpc theHttpc = (httpc) obj; + return true; + } + return true; + } + + /** + * @param obj + * + */ + public void activateObject(Object obj) { + //log.debug(" activateObject..."); + } + + /** + * @param obj + * + */ + public void passivateObject(Object obj) { + //log.debug(" passivateObject..." + obj); + if (obj instanceof Session) { + httpc theHttpc = (httpc) obj; + } + } +} + +final class httpcPool extends GenericObjectPool { + /** + * First constructor. + * @param objFactory + */ + public httpcPool(httpcFactory objFactory) { + super(objFactory); + this.setMaxIdle(75); // Maximum idle threads. + this.setMaxActive(150); // Maximum active threads. + this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs. + //this.setMaxWait(1000); // Wait 1 second till a thread is available + } + + public httpcPool(httpcFactory objFactory, + GenericObjectPool.Config config) { + super(objFactory, config); + } + + /** + * @see org.apache.commons.pool.impl.GenericObjectPool#borrowObject() + */ + public Object borrowObject() throws Exception { + return super.borrowObject(); + } + + /** + * @see org.apache.commons.pool.impl.GenericObjectPool#returnObject(java.lang.Object) + */ + public void returnObject(Object obj) throws Exception { + super.returnObject(obj); + } +} diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index b092e7152..4e8e87c0e 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -53,9 +53,15 @@ import java.io.*; import java.net.*; import java.util.*; import java.text.*; + +import org.apache.commons.pool.impl.GenericObjectPool; + import de.anomic.server.*; +import de.anomic.server.serverCore.Session; +import de.anomic.server.serverCore.SessionFactory; +import de.anomic.server.serverCore.SessionPool; -public class httpd implements serverHandler { +public final class httpd implements serverHandler { // static objects public static final String vDATE = "<>"; @@ -77,15 +83,19 @@ public class httpd implements serverHandler { private String proxyAccountBase64MD5; private String serverAccountBase64MD5; private String clientIP; + + // the connection properties + private static final Properties prop = new Properties(); + + // class methods - public httpd(serverSwitch s, httpdHandler fileHandler, httpdHandler proxyHandler) { // handler info - this.switchboard = s; - this.fileHandler = fileHandler; - this.proxyHandler = proxyHandler; - this.virtualHost = switchboard.getConfig("fileHost","localhost"); + httpd.switchboard = s; + httpd.fileHandler = fileHandler; + httpd.proxyHandler = proxyHandler; + httpd.virtualHost = switchboard.getConfig("fileHost","localhost"); // authentication: by default none this.proxyAccountBase64MD5 = null; @@ -93,99 +103,109 @@ public class httpd implements serverHandler { this.clientIP = null; } + public void reset() { + this.session = null; + this.userAddress = null; + this.allowProxy = false; + this.allowServer = false; + this.proxyAccountBase64MD5 = null; + this.serverAccountBase64MD5 = null; + this.clientIP = null; + } + // must be called at least once, but can be called again to re-use the object. public void initSession(serverCore.Session session) throws IOException { this.session = session; this.userAddress = session.userAddress; // client InetAddress this.clientIP = userAddress.getHostAddress(); if (this.userAddress.isAnyLocalAddress()) this.clientIP = "localhost"; - if (this.clientIP.equals("0:0:0:0:0:0:0:1")) this.clientIP = "localhost"; + if (this.clientIP.equals("0:0:0:0:0:0:0:1")) this.clientIP = "localhost"; if (this.clientIP.equals("127.0.0.1")) this.clientIP = "localhost"; - String proxyClient = switchboard.getConfig("proxyClient", "*"); - String serverClient = switchboard.getConfig("serverClient", "*"); + String proxyClient = switchboard.getConfig("proxyClient", "*"); + String serverClient = switchboard.getConfig("serverClient", "*"); this.allowProxy = (proxyClient.equals("*")) ? true : match(clientIP, proxyClient); - this.allowServer = (serverClient.equals("*")) ? true : match(clientIP, serverClient); - - // check if we want to allow this socket to connect us - if (!((allowProxy) || (allowServer))) { - throw new IOException("CONNECTION FROM " + clientIP + " FORBIDDEN"); - } + this.allowServer = (serverClient.equals("*")) ? true : match(clientIP, serverClient); + + // check if we want to allow this socket to connect us + if (!((allowProxy) || (allowServer))) { + throw new IOException("CONNECTION FROM " + clientIP + " FORBIDDEN"); + } proxyAccountBase64MD5 = null; - serverAccountBase64MD5 = null; + serverAccountBase64MD5 = null; } private static boolean match(String key, String latch) { - // the latch is a comma-separated list of patterns - // each pattern may contain one wildcard-character '*' which matches anything - StringTokenizer st = new StringTokenizer(latch,","); - String pattern; - int pos; - while (st.hasMoreTokens()) { - pattern = st.nextToken(); - pos = pattern.indexOf("*"); - if (pos < 0) { - // no wild card: exact match - if (key.equals(pattern)) return true; - } else { - // wild card: match left and right side of pattern - if ((key.startsWith(pattern.substring(0, pos))) && - (key.endsWith(pattern.substring(pos + 1)))) return true; - } - } - return false; + // the latch is a comma-separated list of patterns + // each pattern may contain one wildcard-character '*' which matches anything + StringTokenizer st = new StringTokenizer(latch,","); + String pattern; + int pos; + while (st.hasMoreTokens()) { + pattern = st.nextToken(); + pos = pattern.indexOf("*"); + if (pos < 0) { + // no wild card: exact match + if (key.equals(pattern)) return true; + } else { + // wild card: match left and right side of pattern + if ((key.startsWith(pattern.substring(0, pos))) && + (key.endsWith(pattern.substring(pos + 1)))) return true; + } + } + return false; } public String greeting() { // OBLIGATORIC FUNCTION - // a response line upon connection is send to client - // if no response line is wanted, return "" or null - return null; + // a response line upon connection is send to client + // if no response line is wanted, return "" or null + return null; } public String error(Throwable e) { // OBLIGATORIC FUNCTION - // return string in case of any error that occurs during communication - // is always (but not only) called if an IO-dependent exception occurrs. - e.printStackTrace(); - return "501 Exception occurred: " + e.getMessage(); + // return string in case of any error that occurs during communication + // is always (but not only) called if an IO-dependent exception occurrs. + e.printStackTrace(); + return "501 Exception occurred: " + e.getMessage(); } private String readLine() { - // reads a line from the input socket - // this function is provided by the server through a passed method on initialization - byte[] l = this.session.readLine(); - if (l == null) return null; else return new String(l); + // reads a line from the input socket + // this function is provided by the server through a passed method on initialization + byte[] l = this.session.readLine(); + if (l == null) return null; else return new String(l); } private httpHeader readHeader() { - httpHeader header = new httpHeader(reverseMappingCache); - int p; - String line; - String key; - String value; - while ((line = readLine()) != null) { - if (line.length() == 0) break; // this seperates the header of the HTTP request from the body - //System.out.println("***" + line); // debug - // parse the header line: a property seperated with the ':' sign - p = line.indexOf(":"); - if (p >= 0) { - // store a property - key = line.substring(0, p).trim(); - value = (String) header.get(key); - // check if the header occurred already - if (value == null) { - // create new entry - header.put(key, line.substring(p + 1).trim()); - } else { - // value can occur double times, attach with '#' - separator - header.put(key, value + "#" + line.substring(p + 1).trim()); - } - } - } - return header; + httpHeader header = new httpHeader(reverseMappingCache); + int p; + String line; + String key; + String value; + while ((line = readLine()) != null) { + if (line.length() == 0) break; // this seperates the header of the HTTP request from the body + //System.out.println("***" + line); // debug + // parse the header line: a property seperated with the ':' sign + p = line.indexOf(":"); + if (p >= 0) { + // store a property + key = line.substring(0, p).trim(); + value = (String) header.get(key); + // check if the header occurred already + if (value == null) { + // create new entry + header.put(key, line.substring(p + 1).trim()); + } else { + // value can occur double times, attach with '#' - separator + header.put(key, value + "#" + line.substring(p + 1).trim()); + } + } + } + return header; } public Boolean GET(String arg) throws IOException { - Properties prop = parseQuery(arg); + parseQuery(prop, arg); prop.setProperty("METHOD", "GET"); prop.setProperty("CLIENTIP", clientIP); @@ -268,7 +288,7 @@ public class httpd implements serverHandler { } public Boolean HEAD(String arg) throws IOException { - Properties prop = parseQuery(arg); + parseQuery(prop,arg); prop.setProperty("METHOD", "HEAD"); prop.setProperty("CLIENTIP", clientIP); @@ -342,7 +362,7 @@ public class httpd implements serverHandler { } public Boolean POST(String arg) throws IOException { - Properties prop = parseQuery(arg); + parseQuery(prop, arg); prop.setProperty("METHOD", "POST"); prop.setProperty("CLIENTIP", clientIP); @@ -487,8 +507,13 @@ public class httpd implements serverHandler { } - private Properties parseQuery(String s) { - Properties prop = new Properties(); + private static final Properties parseQuery(Properties prop, String s) { + + if (prop == null) { + prop = new Properties(); + } else { + prop.clear(); + } // this parses a whole URL if (s.length() == 0) { @@ -819,7 +844,6 @@ permission if (pos < 0) return false; return whitelist.contains(mime.substring(0, pos)); } - } /* diff --git a/source/de/anomic/http/httpdAbstractHandler.java b/source/de/anomic/http/httpdAbstractHandler.java index 1f1c6be91..3a459f3fa 100644 --- a/source/de/anomic/http/httpdAbstractHandler.java +++ b/source/de/anomic/http/httpdAbstractHandler.java @@ -51,11 +51,9 @@ package de.anomic.http; -import java.io.*; -import java.util.*; import java.text.*; -public abstract class httpdAbstractHandler { +abstract class httpdAbstractHandler { // static tools diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index 4123ab756..7d38874a1 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -76,7 +76,6 @@ package de.anomic.http; import java.io.*; import java.util.*; -import java.text.*; import java.lang.reflect.*; import de.anomic.server.*; diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 60e9fcb47..4611a1647 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -62,16 +62,13 @@ package de.anomic.http; import java.io.*; import java.net.*; import java.util.*; -import java.text.*; import de.anomic.htmlFilter.*; import de.anomic.server.*; -import de.anomic.tools.*; import de.anomic.yacy.*; -import de.anomic.http.*; import de.anomic.plasma.*; -public class httpdProxyHandler extends httpdAbstractHandler implements httpdHandler { +public final class httpdProxyHandler extends httpdAbstractHandler implements httpdHandler { // static variables // can only be instantiated upon first instantiation of this class object @@ -87,10 +84,10 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand public static int remoteProxyPort = -1; public static String remoteProxyNoProxy = ""; public static String[] remoteProxyNoProxyPatterns = null; - private static HashSet remoteProxyAllowProxySet = new HashSet(); - private static HashSet remoteProxyDisallowProxySet = new HashSet(); + private static final HashSet remoteProxyAllowProxySet = new HashSet(); + private static final HashSet remoteProxyDisallowProxySet = new HashSet(); private static htmlFilterTransformer transformer = null; - public static String userAgent = "yacy (" + httpc.systemOST +") yacy.net"; + public static final String userAgent = "yacy (" + httpc.systemOST +") yacy.net"; private File htRootPath = null; // class methods @@ -108,8 +105,6 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand } remoteProxyUse = switchboard.getConfig("remoteProxyUse","false").equals("true"); remoteProxyNoProxy = switchboard.getConfig("remoteProxyNoProxy",""); - remoteProxyAllowProxySet = new HashSet(); - remoteProxyDisallowProxySet = new HashSet(); remoteProxyNoProxyPatterns = remoteProxyNoProxy.split(","); // set loglevel @@ -586,6 +581,8 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand respond.write(("]\r\n").getBytes()); } } catch (Exception ee) {} + } finally { + if (remote != null) httpc.returnInstance(remote); } respond.flush(); } @@ -686,7 +683,10 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand e.printStackTrace(new PrintStream(respond)); respond.write(("]\r\n").getBytes()); } catch (Exception ee) {} - } + } finally { + if (remote != null) httpc.returnInstance(remote); + } + respond.flush(); } @@ -740,71 +740,80 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand e.printStackTrace(new PrintStream(respond)); respond.write(("]\r\n").getBytes()); } catch (Exception ee) {} - } + } finally { + if (remote != null) httpc.returnInstance(remote); + } respond.flush(); } public void doConnect(Properties conProp, de.anomic.http.httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) throws IOException { String host = conProp.getProperty("HOST"); - int port = Integer.parseInt(conProp.getProperty("PORT")); - String httpVersion = conProp.getProperty("HTTP"); - int timeout = Integer.parseInt(switchboard.getConfig("clientTimeout", "10000")); - - // possibly branch into PROXY-PROXY connection - if (remoteProxyUse) { - httpc remoteProxy = new httpc(host, port, timeout, false, remoteProxyHost, remoteProxyPort); - httpc.response response = remoteProxy.CONNECT(host, port, requestHeader); - response.print(); - if (response.success()) { - // replace connection details - host = remoteProxyHost; - port = remoteProxyPort; - // go on (see below) - } else { - // pass error response back to client - respondHeader(clientOut, response.status, response.responseHeader); - return; - } - } - - // try to establish connection to remote host - Socket sslSocket = new Socket(host, port); - sslSocket.setSoTimeout(timeout); // waiting time for write - sslSocket.setSoLinger(true, timeout); // waiting time for read - InputStream promiscuousIn = sslSocket.getInputStream(); - OutputStream promiscuousOut = sslSocket.getOutputStream(); - - // now then we can return a success message - clientOut.write((httpVersion + " 200 Connection established" + serverCore.crlfString + - "Proxy-agent: YACY" + serverCore.crlfString + - serverCore.crlfString).getBytes()); - - log.logInfo("SSL CONNECTION TO " + host + ":" + port + " ESTABLISHED"); - - // start stream passing with mediate processes - try { - Mediate cs = new Mediate(sslSocket, clientIn, promiscuousOut); - Mediate sc = new Mediate(sslSocket, promiscuousIn, clientOut); - cs.start(); - sc.start(); - while ((sslSocket != null) && - (sslSocket.isBound()) && - (!(sslSocket.isClosed())) && - (sslSocket.isConnected()) && - ((cs.isAlive()) || (sc.isAlive()))) { - // idle - try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait a while - } - // set stop mode - cs.pleaseTerminate(); - sc.pleaseTerminate(); - // wake up thread - cs.interrupt(); - sc.interrupt(); - // ...hope they have terminated... - } catch (IOException e) { - //System.out.println("promiscuous termination: " + e.getMessage()); - } + int port = Integer.parseInt(conProp.getProperty("PORT")); + String httpVersion = conProp.getProperty("HTTP"); + int timeout = Integer.parseInt(switchboard.getConfig("clientTimeout", "10000")); + + // possibly branch into PROXY-PROXY connection + if (remoteProxyUse) { + httpc remoteProxy = null; + try { + remoteProxy = httpc.getInstance(host, port, timeout, false, remoteProxyHost, remoteProxyPort); + httpc.response response = remoteProxy.CONNECT(host, port, requestHeader); + response.print(); + if (response.success()) { + // replace connection details + host = remoteProxyHost; + port = remoteProxyPort; + // go on (see below) + } else { + // pass error response back to client + respondHeader(clientOut, response.status, response.responseHeader); + return; + } + } catch (Exception e) { + throw new IOException(e.getMessage()); + } finally { + if (remoteProxy != null) httpc.returnInstance(remoteProxy); + } + } + + // try to establish connection to remote host + Socket sslSocket = new Socket(host, port); + sslSocket.setSoTimeout(timeout); // waiting time for write + sslSocket.setSoLinger(true, timeout); // waiting time for read + InputStream promiscuousIn = sslSocket.getInputStream(); + OutputStream promiscuousOut = sslSocket.getOutputStream(); + + // now then we can return a success message + clientOut.write((httpVersion + " 200 Connection established" + serverCore.crlfString + + "Proxy-agent: YACY" + serverCore.crlfString + + serverCore.crlfString).getBytes()); + + log.logInfo("SSL CONNECTION TO " + host + ":" + port + " ESTABLISHED"); + + // start stream passing with mediate processes + try { + Mediate cs = new Mediate(sslSocket, clientIn, promiscuousOut); + Mediate sc = new Mediate(sslSocket, promiscuousIn, clientOut); + cs.start(); + sc.start(); + while ((sslSocket != null) && + (sslSocket.isBound()) && + (!(sslSocket.isClosed())) && + (sslSocket.isConnected()) && + ((cs.isAlive()) || (sc.isAlive()))) { + // idle + try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait a while + } + // set stop mode + cs.pleaseTerminate(); + sc.pleaseTerminate(); + // wake up thread + cs.interrupt(); + sc.interrupt(); + // ...hope they have terminated... + } catch (IOException e) { + //System.out.println("promiscuous termination: " + e.getMessage()); + } } @@ -873,9 +882,9 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand } // branch to server/proxy if (useProxy) { - return new httpc(server, port, timeout, false, remoteProxyHost, remoteProxyPort); + return httpc.getInstance(server, port, timeout, false, remoteProxyHost, remoteProxyPort); } else { - return new httpc(server, port, timeout, false); + return httpc.getInstance(server, port, timeout, false); } } @@ -895,15 +904,18 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand } private void respondHeader(OutputStream respond, String status, httpHeader header) throws IOException, SocketException { - String s; // prepare header //header.put("Server", "AnomicHTTPD (www.anomic.de)"); if (!(header.containsKey("date"))) header.put("Date", httpc.dateString(httpc.nowDate())); if (!(header.containsKey("content-type"))) header.put("Content-type", "text/html"); // fix this + StringBuffer headerStringBuffer = new StringBuffer(); + // write status line - respond.write(("HTTP/1.1 " + status + "\r\n").getBytes()); + headerStringBuffer.append("HTTP/1.1 ") + .append(status) + .append("\r\n"); //System.out.println("HEADER: PROXY TO CLIENT = " + header.toString()); // DEBUG @@ -920,17 +932,27 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand if (!(key.equals("Location"))) while ((pos = value.lastIndexOf("#")) >= 0) { // special handling is needed if a key appeared several times, which is valid. // all lines with same key are combined in one value, separated by a "#" - respond.write((key + ": " + value.substring(pos + 1).trim() + "\r\n").getBytes()); + headerStringBuffer + .append(key) + .append(": ") + .append(value.substring(pos + 1).trim()) + .append("\r\n"); //System.out.println("#" + key + ": " + value.substring(pos + 1).trim()); value = value.substring(0, pos).trim(); } - respond.write((key + ": " + value + "\r\n").getBytes()); + headerStringBuffer + .append(key) + .append(": ") + .append(value) + .append("\r\n"); //System.out.println("#" + key + ": " + value); } } + headerStringBuffer.append("\r\n"); + // end header - respond.write(("\r\n").getBytes()); + respond.write(headerStringBuffer.toString().getBytes()); respond.flush(); } diff --git a/source/de/anomic/http/httpdSwitchboard.java b/source/de/anomic/http/httpdSwitchboard.java index e36e58ea9..55f20cae9 100644 --- a/source/de/anomic/http/httpdSwitchboard.java +++ b/source/de/anomic/http/httpdSwitchboard.java @@ -44,9 +44,9 @@ import java.io.*; import java.util.*; import de.anomic.server.*; -public class httpdSwitchboard extends serverAbstractSwitch implements serverSwitch { +public final class httpdSwitchboard extends serverAbstractSwitch implements serverSwitch { - private LinkedList cacheStack; + private final LinkedList cacheStack; public httpdSwitchboard(String rootPath, String initPath, String configPath) throws IOException { super(rootPath, initPath, configPath); diff --git a/source/de/anomic/plasma/plasmaCrawlLoader.java b/source/de/anomic/plasma/plasmaCrawlLoader.java index 5fbbc4ceb..c1dcaf58b 100644 --- a/source/de/anomic/plasma/plasmaCrawlLoader.java +++ b/source/de/anomic/plasma/plasmaCrawlLoader.java @@ -50,7 +50,7 @@ import de.anomic.server.*; import de.anomic.tools.*; import de.anomic.htmlFilter.*; -public class plasmaCrawlLoader { +public final class plasmaCrawlLoader { private plasmaHTCache cacheManager; private int socketTimeout; @@ -130,7 +130,7 @@ public class plasmaCrawlLoader { return result; } - public class Exec extends Thread { + public final class Exec extends Thread { public URL url; public String referer; @@ -160,9 +160,8 @@ public class plasmaCrawlLoader { private httpc newhttpc(String server, int port, boolean ssl) throws IOException { // a new httpc connection, combined with possible remote proxy if (remoteProxyUse) - return new httpc(server, port, socketTimeout, ssl, remoteProxyHost, remoteProxyPort); - else - return new httpc(server, port, socketTimeout, ssl); + return httpc.getInstance(server, port, socketTimeout, ssl, remoteProxyHost, remoteProxyPort); + else return httpc.getInstance(server, port, socketTimeout, ssl); } private void load(URL url, String referer, String initiator, int depth, plasmaCrawlProfile.entry profile) throws IOException { @@ -179,20 +178,21 @@ public class plasmaCrawlLoader { if (referer.length() == 0) referer = "http://www.yacy.net/yacy/"; // take a file from the net + httpc remote = null; try { - // create a request header - httpHeader requestHeader = new httpHeader(); - requestHeader.put("User-Agent", httpdProxyHandler.userAgent); - requestHeader.put("Referer", referer); - requestHeader.put("Accept-Encoding", "gzip,deflate"); - - //System.out.println("CRAWLER_REQUEST_HEADER=" + requestHeader.toString()); // DEBUG - - // open the connection - httpc remote = newhttpc(host, port, ssl); - - // send request - httpc.response res = remote.GET(path, requestHeader); + // create a request header + httpHeader requestHeader = new httpHeader(); + requestHeader.put("User-Agent", httpdProxyHandler.userAgent); + requestHeader.put("Referer", referer); + requestHeader.put("Accept-Encoding", "gzip,deflate"); + + //System.out.println("CRAWLER_REQUEST_HEADER=" + requestHeader.toString()); // DEBUG + + // open the connection + remote = newhttpc(host, port, ssl); + + // send request + httpc.response res = remote.GET(path, requestHeader); if (res.status.startsWith("200")) { // the transfer is ok @@ -250,6 +250,8 @@ public class plasmaCrawlLoader { // remote server was wrong. log.logError("CRAWLER LOADER ERROR2 with url=" + url.toString() + ": " + e.toString()); e.printStackTrace(); + } finally { + if (remote != null) httpc.returnInstance(remote); } } diff --git a/source/de/anomic/server/serverAbstractSwitch.java b/source/de/anomic/server/serverAbstractSwitch.java index 2deedd86a..25aeb2756 100644 --- a/source/de/anomic/server/serverAbstractSwitch.java +++ b/source/de/anomic/server/serverAbstractSwitch.java @@ -47,12 +47,12 @@ import java.util.*; public abstract class serverAbstractSwitch implements serverSwitch { // configuration management - private File configFile; + private final File configFile; private Hashtable configProps; - private String configComment; - private Hashtable authorization; + private final String configComment; + private final Hashtable authorization; private String rootPath; - private TreeMap workerThreads; + private final TreeMap workerThreads; public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException { // we initialize the switchboard with a property file, diff --git a/source/de/anomic/server/serverAbstractThread.java b/source/de/anomic/server/serverAbstractThread.java index 88d45e3e0..c405a5110 100644 --- a/source/de/anomic/server/serverAbstractThread.java +++ b/source/de/anomic/server/serverAbstractThread.java @@ -58,14 +58,14 @@ public abstract class serverAbstractThread extends Thread implements serverThrea private long threadBlockTimestamp = System.currentTimeMillis(); private long idleCycles = 0, busyCycles = 0; - protected void announceThreadBlockApply() { + protected final void announceThreadBlockApply() { // shall only be used, if a thread blocks for an important reason // like a socket connect and must renew the timestamp to correct // statistics this.threadBlockTimestamp = System.currentTimeMillis(); } - protected void announceThreadBlockRelease() { + protected final void announceThreadBlockRelease() { // shall only be used, if a thread blocks for an important reason // like a socket connect and must renew the timestamp to correct // statistics @@ -74,69 +74,69 @@ public abstract class serverAbstractThread extends Thread implements serverThrea this.busytime -= thisBlockTime; } - protected void announceMoreExecTime(long millis) { + protected final void announceMoreExecTime(long millis) { this.busytime += millis; } - protected void announceMoreSleepTime(long millis) { + protected final void announceMoreSleepTime(long millis) { this.idletime += millis; } - public void setDescription(String shortText, String longText) { + public final void setDescription(String shortText, String longText) { // sets a visible description string this.shortDescr = shortText; this.longDescr = longText; } - public void setStartupSleep(long milliseconds) { + public final void setStartupSleep(long milliseconds) { // sets a sleep time before execution of the job-loop startup = milliseconds; } - public void setIdleSleep(long milliseconds) { + public final void setIdleSleep(long milliseconds) { // sets a sleep time for pauses between two jobs idlePause = milliseconds; } - public void setBusySleep(long milliseconds) { + public final void setBusySleep(long milliseconds) { // sets a sleep time for pauses between two jobs busyPause = milliseconds; } - public String getShortDescription() { + public final String getShortDescription() { return this.shortDescr; } - public String getLongDescription() { + public final String getLongDescription() { return this.longDescr; } - public long getIdleCycles() { + public final long getIdleCycles() { // returns the total number of cycles of job execution with idle-result return this.idleCycles; } - public long getBusyCycles() { + public final long getBusyCycles() { // returns the total number of cycles of job execution with busy-result return this.busyCycles; } - public long getBlockTime() { + public final long getBlockTime() { // returns the total time that this thread has been blocked so far return this.blockPause; } - public long getSleepTime() { + public final long getSleepTime() { // returns the total time that this thread has slept so far return this.idletime; } - public long getExecTime() { + public final long getExecTime() { // returns the total time that this thread has worked so far return this.busytime; } - public void setLog(serverLog log) { + public final void setLog(serverLog log) { // defines a log where process states can be written to this.log = log; } @@ -145,12 +145,15 @@ public abstract class serverAbstractThread extends Thread implements serverThrea // after calling this method, the thread shall terminate this.running = false; // wait for termination - if (waitFor) while (this.isAlive()) - try {this.sleep(100);} catch (InterruptedException e) {break;} + if (waitFor) { + // Busy waiting removed: while (this.isAlive()) try {this.sleep(100);} catch (InterruptedException e) {break;} + try { this.join(); } catch (InterruptedException e) {return;} + } + // If we reach this point, the process is closed } - private void logError(String text) { + private final void logError(String text) { if (log == null) serverLog.logError("THREAD-CONTROL", text); else diff --git a/source/de/anomic/server/serverByteBuffer.java b/source/de/anomic/server/serverByteBuffer.java index cc5b8b9cc..3ce4e258b 100644 --- a/source/de/anomic/server/serverByteBuffer.java +++ b/source/de/anomic/server/serverByteBuffer.java @@ -43,7 +43,7 @@ package de.anomic.server; import java.io.*; import java.util.*; -public class serverByteBuffer extends OutputStream { +public final class serverByteBuffer extends OutputStream { public static final byte singlequote = (byte) 39; public static final byte doublequote = (byte) 34; @@ -94,13 +94,14 @@ public class serverByteBuffer extends OutputStream { try { FileInputStream fis = new FileInputStream(f); - byte buf[] = new byte[512]; - int p = 0; +// byte buf[] = new byte[512]; +// int p = 0; int l; - while ((l = fis.read(buf)) > 0) { - System.arraycopy(buf, 0, buffer, p, l); - p += l; - } +// while ((l = fis.read(buf)) > 0) { +// System.arraycopy(buf, 0, buffer, p, l); +// p += l; + l = fis.read(buffer); +// } fis.close(); } catch (FileNotFoundException e) { throw new IOException("File not found: " + f.toString() + "; " + e.getMessage()); diff --git a/source/de/anomic/server/serverClassLoader.java b/source/de/anomic/server/serverClassLoader.java index c1de26f13..c090d185d 100644 --- a/source/de/anomic/server/serverClassLoader.java +++ b/source/de/anomic/server/serverClassLoader.java @@ -45,9 +45,9 @@ import java.io.*; import java.util.*; import java.lang.reflect.*; -public class serverClassLoader extends ClassLoader { +public final class serverClassLoader extends ClassLoader { - Hashtable classes; + private final Hashtable classes; public serverClassLoader() { super(ClassLoader.getSystemClassLoader()); diff --git a/source/de/anomic/server/serverCodings.java b/source/de/anomic/server/serverCodings.java index 4bd3e49b4..7bcabbb9d 100644 --- a/source/de/anomic/server/serverCodings.java +++ b/source/de/anomic/server/serverCodings.java @@ -44,7 +44,7 @@ import java.io.*; import java.security.*; -public class serverCodings { +public final class serverCodings { // this provides encoding and decoding of long cardinals into a 6-bit - based number format // expressed by a string. This is probably the most compact form to encode numbers as strings. diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 769d79a10..d7d3c603b 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -41,19 +41,30 @@ package de.anomic.server; // standard server -import java.io.*; -import java.net.*; -import java.lang.*; -import java.util.*; -import java.lang.reflect.*; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PushbackInputStream; +import java.lang.reflect.InvocationTargetException; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.net.UnknownHostException; + +import java.util.Hashtable; // needed for ssl import javax.net.*; import javax.net.ssl.*; import java.security.KeyStore; -import javax.security.cert.X509Certificate; -public class serverCore extends serverAbstractThread implements serverThread { +import org.apache.commons.pool.impl.GenericObjectPool; + +public final class serverCore extends serverAbstractThread implements serverThread { // generic input/output static methods public static final byte cr = 13; @@ -69,21 +80,29 @@ public class serverCore extends serverAbstractThread implements serverThread { private int port; // the listening port private ServerSocket socket; // listener private int maxSessions = 0; // max. number of sessions; 0=unlimited - private serverLog log; // log object + serverLog log; // log object //private serverSwitch switchboard; // external values private int timeout; // connection time-out of the socket - private Hashtable activeThreads; // contains the active threads - private Hashtable sleepingThreads; // contains the threads that are alive since the sleepthreashold + +// private Hashtable activeThreads; // contains the active threads +// private Hashtable sleepingThreads; // contains the threads that are alive since the sleepthreashold + private boolean termSleepingThreads; // if true then threads over sleepthreashold are killed private int thresholdActive = 5000; // after that time a thread should have got a command line private int thresholdSleep = 30000; // after that time a thread is considered as beeing sleeping (30 seconds) private int thresholdDead = 3600000; // after that time a thread is considered as beeing dead-locked (1 hour) - private serverHandler handlerPrototype;// the command class (a serverHandler) + serverHandler handlerPrototype;// the command class (a serverHandler) private Class[] initHandlerClasses; // the init's methods arguments private Class[] initSessionClasses; // the init's methods arguments private serverSwitch switchboard; // the command class switchboard private Hashtable denyHost; private int commandMaxLength; + + /** + * The session-object pool + */ + final SessionPool theSessionPool; + final ThreadGroup theSessionThreadGroup = new ThreadGroup("sessionThreadGroup"); private static ServerSocketFactory getServerSocketFactory(boolean dflt, File keyfile, String passphrase) { // see doc's at @@ -128,9 +147,9 @@ public class serverCore extends serverAbstractThread implements serverThread { boolean termSleepingThreads, boolean blockAttack, serverHandler handlerPrototype, serverSwitch switchboard, int commandMaxLength, int logl) throws IOException { - this.port = port; + this.port = port; this.commandMaxLength = commandMaxLength; - this.denyHost = (blockAttack) ? new Hashtable() : null; + this.denyHost = (blockAttack) ? new Hashtable() : null; /* try { @@ -142,27 +161,50 @@ public class serverCore extends serverAbstractThread implements serverThread { } */ - try { - this.socket = new ServerSocket(port); - } catch (java.net.BindException e) { - System.out.println("FATAL ERROR: " + e.getMessage() + " - probably root access rights needed. check port number"); System.exit(0); - } + try { + this.socket = new ServerSocket(port); + } catch (java.net.BindException e) { + System.out.println("FATAL ERROR: " + e.getMessage() + " - probably root access rights needed. check port number"); System.exit(0); + } try { - this.handlerPrototype = handlerPrototype; + this.handlerPrototype = handlerPrototype; this.switchboard = switchboard; - this.initHandlerClasses = new Class[] {Class.forName("de.anomic.server.serverSwitch")}; - this.initSessionClasses = new Class[] {Class.forName("de.anomic.server.serverCore$Session")}; - this.maxSessions = maxSessions; - this.socket.setSoTimeout(0); // unlimited - this.timeout = timeout; - this.termSleepingThreads = termSleepingThreads; + this.initHandlerClasses = new Class[] {Class.forName("de.anomic.server.serverSwitch")}; + this.initSessionClasses = new Class[] {Class.forName("de.anomic.server.serverCore$Session")}; + this.maxSessions = maxSessions; + this.timeout = timeout; + this.termSleepingThreads = termSleepingThreads; this.log = new serverLog("SERVER", logl); - activeThreads = new Hashtable(); - sleepingThreads = new Hashtable(); - } catch (java.lang.ClassNotFoundException e) { - System.out.println("FATAL ERROR: " + e.getMessage() + " - Class Not Found"); System.exit(0); - } +// activeThreads = new Hashtable(); +// sleepingThreads = new Hashtable(); + } catch (java.lang.ClassNotFoundException e) { + System.out.println("FATAL ERROR: " + e.getMessage() + " - Class Not Found"); System.exit(0); + } + + // implementation of session thread pool + GenericObjectPool.Config config = new GenericObjectPool.Config(); + + // The maximum number of active connections that can be allocated from pool at the same time, + // 0 for no limit + config.maxActive = this.maxSessions; + + // The maximum number of idle connections connections in the pool + // 0 = no limit. + config.maxIdle = this.maxSessions / 2; + config.minIdle = this.maxSessions / 4; + + // block undefinitely + config.maxWait = timeout; + + // Action to take in case of an exhausted DBCP statement pool + // 0 = fail, 1 = block, 2= grow + config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; + config.minEvictableIdleTimeMillis = this.thresholdSleep; + config.testOnReturn = true; + + this.theSessionPool = new SessionPool(new SessionFactory(this.theSessionThreadGroup),config); + } public static boolean isNotLocal(URL url) { @@ -234,12 +276,12 @@ public class serverCore extends serverAbstractThread implements serverThread { // class body public boolean job() throws Exception { // prepare for new connection - idleThreadCheck(); - switchboard.handleBusyState(activeThreads.size()); + // idleThreadCheck(); + this.switchboard.handleBusyState(this.theSessionPool.getNumActive() /*activeThreads.size() */); log.logDebug( - "* waiting for connections, " + activeThreads.size() + " sessions running, " + - sleepingThreads.size() + " sleeping"); + "* waiting for connections, " + this.theSessionPool.getNumActive() + " sessions running, " + + this.theSessionPool.getNumIdle() + " sleeping"); // list all connection (debug) /* @@ -257,17 +299,17 @@ public class serverCore extends serverAbstractThread implements serverThread { // wait for new connection announceThreadBlockApply(); - Socket controlSocket = socket.accept(); + Socket controlSocket = this.socket.accept(); announceThreadBlockRelease(); - if ((denyHost == null) || (denyHost.get((""+controlSocket.getInetAddress().getHostAddress())) == null)) { + if ((this.denyHost == null) || (this.denyHost.get((""+controlSocket.getInetAddress().getHostAddress())) == null)) { //log.logDebug("* catched request from " + controlSocket.getInetAddress().getHostAddress()); - controlSocket.setSoTimeout(timeout); + controlSocket.setSoTimeout(this.timeout); + + Session connection = (Session) this.theSessionPool.borrowObject(); + connection.execute(controlSocket); - Session connection = new Session(controlSocket); - // start the thread - connection.start(); //try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait for debug - activeThreads.put(connection, new Long(System.currentTimeMillis())); + // activeThreads.put(connection, new Long(System.currentTimeMillis())); //log.logDebug("* NEW SESSION: " + connection.request); } else { @@ -275,380 +317,663 @@ public class serverCore extends serverAbstractThread implements serverThread { } // idle until number of maximal threads is (again) reached //synchronized(this) { - while ((maxSessions > 0) && (activeThreads.size() >= maxSessions)) try { - log.logDebug("* Waiting for activeThreads=" + activeThreads.size() + " < maxSessions=" + maxSessions); - Thread.currentThread().sleep(2000); - idleThreadCheck(); - } catch (InterruptedException e) {} +// while ((maxSessions > 0) && (activeThreads.size() >= maxSessions)) try { +// log.logDebug("* Waiting for activeThreads=" + activeThreads.size() + " < maxSessions=" + maxSessions); +// Thread.currentThread().sleep(2000); +// idleThreadCheck(); +// } catch (InterruptedException e) {} return true; } public void close() { + try { + // consuming the isInterrupted Flag. Otherwise we could not properly colse the session pool + Thread.interrupted(); + + // close the session pool + this.theSessionPool.close(); + } + catch (Exception e) { + this.log.logSystem("Unable to close session pool: " + e.getMessage()); + } log.logSystem("* terminated"); } public int getJobCount() { - return activeThreads.size(); + return this.theSessionPool.getNumActive(); } // idle sensor: the thread is idle if there are no sessions running public boolean idle() { - idleThreadCheck(); - return (activeThreads.size() == 0); + // idleThreadCheck(); + return (this.theSessionPool.getNumActive() == 0); } - public void idleThreadCheck() { - // a 'garbage collector' for session threads - Enumeration threadEnum; - Session session; - - // look for sleeping threads - threadEnum = activeThreads.keys(); - long time; - while (threadEnum.hasMoreElements()) { - session = (Session) (threadEnum.nextElement()); - //if (session.request == null) session.interrupt(); - if (session.isAlive()) { - // check if socket still exists - time = System.currentTimeMillis() - ((Long) activeThreads.get(session)).longValue(); - if (/*(session.controlSocket.isClosed()) || */ - (!(session.controlSocket.isBound())) || - (!(session.controlSocket.isConnected())) || - ((session.request == null) && (time > 1000))) { - // kick it - try { - session.out.close(); - session.in.close(); - session.controlSocket.close(); - } catch (IOException e) {} - session.interrupt(); // hopefully this wakes him up. - activeThreads.remove(session); - String reason = ""; - if (session.controlSocket.isClosed()) reason = "control socked closed"; - if (!(session.controlSocket.isBound())) reason = "control socked unbound"; - if (!(session.controlSocket.isConnected())) reason = "control socked not connected"; - if (session.request == null) reason = "no request placed"; - log.logDebug("* canceled disconnected connection (" + reason + ") '" + session.request + "'"); - } else if (time > thresholdSleep) { - // move thread from the active threads to the sleeping - sleepingThreads.put(session, activeThreads.remove(session)); - log.logDebug("* sleeping connection '" + session.request + "'"); - } else if ((time > thresholdActive) && (session.request == null)) { - // thread is not in use (or too late). kickk it. - try { - session.out.close(); - session.in.close(); - session.controlSocket.close(); - } catch (IOException e) {} - session.interrupt(); // hopefully this wakes him up. - activeThreads.remove(session); - log.logDebug("* canceled inactivated connection"); - } - } else { - // the thread is dead, remove it - log.logDebug("* normal close of connection to '" + session.request + "', time=" + session.getTime()); - activeThreads.remove(session); - } - } +// public void idleThreadCheck() { +// // a 'garbage collector' for session threads +// Enumeration threadEnum; +// Session session; +// +// // look for sleeping threads +// threadEnum = activeThreads.keys(); +// long time; +// while (threadEnum.hasMoreElements()) { +// session = (Session) (threadEnum.nextElement()); +// //if (session.request == null) session.interrupt(); +// if (session.isAlive()) { +// // check if socket still exists +// time = System.currentTimeMillis() - ((Long) activeThreads.get(session)).longValue(); +// if (/*(session.controlSocket.isClosed()) || */ +// (!(session.controlSocket.isBound())) || +// (!(session.controlSocket.isConnected())) || +// ((session.request == null) && (time > 1000))) { +// // kick it +// try { +// session.out.close(); +// session.in.close(); +// session.controlSocket.close(); +// } catch (IOException e) {} +// session.interrupt(); // hopefully this wakes him up. +// activeThreads.remove(session); +// String reason = ""; +// if (session.controlSocket.isClosed()) reason = "control socked closed"; +// if (!(session.controlSocket.isBound())) reason = "control socked unbound"; +// if (!(session.controlSocket.isConnected())) reason = "control socked not connected"; +// if (session.request == null) reason = "no request placed"; +// log.logDebug("* canceled disconnected connection (" + reason + ") '" + session.request + "'"); +// } else if (time > thresholdSleep) { +// // move thread from the active threads to the sleeping +// sleepingThreads.put(session, activeThreads.remove(session)); +// log.logDebug("* sleeping connection '" + session.request + "'"); +// } else if ((time > thresholdActive) && (session.request == null)) { +// // thread is not in use (or too late). kickk it. +// try { +// session.out.close(); +// session.in.close(); +// session.controlSocket.close(); +// } catch (IOException e) {} +// session.interrupt(); // hopefully this wakes him up. +// activeThreads.remove(session); +// log.logDebug("* canceled inactivated connection"); +// } +// } else { +// // the thread is dead, remove it +// log.logDebug("* normal close of connection to '" + session.request + "', time=" + session.getTime()); +// activeThreads.remove(session); +// } +// } +// +// // look for dead threads +// threadEnum = sleepingThreads.keys(); +// while (threadEnum.hasMoreElements()) { +// session = (Session) (threadEnum.nextElement()); +// if (session.isAlive()) { +// // check the age of the thread +// if (System.currentTimeMillis() - ((Long) sleepingThreads.get(session)).longValue() > thresholdDead) { +// // kill the thread +// if (termSleepingThreads) { +// try { +// session.out.close(); +// session.in.close(); +// session.controlSocket.close(); +// } catch (IOException e) {} +// session.interrupt(); // hopefully this wakes him up. +// } +// sleepingThreads.remove(session); +// log.logDebug("* out-timed connection '" + session.request + "'"); +// } +// } else { +// // the thread is dead, remove it +// sleepingThreads.remove(session); +// log.logDebug("* dead connection '" + session.request + "'"); +// } +// } +// +// } - // look for dead threads - threadEnum = sleepingThreads.keys(); - while (threadEnum.hasMoreElements()) { - session = (Session) (threadEnum.nextElement()); - if (session.isAlive()) { - // check the age of the thread - if (System.currentTimeMillis() - ((Long) sleepingThreads.get(session)).longValue() > thresholdDead) { - // kill the thread - if (termSleepingThreads) { - try { - session.out.close(); - session.in.close(); - session.controlSocket.close(); - } catch (IOException e) {} - session.interrupt(); // hopefully this wakes him up. - } - sleepingThreads.remove(session); - log.logDebug("* out-timed connection '" + session.request + "'"); - } - } else { - // the thread is dead, remove it - sleepingThreads.remove(session); - log.logDebug("* dead connection '" + session.request + "'"); - } - } + + public final class SessionPool extends GenericObjectPool + { + public boolean isClosed = false; + + /** + * First constructor. + * @param objFactory + */ + public SessionPool(SessionFactory objFactory) { + super(objFactory); + this.setMaxIdle(75); // Maximum idle threads. + this.setMaxActive(150); // Maximum active threads. + this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs. + //this.setMaxWait(1000); // Wait 1 second till a thread is available + } + + public SessionPool(SessionFactory objFactory, + GenericObjectPool.Config config) { + super(objFactory, config); + } + + /** + * @see org.apache.commons.pool.impl.GenericObjectPool#borrowObject() + */ + public Object borrowObject() throws Exception { + return super.borrowObject(); + } + + /** + * @see org.apache.commons.pool.impl.GenericObjectPool#returnObject(java.lang.Object) + */ + public void returnObject(Object obj) throws Exception { + super.returnObject(obj); + } + + public synchronized void close() throws Exception { + /* + * shutdown all still running session threads ... + */ + // interrupting all still running or pooled threads ... + serverCore.this.theSessionThreadGroup.interrupt(); + + /* waiting for all threads to finish */ + int threadCount = serverCore.this.theSessionThreadGroup.activeCount(); + Thread[] threadList = new Thread[threadCount]; + threadCount = serverCore.this.theSessionThreadGroup.enumerate(threadList); + + try { + for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) { + // we need to use a timeout here because of missing interruptable session threads ... + threadList[currentThreadIdx].join(500); + } + } + catch (InterruptedException e) { + serverCore.this.log.logWarning("Interruption while trying to shutdown all session threads."); + } + finally { + this.isClosed = true; + } + + super.close(); + } + } + + public final class SessionFactory implements org.apache.commons.pool.PoolableObjectFactory { - public class Session extends Thread { + final ThreadGroup sessionThreadGroup; + public SessionFactory(ThreadGroup theSessionThreadGroup) { + super(); + + if (theSessionThreadGroup == null) + throw new IllegalArgumentException("The threadgroup object must not be null."); + + this.sessionThreadGroup = theSessionThreadGroup; + } + + /** + * @see org.apache.commons.pool.PoolableObjectFactory#makeObject() + */ + public Object makeObject() { + return new Session(this.sessionThreadGroup); + } + + /** + * @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object) + */ + public void destroyObject(Object obj) { + if (obj instanceof Session) { + Session theSession = (Session) obj; + theSession.setStopped(true); + } + } + + /** + * @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object) + */ + public boolean validateObject(Object obj) { + if (obj instanceof Session) + { + Session theSession = (Session) obj; + if (!theSession.isAlive() || theSession.isInterrupted()) return false; + if (theSession.isRunning()) return true; + return false; + } + return true; + } + + /** + * @param obj + * + */ + public void activateObject(Object obj) { + //log.debug(" activateObject..."); + } + + /** + * @param obj + * + */ + public void passivateObject(Object obj) { + //log.debug(" passivateObject..." + obj); + if (obj instanceof Session) { + Session theSession = (Session) obj; + + // Clean up the result of the execution + theSession.setResult(null); + } + } + } + + public final class Session extends Thread { + // used as replacement for activeThreads, sleepingThreads + // static ThreadGroup sessionThreadGroup = new ThreadGroup("sessionThreadGroup"); + + // synchronization object needed for the threadpool implementation + private Object syncObject; + + private Object processingResult = null; + + private boolean running = false; + private boolean stopped = false; + private boolean done = false; + + private long start; // startup time private serverHandler commandObj; - private String request; // current command line - private int commandCounter; // for logging: number of commands in this session - private String identity; // a string that identifies the client (i.e. ftp: account name) - //private boolean promiscuous; // if true, no lines are read and streams are only passed - public Socket controlSocket; // dialog socket - public InetAddress userAddress; // the address of the client - public PushbackInputStream in; // on control input stream - public OutputStream out; // on control output stream, autoflush - - public Session(Socket controlSocket) throws IOException { - //this.promiscuous = false; - this.start = System.currentTimeMillis(); - //log.logDebug("* session " + handle + " allocated"); - this.identity = "-"; - this.userAddress = controlSocket.getInetAddress(); - String ipname = userAddress.getHostAddress(); - // check if we want to allow this socket to connect us - this.controlSocket = controlSocket; - this.in = new PushbackInputStream(controlSocket.getInputStream()); - this.out = controlSocket.getOutputStream(); - commandCounter = 0; - // initiate the command class - // we pass the input and output stream to the commands, - // so that they can take over communication, if needed - try { - // use the handler prototype to create a new command object class - commandObj = (serverHandler) handlerPrototype.clone(); - commandObj.initSession(this); - } catch (Exception e) { - e.printStackTrace(); - } - //log.logDebug("* session " + handle + " initialized. time = " + (System.currentTimeMillis() - handle)); - } - + private String request; // current command line + private int commandCounter; // for logging: number of commands in this session + private String identity; // a string that identifies the client (i.e. ftp: account name) + //private boolean promiscuous; // if true, no lines are read and streams are only passed + public Socket controlSocket; // dialog socket + public InetAddress userAddress; // the address of the client + public PushbackInputStream in; // on control input stream + public OutputStream out; // on control output stream, autoflush + + private final ByteArrayOutputStream readLineBuffer = new ByteArrayOutputStream(256); + + public Session(ThreadGroup theThreadGroup) { + super(theThreadGroup,"Session"); + } + + public void setStopped(boolean stopped) { + this.stopped = stopped; + } + + public void execute(Socket controlSocket) { + this.execute(controlSocket, null); + } + + public synchronized void execute(Socket controlSocket, Object synObj) { + this.controlSocket = controlSocket; + this.syncObject = synObj; + this.done = false; + + if (!this.running) { + // this.setDaemon(true); + this.start(); + } else { + this.notifyAll(); + } + } + public long getTime() { return System.currentTimeMillis() - start; } + + public void setIdentity(String id) { + this.identity = id; + } + + /* + public void setPromiscuous() { + this.promiscuous = true; + } + */ + + public void log(boolean outgoing, String request) { + serverCore.this.log.logInfo(userAddress.getHostAddress() + "/" + this.identity + " " + + "[" + serverCore.this.theSessionPool.getNumActive() + ", " + this.commandCounter + + ((outgoing) ? "] > " : "] < ") + + request); + } + + public void writeLine(String messg) throws IOException { + send(this.out, messg); + log(true, messg); + } + + public byte[] readLine() { + return receive(in, this.readLineBuffer, timeout, commandMaxLength, false); + } + + + /** + * @return + */ + public boolean isRunning() { + return this.running; + } + + /** + * @param object + */ + public void setResult(Object object) { + this.processingResult = object; + } - public void setIdentity(String id) { - this.identity = id; - } - - /* - public void setPromiscuous() { - this.promiscuous = true; - } - */ - - public void log(boolean outgoing, String request) { - log.logInfo(userAddress.getHostAddress() + "/" + this.identity + " " + - "[" + activeThreads.size() + ", " + commandCounter + - ((outgoing) ? "] > " : "] < ") + - request); - } - - public void writeLine(String messg) throws IOException { - send(out, messg); - log(true, messg); - } - - public byte[] readLine() { - return receive(in, timeout, commandMaxLength, false); - } - - public final void run() { - //log.logDebug("* session " + handle + " started. time = " + (System.currentTimeMillis() - handle)); - try { - listen(); - } finally { - try { - out.flush(); + /** + * + */ + public void reset() { + this.done = true; + this.syncObject = null; + this.readLineBuffer.reset(); + } + + /** + * + * + * @see java.lang.Thread#run() + */ + public void run() { + this.running = true; + + // The thread keeps running. + while (!this.stopped && !Thread.interrupted()) { + if (this.done) { + // We are waiting for a task now. + synchronized (this) { + try { + this.wait(); //Wait until we get a request to process. + } + catch (InterruptedException e) { + this.stopped = true; + // log.error("", e); + } + } + } + else + { + //There is a task....let us execute it. + try { + execute(); + if (this.syncObject != null) { + synchronized (this.syncObject) { + //Notify the completion. + this.syncObject.notifyAll(); + } + } + } catch (Exception e) { + // log.error("", e); + } + finally { + reset(); + + if (!this.stopped && !this.isInterrupted()) { + try { + serverCore.this.theSessionPool.returnObject(this); + } + catch (Exception e1) { + e1.printStackTrace(); + } + } + } + } + } + } + + private void execute() { + + try { + // setting the session startup time + this.start = System.currentTimeMillis(); + + // settin the session identity + this.identity = "-"; + + // getting some client information + this.userAddress = this.controlSocket.getInetAddress(); + + // TODO: check if we want to allow this socket to connect us + + // getting input and output stream for communication with client + this.in = new PushbackInputStream(this.controlSocket.getInputStream()); + this.out = this.controlSocket.getOutputStream(); + + + // initiate the command class + this.commandCounter = 0; + if ((this.commandObj != null) && + (this.commandObj.getClass().getName().equals(serverCore.this.handlerPrototype.getClass().getName()))) { + this.commandObj.reset(); + } + else { + this.commandObj = (serverHandler) serverCore.this.handlerPrototype.clone(); + } + this.commandObj.initSession(this); + + listen(); + } catch (Exception e) { + System.err.println("ERROR: (internal) " + e); + } finally { + try { + this.out.flush(); // close everything - out.close(); - in.close(); - controlSocket.close(); - } catch (IOException e) { - System.err.println("ERROR: (internal) " + e); - } - synchronized (this) {this.notify();} - } + this.out.close(); + this.in.close(); + this.controlSocket.close(); + } catch (IOException e) { + System.err.println("ERROR: (internal) " + e); + } + } + //log.logDebug("* session " + handle + " completed. time = " + (System.currentTimeMillis() - handle)); - announceMoreExecTime(System.currentTimeMillis() - start); - } - - private void listen() { - try { - // set up some reflection - Class[] stringType = {"".getClass()}; - Class[] exceptionType = {Class.forName("java.lang.Throwable")}; - - // send greeting - Object result = commandObj.greeting(); - if (result != null) { - if ((result instanceof String) && (((String) result).length() > 0)) writeLine((String) result); - } - - // start dialog - byte[] requestBytes = null; - boolean terminate = false; - int pos; - String cmd; - String tmp; - Object[] stringParameter = new String[1]; - while ((in != null) && ((requestBytes = readLine()) != null)) { - commandCounter++; - request = new String(requestBytes); - //log.logDebug("* session " + handle + " received command '" + request + "'. time = " + (System.currentTimeMillis() - handle)); - log(false, request); - try { - pos = request.indexOf(' '); - if (pos < 0) { - cmd = request.trim().toUpperCase(); - stringParameter[0] = ""; - } else { - cmd = request.substring(0, pos).trim().toUpperCase(); - stringParameter[0] = request.substring(pos).trim(); - } - - // exec command and return value - result = commandObj.getClass().getMethod(cmd, stringType).invoke(commandObj, stringParameter); - //log.logDebug("* session " + handle + " completed command '" + request + "'. time = " + (System.currentTimeMillis() - handle)); + announceMoreExecTime(System.currentTimeMillis() - this.start); + } + + private void listen() { + try { + // set up some reflection + Class[] stringType = {"".getClass()}; + Class[] exceptionType = {Class.forName("java.lang.Throwable")}; + + // send greeting + Object result = commandObj.greeting(); + if (result != null) { + if ((result instanceof String) && (((String) result).length() > 0)) writeLine((String) result); + } + + // start dialog + byte[] requestBytes = null; + boolean terminate = false; + int pos; + String cmd; + String tmp; + Object[] stringParameter = new String[1]; + while ((this.in != null) && ((requestBytes = readLine()) != null)) { + commandCounter++; + request = new String(requestBytes); + //log.logDebug("* session " + handle + " received command '" + request + "'. time = " + (System.currentTimeMillis() - handle)); + log(false, request); + try { + pos = request.indexOf(' '); + if (pos < 0) { + cmd = request.trim().toUpperCase(); + stringParameter[0] = ""; + } else { + cmd = request.substring(0, pos).trim().toUpperCase(); + stringParameter[0] = request.substring(pos).trim(); + } + + // exec command and return value + result = this.commandObj.getClass().getMethod(cmd, stringType).invoke(this.commandObj, stringParameter); + //log.logDebug("* session " + handle + " completed command '" + request + "'. time = " + (System.currentTimeMillis() - handle)); this.out.flush(); - if (result == null) { - /* - log(2, true, "(NULL RETURNED/STREAM PASSED)"); - */ - } else if (result instanceof Boolean) { - if (((Boolean) result) == TERMINATE_CONNECTION) break; - } else if (result instanceof String) { - if (((String) result).startsWith("!")) { - result = ((String) result).substring(1); - terminate = true; - } - writeLine((String) result); - } else if (result instanceof InputStream) { - tmp = send(out, (InputStream) result); - if ((tmp.length() > 4) && (tmp.toUpperCase().startsWith("PASS"))) { - log(true, "PASS ********"); - } else { - log(true, tmp); - } - tmp = null; - } - if (terminate) break; - + if (result == null) { + /* + log(2, true, "(NULL RETURNED/STREAM PASSED)"); + */ + } else if (result instanceof Boolean) { + if (((Boolean) result) == TERMINATE_CONNECTION) break; + } else if (result instanceof String) { + if (((String) result).startsWith("!")) { + result = ((String) result).substring(1); + terminate = true; + } + writeLine((String) result); + } else if (result instanceof InputStream) { + tmp = send(out, (InputStream) result); + if ((tmp.length() > 4) && (tmp.toUpperCase().startsWith("PASS"))) { + log(true, "PASS ********"); + } else { + log(true, tmp); + } + tmp = null; + } + if (terminate) break; + } catch (InvocationTargetException ite) { - System.out.println("ERROR A " + userAddress.getHostAddress()); - // we extract a target exception and let the thread survive - writeLine((String) commandObj.error(ite.getTargetException())); - } catch (NoSuchMethodException nsme) { - System.out.println("ERROR B " + userAddress.getHostAddress()); - if (isNotLocal(userAddress.getHostAddress().toString())) { - if (denyHost != null) - denyHost.put((""+userAddress.getHostAddress()), "deny"); // block client: hacker attempt - } - break; - // the client requested a command that does not exist - //Object[] errorParameter = { nsme }; - //writeLine((String) error.invoke(this.cmdObject, errorParameter)); - } catch (IllegalAccessException iae) { - System.out.println("ERROR C " + userAddress.getHostAddress()); - // wrong parameters: this an only be an internal problem - writeLine((String) commandObj.error(iae)); - } catch (java.lang.ClassCastException e) { - System.out.println("ERROR D " + userAddress.getHostAddress()); - // ?? - writeLine((String) commandObj.error(e)); - } catch (Exception e) { - System.out.println("ERROR E " + userAddress.getHostAddress()); - // whatever happens: the thread has to survive! - writeLine("UNKNOWN REASON:" + (String) commandObj.error(e)); - } - } - } catch (java.lang.ClassNotFoundException e) { - System.out.println("Internal Error: wrapper class not found: " + e.getMessage()); - System.exit(0); - } catch (java.io.IOException e) { - // connection interruption: more or less normal - } - } + System.out.println("ERROR A " + userAddress.getHostAddress()); + // we extract a target exception and let the thread survive + writeLine((String) commandObj.error(ite.getTargetException())); + } catch (NoSuchMethodException nsme) { + System.out.println("ERROR B " + userAddress.getHostAddress()); + if (isNotLocal(userAddress.getHostAddress().toString())) { + if (denyHost != null) + denyHost.put((""+userAddress.getHostAddress()), "deny"); // block client: hacker attempt + } + break; + // the client requested a command that does not exist + //Object[] errorParameter = { nsme }; + //writeLine((String) error.invoke(this.cmdObject, errorParameter)); + } catch (IllegalAccessException iae) { + System.out.println("ERROR C " + userAddress.getHostAddress()); + // wrong parameters: this an only be an internal problem + writeLine((String) commandObj.error(iae)); + } catch (java.lang.ClassCastException e) { + System.out.println("ERROR D " + userAddress.getHostAddress()); + // ?? + writeLine((String) commandObj.error(e)); + } catch (Exception e) { + System.out.println("ERROR E " + userAddress.getHostAddress()); + // whatever happens: the thread has to survive! + writeLine("UNKNOWN REASON:" + (String) commandObj.error(e)); + } + } // end of while + } catch (java.lang.ClassNotFoundException e) { + System.out.println("Internal Error: wrapper class not found: " + e.getMessage()); + System.exit(0); + } catch (java.io.IOException e) { + // connection interruption: more or less normal + } + } } - public static byte[] receive(PushbackInputStream pbis, long timeout, int maxSize, boolean logerr) { + public static byte[] receive(PushbackInputStream pbis, ByteArrayOutputStream readLineBuffer, long timeout, int maxSize, boolean logerr) { + // this is essentially a readln on a PushbackInputStream int bufferSize = 0; bufferSize = 10; - try { - long t = timeout; - while (((bufferSize = pbis.available()) == 0) && (t > 0)) try { - Thread.currentThread().sleep(100); - t -= 100; - } catch (InterruptedException e) {} - if (t <= 0) { - if (logerr) serverLog.logError("SERVER", "receive interrupted - timeout"); - return null; - } - if (bufferSize == 0) { - if (logerr) serverLog.logError("SERVER", "receive interrupted - buffer empty"); + // reuse an existing linebuffer or create a new one ... + if (readLineBuffer == null) { + readLineBuffer = new ByteArrayOutputStream(256); + } else { + readLineBuffer.reset(); + } + + + // TODO: we should remove this statements because calling the available function is very time consuming + // we better should use nio sockets instead because they are interruptable ... + try { + long t = timeout; + while (((bufferSize = pbis.available()) == 0) && (t > 0)) try { + Thread.currentThread().sleep(100); + t -= 100; + } catch (InterruptedException e) {} + if (t <= 0) { + if (logerr) serverLog.logError("SERVER", "receive interrupted - timeout"); + return null; + } + if (bufferSize == 0) { + if (logerr) serverLog.logError("SERVER", "receive interrupted - buffer empty"); + return null; + } + } catch (IOException e) { + if (logerr) serverLog.logError("SERVER", "receive interrupted - exception 1 = " + e.getMessage()); return null; } - } catch (IOException e) { - if (logerr) serverLog.logError("SERVER", "receive interrupted - exception 1 = " + e.getMessage()); - return null; - } - - byte[] buffer = new byte[bufferSize]; - byte[] bufferBkp; - bufferSize = 0; - int b = 0; - - try { - while ((b = pbis.read()) > 31) { - // we have a valid byte in b, add it to the buffer - if (buffer.length == bufferSize) { - // the buffer is full, double its size - bufferBkp = buffer; - buffer = new byte[bufferSize * 2]; - java.lang.System.arraycopy(bufferBkp, 0, buffer, 0, bufferSize); - bufferBkp = null; - } - //if (bufferSize > 10000) {System.out.println("***ERRORDEBUG***:" + new String(buffer));} // debug - buffer[bufferSize++] = (byte) b; // error hier: ArrayIndexOutOfBoundsException: -2007395416 oder 0 - if (bufferSize > maxSize) break; + + // byte[] buffer = new byte[bufferSize]; + // byte[] bufferBkp; + bufferSize = 0; + int b = 0; + + try { + while ((b = pbis.read()) > 31) { +// // we have a valid byte in b, add it to the buffer +// if (buffer.length == bufferSize) { +// // the buffer is full, double its size +// bufferBkp = buffer; +// buffer = new byte[bufferSize * 2]; +// java.lang.System.arraycopy(bufferBkp, 0, buffer, 0, bufferSize); +// bufferBkp = null; +// } +// //if (bufferSize > 10000) {System.out.println("***ERRORDEBUG***:" + new String(buffer));} // debug +// buffer[bufferSize++] = (byte) b; // error hier: ArrayIndexOutOfBoundsException: -2007395416 oder 0 + + readLineBuffer.write(b); + if (bufferSize++ > maxSize) break; } - // we have catched a possible line end - if (b == cr) { - // maybe a lf follows, read it: - if ((b = pbis.read()) != lf) if (b >= 0) pbis.unread(b); // we push back the byte - } - - // finally shrink buffer - bufferBkp = buffer; - buffer = new byte[bufferSize]; - java.lang.System.arraycopy(bufferBkp, 0, buffer, 0, bufferSize); - bufferBkp = null; - - // return only the byte[] - return buffer; - } catch (IOException e) { - if (logerr) serverLog.logError("SERVER", "receive interrupted - exception 2 = " + e.getMessage()); - return null; - } + + // we have catched a possible line end + if (b == cr) { + // maybe a lf follows, read it: + if ((b = pbis.read()) != lf) if (b >= 0) pbis.unread(b); // we push back the byte + } + + // finally shrink buffer +// bufferBkp = buffer; +// buffer = new byte[bufferSize]; +// java.lang.System.arraycopy(bufferBkp, 0, buffer, 0, bufferSize); +// bufferBkp = null; + + // return only the byte[] + // return buffer; + return readLineBuffer.toByteArray(); + } catch (IOException e) { + if (logerr) serverLog.logError("SERVER", "receive interrupted - exception 2 = " + e.getMessage()); + return null; + } } public static void send(OutputStream os, String buf) throws IOException { - os.write(buf.getBytes()); - os.write(crlf); - os.flush(); + os.write(buf.getBytes()); + os.write(crlf); + os.flush(); } public static void send(OutputStream os, byte[] buf) throws IOException { - os.write(buf); - os.write(crlf); - os.flush(); + os.write(buf); + os.write(crlf); + os.flush(); } public static String send(OutputStream os, InputStream is) throws IOException { - int bufferSize = is.available(); - byte[] buffer = new byte[((bufferSize < 1) || (bufferSize > 4096)) ? 4096 : bufferSize]; - int l; - while ((l = is.read(buffer)) > 0) {os.write(buffer, 0, l);} - os.write(crlf); - os.flush(); - if (bufferSize > 80) return ""; else return new String(buffer); + int bufferSize = is.available(); + byte[] buffer = new byte[((bufferSize < 1) || (bufferSize > 4096)) ? 4096 : bufferSize]; + int l; + while ((l = is.read(buffer)) > 0) {os.write(buffer, 0, l);} + os.write(crlf); + os.flush(); + if (bufferSize > 80) return ""; else return new String(buffer); + } + + protected void finalize() throws Throwable { + if (!this.theSessionPool.isClosed) this.theSessionPool.close(); + super.finalize(); } } diff --git a/source/de/anomic/server/serverDate.java b/source/de/anomic/server/serverDate.java index 85de40f9e..dfa85cc55 100644 --- a/source/de/anomic/server/serverDate.java +++ b/source/de/anomic/server/serverDate.java @@ -47,7 +47,7 @@ import java.lang.*; import java.util.*; import java.text.*; -public class serverDate { +public final class serverDate { // statics diff --git a/source/de/anomic/server/serverFileUtils.java b/source/de/anomic/server/serverFileUtils.java index 9e96ec3b6..bba9cf816 100644 --- a/source/de/anomic/server/serverFileUtils.java +++ b/source/de/anomic/server/serverFileUtils.java @@ -42,7 +42,7 @@ package de.anomic.server; import java.io.*; -public class serverFileUtils { +public final class serverFileUtils { public static void copy(InputStream source, OutputStream dest) throws IOException { byte[] buffer = new byte[4096]; diff --git a/source/de/anomic/server/serverHandler.java b/source/de/anomic/server/serverHandler.java index aa94508e5..f5d0446c2 100644 --- a/source/de/anomic/server/serverHandler.java +++ b/source/de/anomic/server/serverHandler.java @@ -110,4 +110,8 @@ public interface serverHandler { // but only the necessary one for a newly initialized instance public Object clone(); + // Instead of using clone this function can be used to reset an existing + // handler prototype so that it can e reused + public void reset(); + } diff --git a/source/de/anomic/server/serverInstantThread.java b/source/de/anomic/server/serverInstantThread.java index e4430d581..6a0ba2257 100644 --- a/source/de/anomic/server/serverInstantThread.java +++ b/source/de/anomic/server/serverInstantThread.java @@ -3,7 +3,7 @@ package de.anomic.server; import java.lang.reflect.*; -public class serverInstantThread extends serverAbstractThread implements serverThread { +public final class serverInstantThread extends serverAbstractThread implements serverThread { private Method jobExecMethod, jobCountMethod; private Object environment; diff --git a/source/de/anomic/server/serverLog.java b/source/de/anomic/server/serverLog.java index c7289f668..fb603be6a 100644 --- a/source/de/anomic/server/serverLog.java +++ b/source/de/anomic/server/serverLog.java @@ -43,7 +43,7 @@ package de.anomic.server; import java.text.*; import java.util.*; -public class serverLog { +public final class serverLog { // statics private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); diff --git a/source/de/anomic/server/serverObjects.java b/source/de/anomic/server/serverObjects.java index d6a87026d..50155f2b2 100644 --- a/source/de/anomic/server/serverObjects.java +++ b/source/de/anomic/server/serverObjects.java @@ -60,7 +60,7 @@ package de.anomic.server; import java.io.*; import java.util.*; -public class serverObjects extends Hashtable implements Cloneable { +public final class serverObjects extends Hashtable implements Cloneable { public serverObjects() { super(); diff --git a/source/de/anomic/server/serverSystem.java b/source/de/anomic/server/serverSystem.java index d086ac79f..1ef41a234 100644 --- a/source/de/anomic/server/serverSystem.java +++ b/source/de/anomic/server/serverSystem.java @@ -41,11 +41,10 @@ package de.anomic.server; import java.io.*; -import java.net.*; import java.util.*; import java.lang.reflect.*; -public class serverSystem { +public final class serverSystem { // constants for system identification public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.* diff --git a/source/yacy.java b/source/yacy.java index e6e40d33e..c4644a8bc 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -75,11 +75,11 @@ import de.anomic.server.*; import de.anomic.yacy.*; //import de.anomic.http.*; -public class yacy { +public final class yacy { // static objects private static final String vString = "0.361"; - private static final String vDATE = "@REPL_DATE@"; + private static final String vDATE = "20050419"; private static final String copyright = "[ YACY Proxy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]"; private static final String hline = "-------------------------------------------------------------------------------"; @@ -321,7 +321,7 @@ public class yacy { httpHeader requestHeader = new httpHeader(); requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify try { - httpc con = new httpc("localhost", port, 10000, false); + httpc con = httpc.getInstance("localhost", port, 10000, false); httpc.response res = con.GET("Steering.html?shutdown=", requestHeader); // read response