boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) Some performance improvements

Browse Source 
- many classes set to final
- implementation of a session-thread pool
- reusage of the server handler class (normally the httpd object)
  within the session thread
- implementation of a httpc object pool
- introduction of a linebuffer in httpd which can be reused
- reusing the properties table in the httpc
- added to apache libs (commons-collections, commons-pool) which 
  are needed for the object/thread pool implementation

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@26 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
(no author) 20 years ago
parent d5ff81c636
commit f39812da91
27 changed files with 1318 additions and 717 deletions
Show Stats Download Patch File Download Diff File

BIN
lib/commons-collections.jar
View File

Binary file not shown.

BIN
lib/commons-pool-1.2.jar
View File

Binary file not shown.

27
makerelease.sh
Unescape View File

@ -52,9 +52,12 @@ release='yacy_dev_v'$version'_'$datestr
target='RELEASE'
classes='classes'
source='source'
lib='lib'
doc='doc'
data='DATA'
mainclass='yacy.java'
classpath='$classes:$lib'
mkdir $release
# clean up
@ -103,18 +106,18 @@ mv -f $source/$mainclass $source/$mainclass.orig
sed `echo 's/<<REPL_DATE>>/'$datestr'/'` $source/$mainclass.orig > $source/$mainclass.sed1
sed `echo 's/<<REPL_VERSION>>/'$version'/'` $source/$mainclass.sed1 > $source/$mainclass
rm $source/$mainclass.sed1
#javac -classpath $classes -sourcepath $source -d $classes -g:none $source/httpd.java
#javac -classpath $classes -sourcepath $source -d $classes -g:none $source/$mainclass
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/net/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/server/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/http/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/kelondro/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/data/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/plasma/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/de/anomic/yacy/*.java
javac -classpath $classes -sourcepath $source -d $classes -g $source/$mainclass
#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/httpd.java
#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/$mainclass
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/net/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/server/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/http/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/kelondro/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/data/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/plasma/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/yacy/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/$mainclass
mv -f $source/$mainclass.orig $source/$mainclass
# compile server pages

17
source/de/anomic/htmlFilter/htmlFilterOutputStream.java
Unescape View File

@ -53,7 +53,7 @@ import java.net.*;
import java.util.*;
import de.anomic.server.*;
public class htmlFilterOutputStream extends OutputStream {
public final class htmlFilterOutputStream extends OutputStream {
public static final byte lb = (byte) '<';
public static final byte rb = (byte) '>';
@ -321,14 +321,15 @@ public class htmlFilterOutputStream extends OutputStream {
}
} else if (inScript) {
buffer.append(b);
int bufferLength = buffer.length();
if ((b == rb) && (buffer.length() > 14) &&
(buffer.byteAt(buffer.length() - 8) == (byte) '/') &&
(buffer.byteAt(buffer.length() - 7) == (byte) 's') &&
(buffer.byteAt(buffer.length() - 6) == (byte) 'c') &&
(buffer.byteAt(buffer.length() - 5) == (byte) 'r') &&
(buffer.byteAt(buffer.length() - 4) == (byte) 'i') &&
(buffer.byteAt(buffer.length() - 3) == (byte) 'p') &&
(buffer.byteAt(buffer.length() - 2) == (byte) 't')) {
(buffer.byteAt(bufferLength - 8) == (byte) '/') &&
(buffer.byteAt(bufferLength - 7) == (byte) 's') &&
(buffer.byteAt(bufferLength - 6) == (byte) 'c') &&
(buffer.byteAt(bufferLength - 5) == (byte) 'r') &&
(buffer.byteAt(bufferLength - 4) == (byte) 'i') &&
(buffer.byteAt(bufferLength - 3) == (byte) 'p') &&
(buffer.byteAt(bufferLength - 2) == (byte) 't')) {
// script is at end
inScript = false;
if (out != null) out.write(buffer.getBytes());

14
source/de/anomic/http/httpHeader.java
Unescape View File

@ -57,9 +57,9 @@ import java.util.*;
import java.text.*;
import de.anomic.server.*;
public class httpHeader extends TreeMap implements Map {
public final class httpHeader extends TreeMap implements Map {
private HashMap reverseMappingCache;
private final HashMap reverseMappingCache;
private static Collator insensitiveCollator = Collator.getInstance(Locale.US);
static {
@ -111,16 +111,17 @@ public class httpHeader extends TreeMap implements Map {
// we override the put method to make use of the reverseMappingCache
public Object put(Object key, Object value) {
String k = (String) key;
String upperK = k.toUpperCase();
if (reverseMappingCache == null) {
return super.put(k, value);
} else {
if (reverseMappingCache.containsKey(k.toUpperCase())) {
if (reverseMappingCache.containsKey(upperK)) {
// we put in the value using the reverse mapping
return super.put(reverseMappingCache.get(k.toUpperCase()), value);
return super.put(reverseMappingCache.get(upperK), value);
} else {
// we put in without a cached key and store the key afterwards
Object r = super.put(k, value);
reverseMappingCache.put(k.toUpperCase(), k);
reverseMappingCache.put(upperK, k);
return r;
}
}
@ -180,10 +181,9 @@ public class httpHeader extends TreeMap implements Map {
} catch (java.lang.NumberFormatException e) {
//System.out.println("ERROR long version parse: " + e.getMessage() + " at position " + e.getErrorOffset());
serverLog.logError("HTTPC-header", "DATE ERROR (NumberFormat): " + s);
new Date();
}
return new Date();
}
}
private Date headerDate(String kind) {
if (containsKey(kind)) return parseHTTPDate((String) get(kind));

2
source/de/anomic/http/httpTemplate.java
Unescape View File

@ -46,7 +46,7 @@ import de.anomic.server.*;
import java.util.*;
import java.io.*;
public class httpTemplate {
final class httpTemplate {
private static final byte hash = (byte)'#';
private static final byte[] hasha = {hash};

272
source/de/anomic/http/httpc.java
Unescape View File

@ -56,23 +56,29 @@ import java.lang.*;
import java.util.*;
import java.util.zip.*;
import de.anomic.server.*;
import de.anomic.server.serverCore.Session;
import de.anomic.server.serverCore.SessionFactory;
import de.anomic.server.serverCore.SessionPool;
import javax.net.ssl.SSLSocketFactory;
public class httpc {
import org.apache.commons.pool.impl.GenericObjectPool;
public final class httpc {
// statics
private static final String vDATE = "20040602";
private static String userAgent;
public static String systemOST;
private static final int terminalMaxLength = 30000;
private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST");
private static final TimeZone GMTTimeZone = TimeZone.getTimeZone("PST");
// --- The GMT standard date format used in the HTTP protocol
private static SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US);
private static SimpleDateFormat EMLFormatter = new SimpleDateFormat("dd MMM yyyy HH:mm:ss", Locale.US);
private static SimpleDateFormat ShortFormatter = new SimpleDateFormat("yyyyMMddHHmmss");
private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US);
private static final SimpleDateFormat EMLFormatter = new SimpleDateFormat("dd MMM yyyy HH:mm:ss", Locale.US);
private static final SimpleDateFormat ShortFormatter = new SimpleDateFormat("yyyyMMddHHmmss");
//Mo 06 Sep 2004 23:32
private static HashMap reverseMappingCache = new HashMap();
private static final HashMap reverseMappingCache = new HashMap();
// class variables
private Socket socket = null; // client socket for commands
@ -89,7 +95,7 @@ public class httpc {
private String requestPath = null;
// the dns cache
private static HashMap nameCacheHit = new HashMap();
private static final HashMap nameCacheHit = new HashMap();
//private static HashSet nameCacheMiss = new HashSet();
static {
@ -98,6 +104,74 @@ public class httpc {
java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0");
}
private static final httpcPool theHttpcPool;
static {
// implementation of session thread pool
GenericObjectPool.Config config = new GenericObjectPool.Config();
// The maximum number of active connections that can be allocated from pool at the same time,
// 0 for no limit
config.maxActive = 150;
// The maximum number of idle connections connections in the pool
// 0 = no limit.
config.maxIdle = 75;
config.minIdle = 10;
config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
config.minEvictableIdleTimeMillis = 30000;
theHttpcPool = new httpcPool(new httpcFactory(),config);
}
private static final ByteArrayOutputStream readLineBuffer = new ByteArrayOutputStream();
public static httpc getInstance(String server, int port, int timeout, boolean ssl,
String remoteProxyHost, int remoteProxyPort) throws IOException {
try {
// fetching a new httpc from the object pool
httpc newHttpc = (httpc) httpc.theHttpcPool.borrowObject();
// initialize it
newHttpc.init(server,port,timeout,ssl,remoteProxyHost, remoteProxyPort);
return newHttpc;
} catch (Exception e) {
throw new IOException("Unable to initialize a new httpc. " + e.getMessage());
}
}
public static httpc getInstance(String server, int port, int timeout, boolean ssl) throws IOException {
try {
// fetching a new httpc from the object pool
httpc newHttpc = (httpc) httpc.theHttpcPool.borrowObject();
// initialize it
newHttpc.init(server,port,timeout,ssl);
return newHttpc;
} catch (Exception e) {
throw new IOException("Unable to initialize a new httpc. " + e.getMessage());
}
}
public static void returnInstance(httpc theHttpc) {
try {
theHttpc.reset();
httpc.theHttpcPool.returnObject(theHttpc);
} catch (Exception e) {
// we could ignore this error
}
}
protected void finalize() throws Throwable {
System.err.println("Httpc object was not returned to object pool.");
this.reset();
httpc.theHttpcPool.invalidateObject(this);
}
public static String dnsResolve(String host) {
// looks for the ip of host <host> and returns ip number as string
String ip = (String) nameCacheHit.get(host);
@ -135,17 +209,44 @@ public class httpc {
}
}
void reset() {
try {
if (this.clientInput != null) {
this.clientInput.close();
this.clientInput = null;
}
if (this.clientOutput != null) {
this.clientOutput.close();
this.clientOutput = null;
}
if (this.socket != null) {
this.socket.close();
this.socket = null;
}
this.host = null;
this.timeout = 0;
this.handle = 0;
this.remoteProxyUse = false;
this.savedRemoteHost = null;
this.requestPath = null;
} catch (Exception e) {
// we could ignore this ...
}
}
// http client
public httpc(String server, int port, int timeout, boolean ssl,
void init(String server, int port, int timeout, boolean ssl,
String remoteProxyHost, int remoteProxyPort) throws IOException {
this(remoteProxyHost, remoteProxyPort, timeout, ssl);
this.init(remoteProxyHost, remoteProxyPort, timeout, ssl);
this.remoteProxyUse = true;
this.savedRemoteHost = server + ((port == 80) ? "" : (":" + port));
}
public httpc(String server, int port, int timeout, boolean ssl) throws IOException {
void init(String server, int port, int timeout, boolean ssl) throws IOException {
handle = System.currentTimeMillis();
//serverLog.logDebug("HTTPC", handle + " initialized");
this.remoteProxyUse = false;
@ -230,7 +331,7 @@ public class httpc {
}
// reads in the http header, right now, right here
byte[] b = serverCore.receive(clientInput, timeout, terminalMaxLength, false);
byte[] b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false);
if (b == null) {
// the server has meanwhile disconnected
status = "503 server has closed connection";
@ -242,7 +343,7 @@ public class httpc {
if (p < 0) {
status = "500 status line parse error";
// flush in anything that comes without parsing
while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)).length != 0) {}
while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)).length != 0) {}
return; // in bad mood
}
// we have a status
@ -252,14 +353,14 @@ public class httpc {
if (status.startsWith("400")) {
// bad request
// flush in anything that comes without parsing
while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)).length != 0) {}
while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)).length != 0) {}
return; // in bad mood
}
// at this point we should have a valid response. read in the header properties
String key = "";
String value = "";
while ((b = serverCore.receive(clientInput, timeout, terminalMaxLength, false)) != null) {
while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)) != null) {
if (b.length == 0) break;
buffer = new String(b);
//System.out.println("#H#" + buffer); // debug
@ -731,11 +832,16 @@ do upload
if ((user != null) && (password != null) && (user.length() != 0)) {
requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password));
}
httpc con;
if ((proxyHost == null) || (proxyPort == 0))
con = new httpc(host, port, timeout, ssl);
else
con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort);
httpc con = null;
try {
if ((proxyHost == null) || (proxyPort == 0)) {
con = httpc.getInstance(host, port, timeout, ssl);
} else {
con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort);
}
httpc.response res = con.GET(path, null);
if (res.status.startsWith("2")) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
@ -745,6 +851,12 @@ do upload
} else {
return res.status.getBytes();
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
}
public static byte[] singleGET(URL u, int timeout,
@ -773,16 +885,20 @@ do upload
String user, String password, boolean ssl,
String proxyHost, int proxyPort,
httpHeader requestHeader, serverObjects props) throws IOException {
if (requestHeader == null) requestHeader = new httpHeader();
if ((user != null) && (password != null) && (user.length() != 0)) {
requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password));
}
httpc con;
httpc con = null;
try {
if ((proxyHost == null) || (proxyPort == 0))
con = new httpc(host, port, timeout, ssl);
con = httpc.getInstance(host, port, timeout, ssl);
else
con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort);
con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort);
httpc.response res = con.POST(path, null, props, null);
//System.out.println("response=" + res.toString());
if (res.status.startsWith("2")) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
@ -792,6 +908,12 @@ do upload
} else {
return res.status.getBytes();
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
}
public static byte[] singlePOST(URL u, int timeout,
@ -837,6 +959,7 @@ do upload
requestHeader.put("Authorization", serverCodings.standardCoder.encodeBase64String(user + ":" + password));
}
// parse query
int port = url.getPort();
boolean ssl = url.getProtocol().equals("https");
if (port < 0) port = (ssl) ? 443 : 80;
@ -844,12 +967,14 @@ do upload
String query = url.getQuery();
if ((query != null) && (query.length() > 0)) path = path + "?" + query;
String host = url.getHost();
// start connection
httpc con;
httpc con = null;
try {
if ((proxyHost == null) || (proxyPort == 0))
con = new httpc(host, port, timeout, ssl);
else
con = new httpc(host, port, timeout, ssl, proxyHost, proxyPort);
con = httpc.getInstance(host, port, timeout, ssl);
else con = httpc.getInstance(host, port, timeout, ssl, proxyHost, proxyPort);
httpc.response res = con.HEAD(path, requestHeader);
if (res.status.startsWith("2")) {
// success
@ -858,6 +983,11 @@ do upload
// fail
return res.responseHeader;
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
}
/*
@ -930,6 +1060,9 @@ do upload
while (i.hasMoreElements()) System.out.println((String) i.nextElement());
}
}
/*
@ -1027,3 +1160,90 @@ public class SSLSocketClientWithClientAuth {
}
}
*/
final class httpcFactory implements org.apache.commons.pool.PoolableObjectFactory {
public httpcFactory() {
super();
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#makeObject()
*/
public Object makeObject() throws Exception {
return new httpc();
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)
*/
public void destroyObject(Object obj) {
if (obj instanceof httpc) {
httpc theHttpc = (httpc) obj;
}
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object)
*/
public boolean validateObject(Object obj) {
if (obj instanceof httpc)
{
httpc theHttpc = (httpc) obj;
return true;
}
return true;
}
/**
* @param obj
*
*/
public void activateObject(Object obj) {
//log.debug(" activateObject...");
}
/**
* @param obj
*
*/
public void passivateObject(Object obj) {
//log.debug(" passivateObject..." + obj);
if (obj instanceof Session) {
httpc theHttpc = (httpc) obj;
}
}
}
final class httpcPool extends GenericObjectPool {
/**
* First constructor.
* @param objFactory
*/
public httpcPool(httpcFactory objFactory) {
super(objFactory);
this.setMaxIdle(75); // Maximum idle threads.
this.setMaxActive(150); // Maximum active threads.
this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs.
//this.setMaxWait(1000); // Wait 1 second till a thread is available
}
public httpcPool(httpcFactory objFactory,
GenericObjectPool.Config config) {
super(objFactory, config);
}
/**
* @see org.apache.commons.pool.impl.GenericObjectPool#borrowObject()
*/
public Object borrowObject() throws Exception {
return super.borrowObject();
}
/**
* @see org.apache.commons.pool.impl.GenericObjectPool#returnObject(java.lang.Object)
*/
public void returnObject(Object obj) throws Exception {
super.returnObject(obj);
}
}

48
source/de/anomic/http/httpd.java
Unescape View File

@ -53,9 +53,15 @@ import java.io.*;
import java.net.*;
import java.util.*;
import java.text.*;
import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.server.*;
import de.anomic.server.serverCore.Session;
import de.anomic.server.serverCore.SessionFactory;
import de.anomic.server.serverCore.SessionPool;
public class httpd implements serverHandler {
public final class httpd implements serverHandler {
// static objects
public static final String vDATE = "<<REPL>>";
@ -78,14 +84,18 @@ public class httpd implements serverHandler {
private String serverAccountBase64MD5;
private String clientIP;
// class methods
// the connection properties
private static final Properties prop = new Properties();
// class methods
public httpd(serverSwitch s, httpdHandler fileHandler, httpdHandler proxyHandler) {
// handler info
this.switchboard = s;
this.fileHandler = fileHandler;
this.proxyHandler = proxyHandler;
this.virtualHost = switchboard.getConfig("fileHost","localhost");
httpd.switchboard = s;
httpd.fileHandler = fileHandler;
httpd.proxyHandler = proxyHandler;
httpd.virtualHost = switchboard.getConfig("fileHost","localhost");
// authentication: by default none
this.proxyAccountBase64MD5 = null;
@ -93,6 +103,16 @@ public class httpd implements serverHandler {
this.clientIP = null;
}
public void reset() {
this.session = null;
this.userAddress = null;
this.allowProxy = false;
this.allowServer = false;
this.proxyAccountBase64MD5 = null;
this.serverAccountBase64MD5 = null;
this.clientIP = null;
}
// must be called at least once, but can be called again to re-use the object.
public void initSession(serverCore.Session session) throws IOException {
this.session = session;
@ -185,7 +205,7 @@ public class httpd implements serverHandler {
}
public Boolean GET(String arg) throws IOException {
Properties prop = parseQuery(arg);
parseQuery(prop, arg);
prop.setProperty("METHOD", "GET");
prop.setProperty("CLIENTIP", clientIP);
@ -268,7 +288,7 @@ public class httpd implements serverHandler {
}
public Boolean HEAD(String arg) throws IOException {
Properties prop = parseQuery(arg);
parseQuery(prop,arg);
prop.setProperty("METHOD", "HEAD");
prop.setProperty("CLIENTIP", clientIP);
@ -342,7 +362,7 @@ public class httpd implements serverHandler {
}
public Boolean POST(String arg) throws IOException {
Properties prop = parseQuery(arg);
parseQuery(prop, arg);
prop.setProperty("METHOD", "POST");
prop.setProperty("CLIENTIP", clientIP);
@ -487,8 +507,13 @@ public class httpd implements serverHandler {
}
private Properties parseQuery(String s) {
Properties prop = new Properties();
private static final Properties parseQuery(Properties prop, String s) {
if (prop == null) {
prop = new Properties();
} else {
prop.clear();
}
// this parses a whole URL
if (s.length() == 0) {
@ -819,7 +844,6 @@ permission
if (pos < 0) return false;
return whitelist.contains(mime.substring(0, pos));
}
}
/*

4
source/de/anomic/http/httpdAbstractHandler.java
Unescape View File

@ -51,11 +51,9 @@
package de.anomic.http;
import java.io.*;
import java.util.*;
import java.text.*;
public abstract class httpdAbstractHandler {
abstract class httpdAbstractHandler {
// static tools

1
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -76,7 +76,6 @@ package de.anomic.http;
import java.io.*;
import java.util.*;
import java.text.*;
import java.lang.reflect.*;
import de.anomic.server.*;

56
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -62,16 +62,13 @@ package de.anomic.http;
import java.io.*;
import java.net.*;
import java.util.*;
import java.text.*;
import de.anomic.htmlFilter.*;
import de.anomic.server.*;
import de.anomic.tools.*;
import de.anomic.yacy.*;
import de.anomic.http.*;
import de.anomic.plasma.*;
public class httpdProxyHandler extends httpdAbstractHandler implements httpdHandler {
public final class httpdProxyHandler extends httpdAbstractHandler implements httpdHandler {
// static variables
// can only be instantiated upon first instantiation of this class object
@ -87,10 +84,10 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
public static int remoteProxyPort = -1;
public static String remoteProxyNoProxy = "";
public static String[] remoteProxyNoProxyPatterns = null;
private static HashSet remoteProxyAllowProxySet = new HashSet();
private static HashSet remoteProxyDisallowProxySet = new HashSet();
private static final HashSet remoteProxyAllowProxySet = new HashSet();
private static final HashSet remoteProxyDisallowProxySet = new HashSet();
private static htmlFilterTransformer transformer = null;
public static String userAgent = "yacy (" + httpc.systemOST +") yacy.net";
public static final String userAgent = "yacy (" + httpc.systemOST +") yacy.net";
private File htRootPath = null;
// class methods
@ -108,8 +105,6 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
}
remoteProxyUse = switchboard.getConfig("remoteProxyUse","false").equals("true");
remoteProxyNoProxy = switchboard.getConfig("remoteProxyNoProxy","");
remoteProxyAllowProxySet = new HashSet();
remoteProxyDisallowProxySet = new HashSet();
remoteProxyNoProxyPatterns = remoteProxyNoProxy.split(",");
// set loglevel
@ -586,6 +581,8 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
respond.write(("]\r\n").getBytes());
}
} catch (Exception ee) {}
} finally {
if (remote != null) httpc.returnInstance(remote);
}
respond.flush();
}
@ -686,7 +683,10 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
e.printStackTrace(new PrintStream(respond));
respond.write(("]\r\n").getBytes());
} catch (Exception ee) {}
} finally {
if (remote != null) httpc.returnInstance(remote);
}
respond.flush();
}
@ -740,6 +740,8 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
e.printStackTrace(new PrintStream(respond));
respond.write(("]\r\n").getBytes());
} catch (Exception ee) {}
} finally {
if (remote != null) httpc.returnInstance(remote);
}
respond.flush();
}
@ -752,7 +754,9 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
// possibly branch into PROXY-PROXY connection
if (remoteProxyUse) {
httpc remoteProxy = new httpc(host, port, timeout, false, remoteProxyHost, remoteProxyPort);
httpc remoteProxy = null;
try {
remoteProxy = httpc.getInstance(host, port, timeout, false, remoteProxyHost, remoteProxyPort);
httpc.response response = remoteProxy.CONNECT(host, port, requestHeader);
response.print();
if (response.success()) {
@ -765,6 +769,11 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
respondHeader(clientOut, response.status, response.responseHeader);
return;
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (remoteProxy != null) httpc.returnInstance(remoteProxy);
}
}
// try to establish connection to remote host
@ -873,9 +882,9 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
}
// branch to server/proxy
if (useProxy) {
return new httpc(server, port, timeout, false, remoteProxyHost, remoteProxyPort);
return httpc.getInstance(server, port, timeout, false, remoteProxyHost, remoteProxyPort);
} else {
return new httpc(server, port, timeout, false);
return httpc.getInstance(server, port, timeout, false);
}
}
@ -895,15 +904,18 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
}
private void respondHeader(OutputStream respond, String status, httpHeader header) throws IOException, SocketException {
String s;
// prepare header
//header.put("Server", "AnomicHTTPD (www.anomic.de)");
if (!(header.containsKey("date"))) header.put("Date", httpc.dateString(httpc.nowDate()));
if (!(header.containsKey("content-type"))) header.put("Content-type", "text/html"); // fix this
StringBuffer headerStringBuffer = new StringBuffer();
// write status line
respond.write(("HTTP/1.1 " + status + "\r\n").getBytes());
headerStringBuffer.append("HTTP/1.1 ")
.append(status)
.append("\r\n");
//System.out.println("HEADER: PROXY TO CLIENT = " + header.toString()); // DEBUG
@ -920,17 +932,27 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
if (!(key.equals("Location"))) while ((pos = value.lastIndexOf("#")) >= 0) {
// special handling is needed if a key appeared several times, which is valid.
// all lines with same key are combined in one value, separated by a "#"
respond.write((key + ": " + value.substring(pos + 1).trim() + "\r\n").getBytes());
headerStringBuffer
.append(key)
.append(": ")
.append(value.substring(pos + 1).trim())
.append("\r\n");
//System.out.println("#" + key + ": " + value.substring(pos + 1).trim());
value = value.substring(0, pos).trim();
}
respond.write((key + ": " + value + "\r\n").getBytes());
headerStringBuffer
.append(key)
.append(": ")
.append(value)
.append("\r\n");
//System.out.println("#" + key + ": " + value);
}
}
headerStringBuffer.append("\r\n");
// end header
respond.write(("\r\n").getBytes());
respond.write(headerStringBuffer.toString().getBytes());
respond.flush();
}

4
source/de/anomic/http/httpdSwitchboard.java
Unescape View File

@ -44,9 +44,9 @@ import java.io.*;
import java.util.*;
import de.anomic.server.*;
public class httpdSwitchboard extends serverAbstractSwitch implements serverSwitch {
public final class httpdSwitchboard extends serverAbstractSwitch implements serverSwitch {
private LinkedList cacheStack;
private final LinkedList cacheStack;
public httpdSwitchboard(String rootPath, String initPath, String configPath) throws IOException {
super(rootPath, initPath, configPath);

14
source/de/anomic/plasma/plasmaCrawlLoader.java
Unescape View File

@ -50,7 +50,7 @@ import de.anomic.server.*;
import de.anomic.tools.*;
import de.anomic.htmlFilter.*;
public class plasmaCrawlLoader {
public final class plasmaCrawlLoader {
private plasmaHTCache cacheManager;
private int socketTimeout;
@ -130,7 +130,7 @@ public class plasmaCrawlLoader {
return result;
}
public class Exec extends Thread {
public final class Exec extends Thread {
public URL url;
public String referer;
@ -160,9 +160,8 @@ public class plasmaCrawlLoader {
private httpc newhttpc(String server, int port, boolean ssl) throws IOException {
// a new httpc connection, combined with possible remote proxy
if (remoteProxyUse)
return new httpc(server, port, socketTimeout, ssl, remoteProxyHost, remoteProxyPort);
else
return new httpc(server, port, socketTimeout, ssl);
return httpc.getInstance(server, port, socketTimeout, ssl, remoteProxyHost, remoteProxyPort);
else return httpc.getInstance(server, port, socketTimeout, ssl);
}
private void load(URL url, String referer, String initiator, int depth, plasmaCrawlProfile.entry profile) throws IOException {
@ -179,6 +178,7 @@ public class plasmaCrawlLoader {
if (referer.length() == 0) referer = "http://www.yacy.net/yacy/";
// take a file from the net
httpc remote = null;
try {
// create a request header
httpHeader requestHeader = new httpHeader();
@ -189,7 +189,7 @@ public class plasmaCrawlLoader {
//System.out.println("CRAWLER_REQUEST_HEADER=" + requestHeader.toString()); // DEBUG
// open the connection
httpc remote = newhttpc(host, port, ssl);
remote = newhttpc(host, port, ssl);
// send request
httpc.response res = remote.GET(path, requestHeader);
@ -250,6 +250,8 @@ public class plasmaCrawlLoader {
// remote server was wrong.
log.logError("CRAWLER LOADER ERROR2 with url=" + url.toString() + ": " + e.toString());
e.printStackTrace();
} finally {
if (remote != null) httpc.returnInstance(remote);
}
}

8
source/de/anomic/server/serverAbstractSwitch.java
Unescape View File

@ -47,12 +47,12 @@ import java.util.*;
public abstract class serverAbstractSwitch implements serverSwitch {
// configuration management
private File configFile;
private final File configFile;
private Hashtable configProps;
private String configComment;
private Hashtable authorization;
private final String configComment;
private final Hashtable authorization;
private String rootPath;
private TreeMap workerThreads;
private final TreeMap workerThreads;
public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException {
// we initialize the switchboard with a property file,

41
source/de/anomic/server/serverAbstractThread.java
Unescape View File

@ -58,14 +58,14 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
private long threadBlockTimestamp = System.currentTimeMillis();
private long idleCycles = 0, busyCycles = 0;
protected void announceThreadBlockApply() {
protected final void announceThreadBlockApply() {
// shall only be used, if a thread blocks for an important reason
// like a socket connect and must renew the timestamp to correct
// statistics
this.threadBlockTimestamp = System.currentTimeMillis();
}
protected void announceThreadBlockRelease() {
protected final void announceThreadBlockRelease() {
// shall only be used, if a thread blocks for an important reason
// like a socket connect and must renew the timestamp to correct
// statistics
@ -74,69 +74,69 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
this.busytime -= thisBlockTime;
}
protected void announceMoreExecTime(long millis) {
protected final void announceMoreExecTime(long millis) {
this.busytime += millis;
}
protected void announceMoreSleepTime(long millis) {
protected final void announceMoreSleepTime(long millis) {
this.idletime += millis;
}
public void setDescription(String shortText, String longText) {
public final void setDescription(String shortText, String longText) {
// sets a visible description string
this.shortDescr = shortText;
this.longDescr = longText;
}
public void setStartupSleep(long milliseconds) {
public final void setStartupSleep(long milliseconds) {
// sets a sleep time before execution of the job-loop
startup = milliseconds;
}
public void setIdleSleep(long milliseconds) {
public final void setIdleSleep(long milliseconds) {
// sets a sleep time for pauses between two jobs
idlePause = milliseconds;
}
public void setBusySleep(long milliseconds) {
public final void setBusySleep(long milliseconds) {
// sets a sleep time for pauses between two jobs
busyPause = milliseconds;
}
public String getShortDescription() {
public final String getShortDescription() {
return this.shortDescr;
}
public String getLongDescription() {
public final String getLongDescription() {
return this.longDescr;
}
public long getIdleCycles() {
public final long getIdleCycles() {
// returns the total number of cycles of job execution with idle-result
return this.idleCycles;
}
public long getBusyCycles() {
public final long getBusyCycles() {
// returns the total number of cycles of job execution with busy-result
return this.busyCycles;
}
public long getBlockTime() {
public final long getBlockTime() {
// returns the total time that this thread has been blocked so far
return this.blockPause;
}
public long getSleepTime() {
public final long getSleepTime() {
// returns the total time that this thread has slept so far
return this.idletime;
}
public long getExecTime() {
public final long getExecTime() {
// returns the total time that this thread has worked so far
return this.busytime;
}
public void setLog(serverLog log) {
public final void setLog(serverLog log) {
// defines a log where process states can be written to
this.log = log;
}
@ -145,12 +145,15 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
// after calling this method, the thread shall terminate
this.running = false;
// wait for termination
if (waitFor) while (this.isAlive())
try {this.sleep(100);} catch (InterruptedException e) {break;}
if (waitFor) {
// Busy waiting removed: while (this.isAlive()) try {this.sleep(100);} catch (InterruptedException e) {break;}
try { this.join(); } catch (InterruptedException e) {return;}
}
// If we reach this point, the process is closed
}
private void logError(String text) {
private final void logError(String text) {
if (log == null)
serverLog.logError("THREAD-CONTROL", text);
else

15
source/de/anomic/server/serverByteBuffer.java
Unescape View File

@ -43,7 +43,7 @@ package de.anomic.server;
import java.io.*;
import java.util.*;
public class serverByteBuffer extends OutputStream {
public final class serverByteBuffer extends OutputStream {
public static final byte singlequote = (byte) 39;
public static final byte doublequote = (byte) 34;
@ -94,13 +94,14 @@ public class serverByteBuffer extends OutputStream {
try {
FileInputStream fis = new FileInputStream(f);
byte buf[] = new byte[512];
int p = 0;
// byte buf[] = new byte[512];
// int p = 0;
int l;
while ((l = fis.read(buf)) > 0) {
System.arraycopy(buf, 0, buffer, p, l);
p += l;
}
// while ((l = fis.read(buf)) > 0) {
// System.arraycopy(buf, 0, buffer, p, l);
// p += l;
l = fis.read(buffer);
// }
fis.close();
} catch (FileNotFoundException e) {
throw new IOException("File not found: " + f.toString() + "; " + e.getMessage());

4
source/de/anomic/server/serverClassLoader.java
Unescape View File

@ -45,9 +45,9 @@ import java.io.*;
import java.util.*;
import java.lang.reflect.*;
public class serverClassLoader extends ClassLoader {
public final class serverClassLoader extends ClassLoader {
Hashtable classes;
private final Hashtable classes;
public serverClassLoader() {
super(ClassLoader.getSystemClassLoader());

2
source/de/anomic/server/serverCodings.java
Unescape View File

@ -44,7 +44,7 @@ import java.io.*;
import java.security.*;
public class serverCodings {
public final class serverCodings {
// this provides encoding and decoding of long cardinals into a 6-bit - based number format
// expressed by a string. This is probably the most compact form to encode numbers as strings.

641
source/de/anomic/server/serverCore.java
Unescape View File

@ -41,19 +41,30 @@
package de.anomic.server;
// standard server
import java.io.*;
import java.net.*;
import java.lang.*;
import java.util.*;
import java.lang.reflect.*;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PushbackInputStream;
import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.Hashtable;
// needed for ssl
import javax.net.*;
import javax.net.ssl.*;
import java.security.KeyStore;
import javax.security.cert.X509Certificate;
public class serverCore extends serverAbstractThread implements serverThread {
import org.apache.commons.pool.impl.GenericObjectPool;
public final class serverCore extends serverAbstractThread implements serverThread {
// generic input/output static methods
public static final byte cr = 13;
@ -69,22 +80,30 @@ public class serverCore extends serverAbstractThread implements serverThread {
private int port; // the listening port
private ServerSocket socket; // listener
private int maxSessions = 0; // max. number of sessions; 0=unlimited
private serverLog log; // log object
serverLog log; // log object
//private serverSwitch switchboard; // external values
private int timeout; // connection time-out of the socket
private Hashtable activeThreads; // contains the active threads
private Hashtable sleepingThreads; // contains the threads that are alive since the sleepthreashold
// private Hashtable activeThreads; // contains the active threads
// private Hashtable sleepingThreads; // contains the threads that are alive since the sleepthreashold
private boolean termSleepingThreads; // if true then threads over sleepthreashold are killed
private int thresholdActive = 5000; // after that time a thread should have got a command line
private int thresholdSleep = 30000; // after that time a thread is considered as beeing sleeping (30 seconds)
private int thresholdDead = 3600000; // after that time a thread is considered as beeing dead-locked (1 hour)
private serverHandler handlerPrototype;// the command class (a serverHandler)
serverHandler handlerPrototype;// the command class (a serverHandler)
private Class[] initHandlerClasses; // the init's methods arguments
private Class[] initSessionClasses; // the init's methods arguments
private serverSwitch switchboard; // the command class switchboard
private Hashtable denyHost;
private int commandMaxLength;
/**
* The session-object pool
*/
final SessionPool theSessionPool;
final ThreadGroup theSessionThreadGroup = new ThreadGroup("sessionThreadGroup");
private static ServerSocketFactory getServerSocketFactory(boolean dflt, File keyfile, String passphrase) {
// see doc's at
// http://java.sun.com/developer/technicalArticles/Security/secureinternet/
@ -154,15 +173,38 @@ public class serverCore extends serverAbstractThread implements serverThread {
this.initHandlerClasses = new Class[] {Class.forName("de.anomic.server.serverSwitch")};
this.initSessionClasses = new Class[] {Class.forName("de.anomic.server.serverCore$Session")};
this.maxSessions = maxSessions;
this.socket.setSoTimeout(0); // unlimited
this.timeout = timeout;
this.termSleepingThreads = termSleepingThreads;
this.log = new serverLog("SERVER", logl);
activeThreads = new Hashtable();
sleepingThreads = new Hashtable();
// activeThreads = new Hashtable();
// sleepingThreads = new Hashtable();
} catch (java.lang.ClassNotFoundException e) {
System.out.println("FATAL ERROR: " + e.getMessage() + " - Class Not Found"); System.exit(0);
}
// implementation of session thread pool
GenericObjectPool.Config config = new GenericObjectPool.Config();
// The maximum number of active connections that can be allocated from pool at the same time,
// 0 for no limit
config.maxActive = this.maxSessions;
// The maximum number of idle connections connections in the pool
// 0 = no limit.
config.maxIdle = this.maxSessions / 2;
config.minIdle = this.maxSessions / 4;
// block undefinitely
config.maxWait = timeout;
// Action to take in case of an exhausted DBCP statement pool
// 0 = fail, 1 = block, 2= grow
config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
config.minEvictableIdleTimeMillis = this.thresholdSleep;
config.testOnReturn = true;
this.theSessionPool = new SessionPool(new SessionFactory(this.theSessionThreadGroup),config);
}
public static boolean isNotLocal(URL url) {
@ -234,12 +276,12 @@ public class serverCore extends serverAbstractThread implements serverThread {
// class body
public boolean job() throws Exception {
// prepare for new connection
idleThreadCheck();
switchboard.handleBusyState(activeThreads.size());
// idleThreadCheck();
this.switchboard.handleBusyState(this.theSessionPool.getNumActive() /*activeThreads.size() */);
log.logDebug(
"* waiting for connections, " + activeThreads.size() + " sessions running, " +
sleepingThreads.size() + " sleeping");
"* waiting for connections, " + this.theSessionPool.getNumActive() + " sessions running, " +
this.theSessionPool.getNumIdle() + " sleeping");
// list all connection (debug)
/*
@ -257,17 +299,17 @@ public class serverCore extends serverAbstractThread implements serverThread {
// wait for new connection
announceThreadBlockApply();
Socket controlSocket = socket.accept();
Socket controlSocket = this.socket.accept();
announceThreadBlockRelease();
if ((denyHost == null) || (denyHost.get((""+controlSocket.getInetAddress().getHostAddress())) == null)) {
if ((this.denyHost == null) || (this.denyHost.get((""+controlSocket.getInetAddress().getHostAddress())) == null)) {
//log.logDebug("* catched request from " + controlSocket.getInetAddress().getHostAddress());
controlSocket.setSoTimeout(timeout);
controlSocket.setSoTimeout(this.timeout);
Session connection = (Session) this.theSessionPool.borrowObject();
connection.execute(controlSocket);
Session connection = new Session(controlSocket);
// start the thread
connection.start();
//try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait for debug
activeThreads.put(connection, new Long(System.currentTimeMillis()));
// activeThreads.put(connection, new Long(System.currentTimeMillis()));
//log.logDebug("* NEW SESSION: " + connection.request);
} else {
@ -275,111 +317,267 @@ public class serverCore extends serverAbstractThread implements serverThread {
}
// idle until number of maximal threads is (again) reached
//synchronized(this) {
while ((maxSessions > 0) && (activeThreads.size() >= maxSessions)) try {
log.logDebug("* Waiting for activeThreads=" + activeThreads.size() + " < maxSessions=" + maxSessions);
Thread.currentThread().sleep(2000);
idleThreadCheck();
} catch (InterruptedException e) {}
// while ((maxSessions > 0) && (activeThreads.size() >= maxSessions)) try {
// log.logDebug("* Waiting for activeThreads=" + activeThreads.size() + " < maxSessions=" + maxSessions);
// Thread.currentThread().sleep(2000);
// idleThreadCheck();
// } catch (InterruptedException e) {}
return true;
}
public void close() {
try {
// consuming the isInterrupted Flag. Otherwise we could not properly colse the session pool
Thread.interrupted();
// close the session pool
this.theSessionPool.close();
}
catch (Exception e) {
this.log.logSystem("Unable to close session pool: " + e.getMessage());
}
log.logSystem("* terminated");
}
public int getJobCount() {
return activeThreads.size();
return this.theSessionPool.getNumActive();
}
// idle sensor: the thread is idle if there are no sessions running
public boolean idle() {
idleThreadCheck();
return (activeThreads.size() == 0);
// idleThreadCheck();
return (this.theSessionPool.getNumActive() == 0);
}
public void idleThreadCheck() {
// a 'garbage collector' for session threads
Enumeration threadEnum;
Session session;
// public void idleThreadCheck() {
// // a 'garbage collector' for session threads
// Enumeration threadEnum;
// Session session;
//
// // look for sleeping threads
// threadEnum = activeThreads.keys();
// long time;
// while (threadEnum.hasMoreElements()) {
// session = (Session) (threadEnum.nextElement());
// //if (session.request == null) session.interrupt();
// if (session.isAlive()) {
// // check if socket still exists
// time = System.currentTimeMillis() - ((Long) activeThreads.get(session)).longValue();
// if (/*(session.controlSocket.isClosed()) || */
// (!(session.controlSocket.isBound())) ||
// (!(session.controlSocket.isConnected())) ||
// ((session.request == null) && (time > 1000))) {
// // kick it
// try {
// session.out.close();
// session.in.close();
// session.controlSocket.close();
// } catch (IOException e) {}
// session.interrupt(); // hopefully this wakes him up.
// activeThreads.remove(session);
// String reason = "";
// if (session.controlSocket.isClosed()) reason = "control socked closed";
// if (!(session.controlSocket.isBound())) reason = "control socked unbound";
// if (!(session.controlSocket.isConnected())) reason = "control socked not connected";
// if (session.request == null) reason = "no request placed";
// log.logDebug("* canceled disconnected connection (" + reason + ") '" + session.request + "'");
// } else if (time > thresholdSleep) {
// // move thread from the active threads to the sleeping
// sleepingThreads.put(session, activeThreads.remove(session));
// log.logDebug("* sleeping connection '" + session.request + "'");
// } else if ((time > thresholdActive) && (session.request == null)) {
// // thread is not in use (or too late). kickk it.
// try {
// session.out.close();
// session.in.close();
// session.controlSocket.close();
// } catch (IOException e) {}
// session.interrupt(); // hopefully this wakes him up.
// activeThreads.remove(session);
// log.logDebug("* canceled inactivated connection");
// }
// } else {
// // the thread is dead, remove it
// log.logDebug("* normal close of connection to '" + session.request + "', time=" + session.getTime());
// activeThreads.remove(session);
// }
// }
//
// // look for dead threads
// threadEnum = sleepingThreads.keys();
// while (threadEnum.hasMoreElements()) {
// session = (Session) (threadEnum.nextElement());
// if (session.isAlive()) {
// // check the age of the thread
// if (System.currentTimeMillis() - ((Long) sleepingThreads.get(session)).longValue() > thresholdDead) {
// // kill the thread
// if (termSleepingThreads) {
// try {
// session.out.close();
// session.in.close();
// session.controlSocket.close();
// } catch (IOException e) {}
// session.interrupt(); // hopefully this wakes him up.
// }
// sleepingThreads.remove(session);
// log.logDebug("* out-timed connection '" + session.request + "'");
// }
// } else {
// // the thread is dead, remove it
// sleepingThreads.remove(session);
// log.logDebug("* dead connection '" + session.request + "'");
// }
// }
//
// }
// look for sleeping threads
threadEnum = activeThreads.keys();
long time;
while (threadEnum.hasMoreElements()) {
session = (Session) (threadEnum.nextElement());
//if (session.request == null) session.interrupt();
if (session.isAlive()) {
// check if socket still exists
time = System.currentTimeMillis() - ((Long) activeThreads.get(session)).longValue();
if (/*(session.controlSocket.isClosed()) || */
(!(session.controlSocket.isBound())) ||
(!(session.controlSocket.isConnected())) ||
((session.request == null) && (time > 1000))) {
// kick it
try {
session.out.close();
session.in.close();
session.controlSocket.close();
} catch (IOException e) {}
session.interrupt(); // hopefully this wakes him up.
activeThreads.remove(session);
String reason = "";
if (session.controlSocket.isClosed()) reason = "control socked closed";
if (!(session.controlSocket.isBound())) reason = "control socked unbound";
if (!(session.controlSocket.isConnected())) reason = "control socked not connected";
if (session.request == null) reason = "no request placed";
log.logDebug("* canceled disconnected connection (" + reason + ") '" + session.request + "'");
} else if (time > thresholdSleep) {
// move thread from the active threads to the sleeping
sleepingThreads.put(session, activeThreads.remove(session));
log.logDebug("* sleeping connection '" + session.request + "'");
} else if ((time > thresholdActive) && (session.request == null)) {
// thread is not in use (or too late). kickk it.
try {
session.out.close();
session.in.close();
session.controlSocket.close();
} catch (IOException e) {}
session.interrupt(); // hopefully this wakes him up.
activeThreads.remove(session);
log.logDebug("* canceled inactivated connection");
public final class SessionPool extends GenericObjectPool
{
public boolean isClosed = false;
/**
* First constructor.
* @param objFactory
*/
public SessionPool(SessionFactory objFactory) {
super(objFactory);
this.setMaxIdle(75); // Maximum idle threads.
this.setMaxActive(150); // Maximum active threads.
this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs.
//this.setMaxWait(1000); // Wait 1 second till a thread is available
}
} else {
// the thread is dead, remove it
log.logDebug("* normal close of connection to '" + session.request + "', time=" + session.getTime());
activeThreads.remove(session);
public SessionPool(SessionFactory objFactory,
GenericObjectPool.Config config) {
super(objFactory, config);
}
/**
* @see org.apache.commons.pool.impl.GenericObjectPool#borrowObject()
*/
public Object borrowObject() throws Exception {
return super.borrowObject();
}
// look for dead threads
threadEnum = sleepingThreads.keys();
while (threadEnum.hasMoreElements()) {
session = (Session) (threadEnum.nextElement());
if (session.isAlive()) {
// check the age of the thread
if (System.currentTimeMillis() - ((Long) sleepingThreads.get(session)).longValue() > thresholdDead) {
// kill the thread
if (termSleepingThreads) {
/**
* @see org.apache.commons.pool.impl.GenericObjectPool#returnObject(java.lang.Object)
*/
public void returnObject(Object obj) throws Exception {
super.returnObject(obj);
}
public synchronized void close() throws Exception {
/*
* shutdown all still running session threads ...
*/
// interrupting all still running or pooled threads ...
serverCore.this.theSessionThreadGroup.interrupt();
/* waiting for all threads to finish */
int threadCount = serverCore.this.theSessionThreadGroup.activeCount();
Thread[] threadList = new Thread[threadCount];
threadCount = serverCore.this.theSessionThreadGroup.enumerate(threadList);
try {
session.out.close();
session.in.close();
session.controlSocket.close();
} catch (IOException e) {}
session.interrupt(); // hopefully this wakes him up.
for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
// we need to use a timeout here because of missing interruptable session threads ...
threadList[currentThreadIdx].join(500);
}
sleepingThreads.remove(session);
log.logDebug("* out-timed connection '" + session.request + "'");
}
} else {
// the thread is dead, remove it
sleepingThreads.remove(session);
log.logDebug("* dead connection '" + session.request + "'");
catch (InterruptedException e) {
serverCore.this.log.logWarning("Interruption while trying to shutdown all session threads.");
}
finally {
this.isClosed = true;
}
super.close();
}
}
public final class SessionFactory implements org.apache.commons.pool.PoolableObjectFactory {
final ThreadGroup sessionThreadGroup;
public SessionFactory(ThreadGroup theSessionThreadGroup) {
super();
if (theSessionThreadGroup == null)
throw new IllegalArgumentException("The threadgroup object must not be null.");
this.sessionThreadGroup = theSessionThreadGroup;
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#makeObject()
*/
public Object makeObject() {
return new Session(this.sessionThreadGroup);
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)
*/
public void destroyObject(Object obj) {
if (obj instanceof Session) {
Session theSession = (Session) obj;
theSession.setStopped(true);
}
}
public class Session extends Thread {
/**
* @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object)
*/
public boolean validateObject(Object obj) {
if (obj instanceof Session)
{
Session theSession = (Session) obj;
if (!theSession.isAlive() || theSession.isInterrupted()) return false;
if (theSession.isRunning()) return true;
return false;
}
return true;
}
/**
* @param obj
*
*/
public void activateObject(Object obj) {
//log.debug(" activateObject...");
}
/**
* @param obj
*
*/
public void passivateObject(Object obj) {
//log.debug(" passivateObject..." + obj);
if (obj instanceof Session) {
Session theSession = (Session) obj;
// Clean up the result of the execution
theSession.setResult(null);
}
}
}
public final class Session extends Thread {
// used as replacement for activeThreads, sleepingThreads
// static ThreadGroup sessionThreadGroup = new ThreadGroup("sessionThreadGroup");
// synchronization object needed for the threadpool implementation
private Object syncObject;
private Object processingResult = null;
private boolean running = false;
private boolean stopped = false;
private boolean done = false;
private long start; // startup time
private serverHandler commandObj;
@ -392,29 +590,31 @@ public class serverCore extends serverAbstractThread implements serverThread {
public PushbackInputStream in; // on control input stream
public OutputStream out; // on control output stream, autoflush
public Session(Socket controlSocket) throws IOException {
//this.promiscuous = false;
this.start = System.currentTimeMillis();
//log.logDebug("* session " + handle + " allocated");
this.identity = "-";
this.userAddress = controlSocket.getInetAddress();
String ipname = userAddress.getHostAddress();
// check if we want to allow this socket to connect us
private final ByteArrayOutputStream readLineBuffer = new ByteArrayOutputStream(256);
public Session(ThreadGroup theThreadGroup) {
super(theThreadGroup,"Session");
}
public void setStopped(boolean stopped) {
this.stopped = stopped;
}
public void execute(Socket controlSocket) {
this.execute(controlSocket, null);
}
public synchronized void execute(Socket controlSocket, Object synObj) {
this.controlSocket = controlSocket;
this.in = new PushbackInputStream(controlSocket.getInputStream());
this.out = controlSocket.getOutputStream();
commandCounter = 0;
// initiate the command class
// we pass the input and output stream to the commands,
// so that they can take over communication, if needed
try {
// use the handler prototype to create a new command object class
commandObj = (serverHandler) handlerPrototype.clone();
commandObj.initSession(this);
} catch (Exception e) {
e.printStackTrace();
this.syncObject = synObj;
this.done = false;
if (!this.running) {
// this.setDaemon(true);
this.start();
} else {
this.notifyAll();
}
//log.logDebug("* session " + handle + " initialized. time = " + (System.currentTimeMillis() - handle));
}
public long getTime() {
@ -432,39 +632,144 @@ public class serverCore extends serverAbstractThread implements serverThread {
*/
public void log(boolean outgoing, String request) {
log.logInfo(userAddress.getHostAddress() + "/" + this.identity + " " +
"[" + activeThreads.size() + ", " + commandCounter +
serverCore.this.log.logInfo(userAddress.getHostAddress() + "/" + this.identity + " " +
"[" + serverCore.this.theSessionPool.getNumActive() + ", " + this.commandCounter +
((outgoing) ? "] > " : "] < ") +
request);
}
public void writeLine(String messg) throws IOException {
send(out, messg);
send(this.out, messg);
log(true, messg);
}
public byte[] readLine() {
return receive(in, timeout, commandMaxLength, false);
return receive(in, this.readLineBuffer, timeout, commandMaxLength, false);
}
/**
* @return
*/
public boolean isRunning() {
return this.running;
}
/**
* @param object
*/
public void setResult(Object object) {
this.processingResult = object;
}
/**
*
*/
public void reset() {
this.done = true;
this.syncObject = null;
this.readLineBuffer.reset();
}
/**
*
*
* @see java.lang.Thread#run()
*/
public void run() {
this.running = true;
// The thread keeps running.
while (!this.stopped && !Thread.interrupted()) {
if (this.done) {
// We are waiting for a task now.
synchronized (this) {
try {
this.wait(); //Wait until we get a request to process.
}
catch (InterruptedException e) {
this.stopped = true;
// log.error("", e);
}
}
}
else
{
//There is a task....let us execute it.
try {
execute();
if (this.syncObject != null) {
synchronized (this.syncObject) {
//Notify the completion.
this.syncObject.notifyAll();
}
}
} catch (Exception e) {
// log.error("", e);
}
finally {
reset();
public final void run() {
//log.logDebug("* session " + handle + " started. time = " + (System.currentTimeMillis() - handle));
if (!this.stopped && !this.isInterrupted()) {
try {
serverCore.this.theSessionPool.returnObject(this);
}
catch (Exception e1) {
e1.printStackTrace();
}
}
}
}
}
}
private void execute() {
try {
// setting the session startup time
this.start = System.currentTimeMillis();
// settin the session identity
this.identity = "-";
// getting some client information
this.userAddress = this.controlSocket.getInetAddress();
// TODO: check if we want to allow this socket to connect us
// getting input and output stream for communication with client
this.in = new PushbackInputStream(this.controlSocket.getInputStream());
this.out = this.controlSocket.getOutputStream();
// initiate the command class
this.commandCounter = 0;
if ((this.commandObj != null) &&
(this.commandObj.getClass().getName().equals(serverCore.this.handlerPrototype.getClass().getName()))) {
this.commandObj.reset();
}
else {
this.commandObj = (serverHandler) serverCore.this.handlerPrototype.clone();
}
this.commandObj.initSession(this);
listen();
} catch (Exception e) {
System.err.println("ERROR: (internal) " + e);
} finally {
try {
out.flush();
this.out.flush();
// close everything
out.close();
in.close();
controlSocket.close();
this.out.close();
this.in.close();
this.controlSocket.close();
} catch (IOException e) {
System.err.println("ERROR: (internal) " + e);
}
synchronized (this) {this.notify();}
}
//log.logDebug("* session " + handle + " completed. time = " + (System.currentTimeMillis() - handle));
announceMoreExecTime(System.currentTimeMillis() - start);
announceMoreExecTime(System.currentTimeMillis() - this.start);
}
private void listen() {
@ -486,7 +791,7 @@ public class serverCore extends serverAbstractThread implements serverThread {
String cmd;
String tmp;
Object[] stringParameter = new String[1];
while ((in != null) && ((requestBytes = readLine()) != null)) {
while ((this.in != null) && ((requestBytes = readLine()) != null)) {
commandCounter++;
request = new String(requestBytes);
//log.logDebug("* session " + handle + " received command '" + request + "'. time = " + (System.currentTimeMillis() - handle));
@ -502,7 +807,7 @@ public class serverCore extends serverAbstractThread implements serverThread {
}
// exec command and return value
result = commandObj.getClass().getMethod(cmd, stringType).invoke(commandObj, stringParameter);
result = this.commandObj.getClass().getMethod(cmd, stringType).invoke(this.commandObj, stringParameter);
//log.logDebug("* session " + handle + " completed command '" + request + "'. time = " + (System.currentTimeMillis() - handle));
this.out.flush();
if (result == null) {
@ -555,7 +860,7 @@ public class serverCore extends serverAbstractThread implements serverThread {
// whatever happens: the thread has to survive!
writeLine("UNKNOWN REASON:" + (String) commandObj.error(e));
}
}
} // end of while
} catch (java.lang.ClassNotFoundException e) {
System.out.println("Internal Error: wrapper class not found: " + e.getMessage());
System.exit(0);
@ -566,11 +871,22 @@ public class serverCore extends serverAbstractThread implements serverThread {
}
public static byte[] receive(PushbackInputStream pbis, long timeout, int maxSize, boolean logerr) {
public static byte[] receive(PushbackInputStream pbis, ByteArrayOutputStream readLineBuffer, long timeout, int maxSize, boolean logerr) {
// this is essentially a readln on a PushbackInputStream
int bufferSize = 0;
bufferSize = 10;
// reuse an existing linebuffer or create a new one ...
if (readLineBuffer == null) {
readLineBuffer = new ByteArrayOutputStream(256);
} else {
readLineBuffer.reset();
}
// TODO: we should remove this statements because calling the available function is very time consuming
// we better should use nio sockets instead because they are interruptable ...
try {
long t = timeout;
while (((bufferSize = pbis.available()) == 0) && (t > 0)) try {
@ -590,25 +906,28 @@ public class serverCore extends serverAbstractThread implements serverThread {
return null;
}
byte[] buffer = new byte[bufferSize];
byte[] bufferBkp;
// byte[] buffer = new byte[bufferSize];
// byte[] bufferBkp;
bufferSize = 0;
int b = 0;
try {
while ((b = pbis.read()) > 31) {
// we have a valid byte in b, add it to the buffer
if (buffer.length == bufferSize) {
// the buffer is full, double its size
bufferBkp = buffer;
buffer = new byte[bufferSize * 2];
java.lang.System.arraycopy(bufferBkp, 0, buffer, 0, bufferSize);
bufferBkp = null;
}
//if (bufferSize > 10000) {System.out.println("***ERRORDEBUG***:" + new String(buffer));} // debug
buffer[bufferSize++] = (byte) b; // error hier: ArrayIndexOutOfBoundsException: -2007395416 oder 0
if (bufferSize > maxSize) break;
// // we have a valid byte in b, add it to the buffer
// if (buffer.length == bufferSize) {
// // the buffer is full, double its size
// bufferBkp = buffer;
// buffer = new byte[bufferSize * 2];
// java.lang.System.arraycopy(bufferBkp, 0, buffer, 0, bufferSize);
// bufferBkp = null;
// }
// //if (bufferSize > 10000) {System.out.println("***ERRORDEBUG***:" + new String(buffer));} // debug
// buffer[bufferSize++] = (byte) b; // error hier: ArrayIndexOutOfBoundsException: -2007395416 oder 0
readLineBuffer.write(b);
if (bufferSize++ > maxSize) break;
}
// we have catched a possible line end
if (b == cr) {
// maybe a lf follows, read it:
@ -616,13 +935,14 @@ public class serverCore extends serverAbstractThread implements serverThread {
}
// finally shrink buffer
bufferBkp = buffer;
buffer = new byte[bufferSize];
java.lang.System.arraycopy(bufferBkp, 0, buffer, 0, bufferSize);
bufferBkp = null;
// bufferBkp = buffer;
// buffer = new byte[bufferSize];
// java.lang.System.arraycopy(bufferBkp, 0, buffer, 0, bufferSize);
// bufferBkp = null;
// return only the byte[]
return buffer;
// return buffer;
return readLineBuffer.toByteArray();
} catch (IOException e) {
if (logerr) serverLog.logError("SERVER", "receive interrupted - exception 2 = " + e.getMessage());
return null;
@ -651,4 +971,9 @@ public class serverCore extends serverAbstractThread implements serverThread {
if (bufferSize > 80) return "<LONG STREAM>"; else return new String(buffer);
}
protected void finalize() throws Throwable {
if (!this.theSessionPool.isClosed) this.theSessionPool.close();
super.finalize();
}
}

2
source/de/anomic/server/serverDate.java
Unescape View File

@ -47,7 +47,7 @@ import java.lang.*;
import java.util.*;
import java.text.*;
public class serverDate {
public final class serverDate {
// statics

2
source/de/anomic/server/serverFileUtils.java
Unescape View File

@ -42,7 +42,7 @@ package de.anomic.server;
import java.io.*;
public class serverFileUtils {
public final class serverFileUtils {
public static void copy(InputStream source, OutputStream dest) throws IOException {
byte[] buffer = new byte[4096];

4
source/de/anomic/server/serverHandler.java
Unescape View File

@ -110,4 +110,8 @@ public interface serverHandler {
// but only the necessary one for a newly initialized instance
public Object clone();
// Instead of using clone this function can be used to reset an existing
// handler prototype so that it can e reused
public void reset();
}

2
source/de/anomic/server/serverInstantThread.java
Unescape View File

@ -3,7 +3,7 @@ package de.anomic.server;
import java.lang.reflect.*;
public class serverInstantThread extends serverAbstractThread implements serverThread {
public final class serverInstantThread extends serverAbstractThread implements serverThread {
private Method jobExecMethod, jobCountMethod;
private Object environment;

2
source/de/anomic/server/serverLog.java
Unescape View File

@ -43,7 +43,7 @@ package de.anomic.server;
import java.text.*;
import java.util.*;
public class serverLog {
public final class serverLog {
// statics
private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST");

2
source/de/anomic/server/serverObjects.java
Unescape View File

@ -60,7 +60,7 @@ package de.anomic.server;
import java.io.*;
import java.util.*;
public class serverObjects extends Hashtable implements Cloneable {
public final class serverObjects extends Hashtable implements Cloneable {
public serverObjects() {
super();

3
source/de/anomic/server/serverSystem.java
Unescape View File

@ -41,11 +41,10 @@
package de.anomic.server;
import java.io.*;
import java.net.*;
import java.util.*;
import java.lang.reflect.*;
public class serverSystem {
public final class serverSystem {
// constants for system identification
public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.*

6
source/yacy.java
Unescape View File

@ -75,11 +75,11 @@ import de.anomic.server.*;
import de.anomic.yacy.*;
//import de.anomic.http.*;
public class yacy {
public final class yacy {
// static objects
private static final String vString = "0.361";
private static final String vDATE = "@REPL_DATE@";
private static final String vDATE = "20050419";
private static final String copyright = "[ YACY Proxy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
private static final String hline = "-------------------------------------------------------------------------------";
@ -321,7 +321,7 @@ public class yacy {
httpHeader requestHeader = new httpHeader();
requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify
try {
httpc con = new httpc("localhost", port, 10000, false);
httpc con = httpc.getInstance("localhost", port, 10000, false);
httpc.response res = con.GET("Steering.html?shutdown=", requestHeader);
// read response

Write Preview
Loading…
Cancel
Save