boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fixed non-closing client connections

Browse Source 
- added client connection tracker in connections servelet

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4108 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent d352853f2d
commit 11b4f80bde
12 changed files with 162 additions and 75 deletions
Show Stats Download Patch File Download Diff File

23
htroot/Connections_p.html
Unescape View File

@ -8,6 +8,7 @@
<body id="Connections"> <body id="Connections">
#%env/templates/header.template%# #%env/templates/header.template%#
#%env/templates/submenuAccessTracker.template%# #%env/templates/submenuAccessTracker.template%#
<h2>Server Connection Tracking</h2> <h2>Server Connection Tracking</h2>
<h3>Incoming Connections</h3> <h3>Incoming Connections</h3>
<p>Showing #[numActiveRunning]# active, #[numActivePending]# pending connections from a max. of #[numMax]# allowed incoming connections.</p> <p>Showing #[numActiveRunning]# active, #[numActivePending]# pending connections from a max. of #[numMax]# allowed incoming connections.</p>
@ -34,7 +35,27 @@
#{/list}# #{/list}#
</table> </table>
#%env/templates/footer.template%# <h3>Outgoing Connections</h3>
<p>Showing #[clientActive]# active outgoing connections:</p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td>Protocol</td>
<td>Duration</td>
<td>Dest. IP[:Port]</td>
<td>Command</td>
<td>ID</td>
</tr>
#{clientList}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#">
<td>#[clientProtocol]#</td>
<td>#[clientLifetime]#</td>
<td>#[clientTargetHost]#</td>
<td>#[clientCommand]#</td>
<td>#[clientID]#</td>
</tr>
#{/clientList}#
</table>
#%env/templates/footer.template%#
</body> </body>
</html> </html>

18
htroot/Connections_p.java
Unescape View File

@ -49,11 +49,13 @@
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.Iterator;
import java.util.Properties; import java.util.Properties;
import org.apache.commons.pool.impl.GenericObjectPool; import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.http.httpd; import de.anomic.http.httpd;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
@ -238,6 +240,22 @@ public final class Connections_p {
prop.put("numActiveRunning",Integer.toString(numActiveRunning)); prop.put("numActiveRunning",Integer.toString(numActiveRunning));
prop.put("numActivePending",Integer.toString(numActivePending)); prop.put("numActivePending",Integer.toString(numActivePending));
// client sessions
Iterator i = httpc.activeConnections.iterator();
int c = 0;
while (i.hasNext()) {
httpc clientConnection = (httpc) i.next();
prop.put("clientList_" + c + "_clientProtocol", (clientConnection.ssl) ? "HTTPS" : "HTTP");
prop.put("clientList_" + c + "_clientLifetime", System.currentTimeMillis() - clientConnection.initTime);
prop.put("clientList_" + c + "_clientTargetHost", clientConnection.adressed_host + ":" + clientConnection.adressed_port);
prop.put("clientList_" + c + "_clientCommand", clientConnection.command);
prop.put("clientList_" + c + "_clientID", clientConnection.hashCode());
c++;
}
prop.put("clientList", c);
prop.put("clientActive", c);
// return rewrite values for templates // return rewrite values for templates
return prop; return prop;
} }

1
htroot/CrawlURLFetch_p.java
Unescape View File

@ -531,6 +531,7 @@ public class CrawlURLFetch_p {
if (encoding == null) encoding = "US-ASCII"; if (encoding == null) encoding = "US-ASCII";
r = parseText(new String(sbb.getBytes(), encoding)); r = parseText(new String(sbb.getBytes(), encoding));
} }
con.close();
} catch (IOException e) { } } catch (IOException e) { }
return r; return r;
} }

1
source/de/anomic/data/SitemapParser.java
Unescape View File

@ -209,6 +209,7 @@ public class SitemapParser extends DefaultHandler {
+ "\n\tLength: " + this.contentLength); + "\n\tLength: " + this.contentLength);
SAXParser saxParser = SAXParserFactory.newInstance().newSAXParser(); SAXParser saxParser = SAXParserFactory.newInstance().newSAXParser();
saxParser.parse(this.counterStream, this); saxParser.parse(this.counterStream, this);
remote.close();
} catch (Exception e) { } catch (Exception e) {
this.logger.logWarning("Unable to parse sitemap file " + this.siteMapURL,e); this.logger.logWarning("Unable to parse sitemap file " + this.siteMapURL,e);
} }

5
source/de/anomic/data/robotsParser.java
Unescape View File

@ -413,6 +413,7 @@ public final class robotsParser{
if (!res.responseHeader.mime().startsWith("text/plain")) { if (!res.responseHeader.mime().startsWith("text/plain")) {
robotsTxt = null; robotsTxt = null;
serverLog.logFinest("ROBOTS","Robots.txt from URL '" + robotsURL + "' has wrong mimetype '" + res.responseHeader.mime() + "'."); serverLog.logFinest("ROBOTS","Robots.txt from URL '" + robotsURL + "' has wrong mimetype '" + res.responseHeader.mime() + "'.");
con.close();
} else { } else {
// getting some metadata // getting some metadata
@ -435,10 +436,12 @@ public final class robotsParser{
serverLog.logFinest("ROBOTS","Robots.txt successfully loaded from URL '" + robotsURL + "' in " + (downloadEnd-downloadStart) + " ms."); serverLog.logFinest("ROBOTS","Robots.txt successfully loaded from URL '" + robotsURL + "' in " + (downloadEnd-downloadStart) + " ms.");
} }
} else if (res.status.startsWith("304")) { } else if (res.status.startsWith("304")) {
con.close();
return null; return null;
} else if (res.status.startsWith("3")) { } else if (res.status.startsWith("3")) {
// getting redirection URL // getting redirection URL
String redirectionUrlString = (String) res.responseHeader.get(httpHeader.LOCATION); String redirectionUrlString = (String) res.responseHeader.get(httpHeader.LOCATION);
con.close();
if (redirectionUrlString==null) { if (redirectionUrlString==null) {
serverLog.logFinest("ROBOTS","robots.txt could not be downloaded from URL '" + robotsURL + "' because of missing redirecton header. [" + res.status + "]."); serverLog.logFinest("ROBOTS","robots.txt could not be downloaded from URL '" + robotsURL + "' because of missing redirecton header. [" + res.status + "].");
robotsTxt = null; robotsTxt = null;
@ -455,10 +458,12 @@ public final class robotsParser{
return downloadRobotsTxt(redirectionUrl,redirectionCount,entry); return downloadRobotsTxt(redirectionUrl,redirectionCount,entry);
} else if (res.status.startsWith("401") || res.status.startsWith("403")) { } else if (res.status.startsWith("401") || res.status.startsWith("403")) {
con.close();
accessCompletelyRestricted = true; accessCompletelyRestricted = true;
serverLog.logFinest("ROBOTS","Access to Robots.txt not allowed on URL '" + robotsURL + "'."); serverLog.logFinest("ROBOTS","Access to Robots.txt not allowed on URL '" + robotsURL + "'.");
} else { } else {
serverLog.logFinest("ROBOTS","robots.txt could not be downloaded from URL '" + robotsURL + "'. [" + res.status + "]."); serverLog.logFinest("ROBOTS","robots.txt could not be downloaded from URL '" + robotsURL + "'. [" + res.status + "].");
con.close();
robotsTxt = null; robotsTxt = null;
} }
} catch (Exception e) { } catch (Exception e) {

164
source/de/anomic/http/httpc.java
Unescape View File

@ -60,6 +60,7 @@ import java.util.Date;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.GregorianCalendar; import java.util.GregorianCalendar;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
@ -108,6 +109,8 @@ public final class httpc {
private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US); private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
private static final HashMap reverseMappingCache = new HashMap(); private static final HashMap reverseMappingCache = new HashMap();
private static final HashMap openSocketLookupTable = new HashMap(); private static final HashMap openSocketLookupTable = new HashMap();
public static final HashSet activeConnections = new HashSet(); // all connections are stored here and deleted when they are finished
public static int objCounter = 0; // will be increased with each object and is use to return a hash code
// defined during set-up of switchboard // defined during set-up of switchboard
public static boolean yacyDebugMode = false; public static boolean yacyDebugMode = false;
@ -177,8 +180,8 @@ public final class httpc {
// class variables // class variables
private Socket socket = null; // client socket for commands private Socket socket = null; // client socket for commands
private Thread socketOwner = null; private Thread socketOwner = null;
private String adressed_host = null; public String adressed_host = null;
private int adressed_port = 80; public int adressed_port = 80;
private String target_virtual_host = null; private String target_virtual_host = null;
// output and input streams for client control connection // output and input streams for client control connection
@ -194,6 +197,76 @@ public final class httpc {
private String requestPath = null; private String requestPath = null;
private boolean allowContentEncoding = true; private boolean allowContentEncoding = true;
public boolean ssl;
public long initTime;
public String command;
private int hashIndex;
/**
* Initialize the httpc-instance with the given data.
*
* @param remoteProxyHost
* @param remoteProxyPort
* @throws IOException
*/
public httpc(
String server,
String vhost,
int port,
int timeout,
boolean ssl,
httpRemoteProxyConfig theRemoteProxyConfig,
String incomingByteCountAccounting,
String outgoingByteCountAccounting
) throws IOException {
this.hashIndex = objCounter;
objCounter++;
synchronized (activeConnections) {activeConnections.add(this);}
//System.out.println("*** DEBUG init httpc: " + activeConnections.size() + " connections online");
this.ssl = ssl;
this.initTime = System.currentTimeMillis();
this.command = null;
if ((theRemoteProxyConfig == null) ||
(!theRemoteProxyConfig.useProxy())) {
initN(
server,
vhost,
port,
timeout,
ssl,
incomingByteCountAccounting,
outgoingByteCountAccounting
);
return;
}
if (port == -1) {
port = (ssl)? 443 : 80;
}
String remoteProxyHost = theRemoteProxyConfig.getProxyHost();
int remoteProxyPort = theRemoteProxyConfig.getProxyPort();
this.initN(
remoteProxyHost,
vhost,
remoteProxyPort,
timeout,
ssl,
incomingByteCountAccounting,
outgoingByteCountAccounting);
this.remoteProxyUse = true;
this.adressed_host = server;
this.adressed_port = port;
this.target_virtual_host = vhost;
this.remoteProxyConfig = theRemoteProxyConfig;
}
/** /**
* Convert the status of this class into an String object to output it. * Convert the status of this class into an String object to output it.
@ -253,59 +326,9 @@ public final class httpc {
return new GregorianCalendar(GMTTimeZone).getTime(); return new GregorianCalendar(GMTTimeZone).getTime();
} }
/** public int hashCode() {
* Initialize the httpc-instance with the given data. // return a hash code so it is possible to store objects of httpc objects in a HashSet
* return this.hashIndex;
* @param remoteProxyHost
* @param remoteProxyPort
* @throws IOException
*/
public httpc(
String server,
String vhost,
int port,
int timeout,
boolean ssl,
httpRemoteProxyConfig theRemoteProxyConfig,
String incomingByteCountAccounting,
String outgoingByteCountAccounting
) throws IOException {
if ((theRemoteProxyConfig == null) ||
(!theRemoteProxyConfig.useProxy())) {
initN(
server,
vhost,
port,
timeout,
ssl,
incomingByteCountAccounting,
outgoingByteCountAccounting
);
return;
}
if (port == -1) {
port = (ssl)? 443 : 80;
}
String remoteProxyHost = theRemoteProxyConfig.getProxyHost();
int remoteProxyPort = theRemoteProxyConfig.getProxyPort();
this.initN(
remoteProxyHost,
vhost,
remoteProxyPort,
timeout,
ssl,
incomingByteCountAccounting,
outgoingByteCountAccounting);
this.remoteProxyUse = true;
this.adressed_host = server;
this.adressed_port = port;
this.target_virtual_host = vhost;
this.remoteProxyConfig = theRemoteProxyConfig;
} }
/** /**
@ -404,6 +427,9 @@ public final class httpc {
} }
public void close() { public void close() {
synchronized (activeConnections) {activeConnections.remove(this);}
//System.out.println("*** DEBUG close httpc: " + activeConnections.size() + " connections online");
if (this.clientInput != null) { if (this.clientInput != null) {
try {this.clientInput.close();} catch (Exception e) {} try {this.clientInput.close();} catch (Exception e) {}
this.clientInput = null; this.clientInput = null;
@ -569,6 +595,7 @@ public final class httpc {
*/ */
public response GET(String path, httpHeader requestHeader) throws IOException { public response GET(String path, httpHeader requestHeader) throws IOException {
//serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle)); //serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
this.command = "GET " + path;
try { try {
boolean zipped = (!this.allowContentEncoding) ? false : httpd.shallTransportZipped(path); boolean zipped = (!this.allowContentEncoding) ? false : httpd.shallTransportZipped(path);
send(httpHeader.METHOD_GET, path, requestHeader, zipped); send(httpHeader.METHOD_GET, path, requestHeader, zipped);
@ -592,6 +619,7 @@ public final class httpc {
* @throws IOException * @throws IOException
*/ */
public response HEAD(String path, httpHeader requestHeader) throws IOException { public response HEAD(String path, httpHeader requestHeader) throws IOException {
this.command = "HEAD " + path;
try { try {
send(httpHeader.METHOD_HEAD, path, requestHeader, false); send(httpHeader.METHOD_HEAD, path, requestHeader, false);
return new response(false); return new response(false);
@ -612,6 +640,7 @@ public final class httpc {
* @throws IOException * @throws IOException
*/ */
public response POST(String path, httpHeader requestHeader, InputStream ins) throws IOException { public response POST(String path, httpHeader requestHeader, InputStream ins) throws IOException {
this.command = "POST " + path;
try { try {
send(httpHeader.METHOD_POST, path, requestHeader, false); send(httpHeader.METHOD_POST, path, requestHeader, false);
// if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader // if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader
@ -653,6 +682,7 @@ public final class httpc {
*/ */
public response CONNECT(String remotehost, int remoteport, httpHeader requestHeader) throws IOException { public response CONNECT(String remotehost, int remoteport, httpHeader requestHeader) throws IOException {
this.command = "CONNECT " + remotehost + ":" + remoteport;
try { try {
send(httpHeader.METHOD_CONNECT, remotehost + ":" + remoteport, requestHeader, false); send(httpHeader.METHOD_CONNECT, remotehost + ":" + remoteport, requestHeader, false);
return new response(false); return new response(false);
@ -786,8 +816,7 @@ public final class httpc {
requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password)); requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password));
} }
httpc con = null; httpc con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc.response res = con.GET(path, requestHeader); httpc.response res = con.GET(path, requestHeader);
if (res.status.startsWith("2")) { if (res.status.startsWith("2")) {
@ -795,10 +824,12 @@ public final class httpc {
// stream to byte[] // stream to byte[]
serverByteBuffer sbb = new serverByteBuffer(); serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null); res.writeContent(sbb, null);
con.close();
return sbb.getBytes(); return sbb.getBytes();
} else { } else {
// stream to file and return null // stream to file and return null
res.writeContent(null, download); res.writeContent(null, download);
con.close();
return null; return null;
} }
} }
@ -847,11 +878,16 @@ public final class httpc {
httpc.response res = con.POST(path, requestHeader, props, files); httpc.response res = con.POST(path, requestHeader, props, files);
//System.out.println("response=" + res.toString()); //System.out.println("response=" + res.toString());
if (!(res.status.startsWith("2"))) return res.status.getBytes(); if (!(res.status.startsWith("2"))) {
byte[] status = res.status.getBytes();
con.close();
return status;
}
// read connection body and return body // read connection body and return body
serverByteBuffer sbb = new serverByteBuffer(); serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null); res.writeContent(sbb, null);
con.close();
return sbb.getBytes(); return sbb.getBytes();
} }
@ -993,15 +1029,11 @@ public final class httpc {
String realhost = url.getHost(); String realhost = url.getHost();
// start connection // start connection
httpc con = null; httpc con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc.response res = con.HEAD(path, requestHeader); httpc.response res = con.HEAD(path, requestHeader);
if (res.status.startsWith("2")) { httpHeader h = res.responseHeader;
// success con.close();
return res.responseHeader; return h;
}
// fail
return res.responseHeader;
} }
public static byte[] wput( public static byte[] wput(

8
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -708,6 +708,8 @@ public final class httpdProxyHandler {
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
} }
remote.close();
if (gzippedOut != null) { if (gzippedOut != null) {
gzippedOut.finish(); gzippedOut.finish();
} }
@ -954,11 +956,11 @@ public final class httpdProxyHandler {
// sending the server respond back to the client // sending the server respond back to the client
httpd.sendRespondHeader(conProp,respond,httpVer,res.statusCode,res.statusText,res.responseHeader); httpd.sendRespondHeader(conProp,respond,httpVer,res.statusCode,res.statusText,res.responseHeader);
respond.flush();
remote.close();
} catch (Exception e) { } catch (Exception e) {
handleProxyException(e,remote,conProp,respond,url); handleProxyException(e,remote,conProp,respond,url);
} }
respond.flush();
} }
public static void doPost(Properties conProp, httpHeader requestHeader, OutputStream respond, PushbackInputStream body) throws IOException { public static void doPost(Properties conProp, httpHeader requestHeader, OutputStream respond, PushbackInputStream body) throws IOException {
@ -1144,12 +1146,14 @@ public final class httpdProxyHandler {
// replace connection details // replace connection details
host = switchboard.remoteProxyConfig.getProxyHost(); host = switchboard.remoteProxyConfig.getProxyHost();
port = switchboard.remoteProxyConfig.getProxyPort(); port = switchboard.remoteProxyConfig.getProxyPort();
remoteProxy.close();
// go on (see below) // go on (see below)
} else { } else {
// pass error response back to client // pass error response back to client
httpd.sendRespondHeader(conProp,clientOut,httpVersion,response.statusCode,response.statusText,response.responseHeader); httpd.sendRespondHeader(conProp,clientOut,httpVersion,response.statusCode,response.statusText,response.responseHeader);
//respondHeader(clientOut, response.status, response.responseHeader); //respondHeader(clientOut, response.status, response.responseHeader);
forceConnectionClose(conProp); forceConnectionClose(conProp);
remoteProxy.close();
return; return;
} }
} catch (Exception e) { } catch (Exception e) {

2
source/de/anomic/plasma/crawler/http/CrawlWorker.java
Unescape View File

@ -271,10 +271,12 @@ public final class CrawlWorker extends AbstractCrawlWorker {
// we write the new cache entry to file system directly // we write the new cache entry to file system directly
byte[] cacheArray = null; byte[] cacheArray = null;
cacheArray = res.writeContent(fos,this.keepInMemory); cacheArray = res.writeContent(fos,this.keepInMemory);
remote.close();
htCache.setCacheArray(cacheArray); htCache.setCacheArray(cacheArray);
plasmaHTCache.writeFileAnnouncement(cacheFile); plasmaHTCache.writeFileAnnouncement(cacheFile);
} finally { } finally {
if (fos!=null)try{fos.close();}catch(Exception e){/* ignore this */} if (fos!=null)try{fos.close();}catch(Exception e){/* ignore this */}
remote.close();
} }
// enQueue new entry with response header // enQueue new entry with response header

1
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -409,6 +409,7 @@ public final class plasmaCrawlLURL {
remove(urlHash); remove(urlHash);
log.logInfo("UrlDB-Entry with urlHash '" + urlHash + "' removed\n\tURL: " + oldUrlStr + "\n\tConnection Status: " + res.status); log.logInfo("UrlDB-Entry with urlHash '" + urlHash + "' removed\n\tURL: " + oldUrlStr + "\n\tConnection Status: " + res.status);
} }
theHttpc.close();
} }
} catch (Exception e) { } catch (Exception e) {
remove(urlHash); remove(urlHash);

1
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -932,6 +932,7 @@ public final class plasmaParser {
contentMimeType = res.responseHeader.mime(); contentMimeType = res.responseHeader.mime();
charSet = res.responseHeader.getCharacterEncoding(); charSet = res.responseHeader.getCharacterEncoding();
contentLength = res.responseHeader.contentLength(); contentLength = res.responseHeader.contentLength();
remote.close();
} }
if ((args.length >= 4)&&(args[2].equalsIgnoreCase("-m"))) { if ((args.length >= 4)&&(args[2].equalsIgnoreCase("-m"))) {

2
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -791,9 +791,9 @@ public final class yacySeedDB {
throw new IOException("Server returned status: " + res.status); throw new IOException("Server returned status: " + res.status);
} }
// read byte array // read byte array
byte[] content = serverFileUtils.read(res.getContentInputStream()); byte[] content = serverFileUtils.read(res.getContentInputStream());
remote.close();
// uncompress it if it is gzipped // uncompress it if it is gzipped
content = serverFileUtils.uncompressGZipArray(content); content = serverFileUtils.uncompressGZipArray(content);

1
source/yacy.java
Unescape View File

@ -521,6 +521,7 @@ public final class yacy {
con.close(); con.close();
} else { } else {
serverLog.logSevere("REMOTE-SHUTDOWN", "error response from YACY socket: " + res.status); serverLog.logSevere("REMOTE-SHUTDOWN", "error response from YACY socket: " + res.status);
con.close();
System.exit(-1); System.exit(-1);
} }
} catch (IOException e) { } catch (IOException e) {

Write Preview
Loading…
Cancel
Save