boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fixed non-closing client connections

Browse Source 
- added client connection tracker in connections servelet

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4108 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent d352853f2d
commit 11b4f80bde
12 changed files with 162 additions and 75 deletions
Show Stats Download Patch File Download Diff File

27
htroot/Connections_p.html
Unescape View File

@ -7,7 +7,8 @@
</head>
<body id="Connections">
#%env/templates/header.template%#
#%env/templates/submenuAccessTracker.template%#
#%env/templates/submenuAccessTracker.template%#
<h2>Server Connection Tracking</h2>
<h3>Incoming Connections</h3>
<p>Showing #[numActiveRunning]# active, #[numActivePending]# pending connections from a max. of #[numMax]# allowed incoming connections.</p>
@ -33,8 +34,28 @@
</tr>
#{/list}#
</table>
<h3>Outgoing Connections</h3>
<p>Showing #[clientActive]# active outgoing connections:</p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td>Protocol</td>
<td>Duration</td>
<td>Dest. IP[:Port]</td>
<td>Command</td>
<td>ID</td>
</tr>
#{clientList}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#">
<td>#[clientProtocol]#</td>
<td>#[clientLifetime]#</td>
<td>#[clientTargetHost]#</td>
<td>#[clientCommand]#</td>
<td>#[clientID]#</td>
</tr>
#{/clientList}#
</table>
#%env/templates/footer.template%#
</body>
</html>

18
htroot/Connections_p.java
Unescape View File

@ -49,11 +49,13 @@
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URLEncoder;
import java.util.Iterator;
import java.util.Properties;
import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.http.httpd;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
@ -238,6 +240,22 @@ public final class Connections_p {
prop.put("numActiveRunning",Integer.toString(numActiveRunning));
prop.put("numActivePending",Integer.toString(numActivePending));
// client sessions
Iterator i = httpc.activeConnections.iterator();
int c = 0;
while (i.hasNext()) {
httpc clientConnection = (httpc) i.next();
prop.put("clientList_" + c + "_clientProtocol", (clientConnection.ssl) ? "HTTPS" : "HTTP");
prop.put("clientList_" + c + "_clientLifetime", System.currentTimeMillis() - clientConnection.initTime);
prop.put("clientList_" + c + "_clientTargetHost", clientConnection.adressed_host + ":" + clientConnection.adressed_port);
prop.put("clientList_" + c + "_clientCommand", clientConnection.command);
prop.put("clientList_" + c + "_clientID", clientConnection.hashCode());
c++;
}
prop.put("clientList", c);
prop.put("clientActive", c);
// return rewrite values for templates
return prop;
}

1
htroot/CrawlURLFetch_p.java
Unescape View File

@ -531,6 +531,7 @@ public class CrawlURLFetch_p {
if (encoding == null) encoding = "US-ASCII";
r = parseText(new String(sbb.getBytes(), encoding));
}
con.close();
} catch (IOException e) { }
return r;
}

1
source/de/anomic/data/SitemapParser.java
Unescape View File

@ -209,6 +209,7 @@ public class SitemapParser extends DefaultHandler {
+ "\n\tLength: " + this.contentLength);
SAXParser saxParser = SAXParserFactory.newInstance().newSAXParser();
saxParser.parse(this.counterStream, this);
remote.close();
} catch (Exception e) {
this.logger.logWarning("Unable to parse sitemap file " + this.siteMapURL,e);
}

5
source/de/anomic/data/robotsParser.java
Unescape View File

@ -413,6 +413,7 @@ public final class robotsParser{
if (!res.responseHeader.mime().startsWith("text/plain")) {
robotsTxt = null;
serverLog.logFinest("ROBOTS","Robots.txt from URL '" + robotsURL + "' has wrong mimetype '" + res.responseHeader.mime() + "'.");
con.close();
} else {
// getting some metadata
@ -435,10 +436,12 @@ public final class robotsParser{
serverLog.logFinest("ROBOTS","Robots.txt successfully loaded from URL '" + robotsURL + "' in " + (downloadEnd-downloadStart) + " ms.");
}
} else if (res.status.startsWith("304")) {
con.close();
return null;
} else if (res.status.startsWith("3")) {
// getting redirection URL
String redirectionUrlString = (String) res.responseHeader.get(httpHeader.LOCATION);
con.close();
if (redirectionUrlString==null) {
serverLog.logFinest("ROBOTS","robots.txt could not be downloaded from URL '" + robotsURL + "' because of missing redirecton header. [" + res.status + "].");
robotsTxt = null;
@ -455,10 +458,12 @@ public final class robotsParser{
return downloadRobotsTxt(redirectionUrl,redirectionCount,entry);
} else if (res.status.startsWith("401") || res.status.startsWith("403")) {
con.close();
accessCompletelyRestricted = true;
serverLog.logFinest("ROBOTS","Access to Robots.txt not allowed on URL '" + robotsURL + "'.");
} else {
serverLog.logFinest("ROBOTS","robots.txt could not be downloaded from URL '" + robotsURL + "'. [" + res.status + "].");
con.close();
robotsTxt = null;
}
} catch (Exception e) {

170
source/de/anomic/http/httpc.java
Unescape View File

@ -60,6 +60,7 @@ import java.util.Date;
import java.util.Enumeration;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
@ -108,6 +109,8 @@ public final class httpc {
private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
private static final HashMap reverseMappingCache = new HashMap();
private static final HashMap openSocketLookupTable = new HashMap();
public static final HashSet activeConnections = new HashSet(); // all connections are stored here and deleted when they are finished
public static int objCounter = 0; // will be increased with each object and is use to return a hash code
// defined during set-up of switchboard
public static boolean yacyDebugMode = false;
@ -177,8 +180,8 @@ public final class httpc {
// class variables
private Socket socket = null; // client socket for commands
private Thread socketOwner = null;
private String adressed_host = null;
private int adressed_port = 80;
public String adressed_host = null;
public int adressed_port = 80;
private String target_virtual_host = null;
// output and input streams for client control connection
@ -194,6 +197,76 @@ public final class httpc {
private String requestPath = null;
private boolean allowContentEncoding = true;
public boolean ssl;
public long initTime;
public String command;
private int hashIndex;
/**
* Initialize the httpc-instance with the given data.
*
* @param remoteProxyHost
* @param remoteProxyPort
* @throws IOException
*/
public httpc(
String server,
String vhost,
int port,
int timeout,
boolean ssl,
httpRemoteProxyConfig theRemoteProxyConfig,
String incomingByteCountAccounting,
String outgoingByteCountAccounting
) throws IOException {
this.hashIndex = objCounter;
objCounter++;
synchronized (activeConnections) {activeConnections.add(this);}
//System.out.println("*** DEBUG init httpc: " + activeConnections.size() + " connections online");
this.ssl = ssl;
this.initTime = System.currentTimeMillis();
this.command = null;
if ((theRemoteProxyConfig == null) ||
(!theRemoteProxyConfig.useProxy())) {
initN(
server,
vhost,
port,
timeout,
ssl,
incomingByteCountAccounting,
outgoingByteCountAccounting
);
return;
}
if (port == -1) {
port = (ssl)? 443 : 80;
}
String remoteProxyHost = theRemoteProxyConfig.getProxyHost();
int remoteProxyPort = theRemoteProxyConfig.getProxyPort();
this.initN(
remoteProxyHost,
vhost,
remoteProxyPort,
timeout,
ssl,
incomingByteCountAccounting,
outgoingByteCountAccounting);
this.remoteProxyUse = true;
this.adressed_host = server;
this.adressed_port = port;
this.target_virtual_host = vhost;
this.remoteProxyConfig = theRemoteProxyConfig;
}
/**
* Convert the status of this class into an String object to output it.
@ -252,62 +325,12 @@ public final class httpc {
public static Date nowDate() {
return new GregorianCalendar(GMTTimeZone).getTime();
}
/**
* Initialize the httpc-instance with the given data.
*
* @param remoteProxyHost
* @param remoteProxyPort
* @throws IOException
*/
public httpc(
String server,
String vhost,
int port,
int timeout,
boolean ssl,
httpRemoteProxyConfig theRemoteProxyConfig,
String incomingByteCountAccounting,
String outgoingByteCountAccounting
) throws IOException {
if ((theRemoteProxyConfig == null) ||
(!theRemoteProxyConfig.useProxy())) {
initN(
server,
vhost,
port,
timeout,
ssl,
incomingByteCountAccounting,
outgoingByteCountAccounting
);
return;
}
if (port == -1) {
port = (ssl)? 443 : 80;
}
String remoteProxyHost = theRemoteProxyConfig.getProxyHost();
int remoteProxyPort = theRemoteProxyConfig.getProxyPort();
this.initN(
remoteProxyHost,
vhost,
remoteProxyPort,
timeout,
ssl,
incomingByteCountAccounting,
outgoingByteCountAccounting);
this.remoteProxyUse = true;
this.adressed_host = server;
this.adressed_port = port;
this.target_virtual_host = vhost;
this.remoteProxyConfig = theRemoteProxyConfig;
public int hashCode() {
// return a hash code so it is possible to store objects of httpc objects in a HashSet
return this.hashIndex;
}
/**
* Initialize the https-instance with the given data. Opens the sockets to
* the remote server and creats input and output streams.
@ -404,6 +427,9 @@ public final class httpc {
}
public void close() {
synchronized (activeConnections) {activeConnections.remove(this);}
//System.out.println("*** DEBUG close httpc: " + activeConnections.size() + " connections online");
if (this.clientInput != null) {
try {this.clientInput.close();} catch (Exception e) {}
this.clientInput = null;
@ -569,7 +595,8 @@ public final class httpc {
*/
public response GET(String path, httpHeader requestHeader) throws IOException {
//serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
try {
this.command = "GET " + path;
try {
boolean zipped = (!this.allowContentEncoding) ? false : httpd.shallTransportZipped(path);
send(httpHeader.METHOD_GET, path, requestHeader, zipped);
response r = new response(zipped);
@ -592,6 +619,7 @@ public final class httpc {
* @throws IOException
*/
public response HEAD(String path, httpHeader requestHeader) throws IOException {
this.command = "HEAD " + path;
try {
send(httpHeader.METHOD_HEAD, path, requestHeader, false);
return new response(false);
@ -612,6 +640,7 @@ public final class httpc {
* @throws IOException
*/
public response POST(String path, httpHeader requestHeader, InputStream ins) throws IOException {
this.command = "POST " + path;
try {
send(httpHeader.METHOD_POST, path, requestHeader, false);
// if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader
@ -653,6 +682,7 @@ public final class httpc {
*/
public response CONNECT(String remotehost, int remoteport, httpHeader requestHeader) throws IOException {
this.command = "CONNECT " + remotehost + ":" + remoteport;
try {
send(httpHeader.METHOD_CONNECT, remotehost + ":" + remoteport, requestHeader, false);
return new response(false);
@ -786,8 +816,7 @@ public final class httpc {
requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password));
}
httpc con = null;
con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc.response res = con.GET(path, requestHeader);
if (res.status.startsWith("2")) {
@ -795,10 +824,12 @@ public final class httpc {
// stream to byte[]
serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null);
con.close();
return sbb.getBytes();
} else {
// stream to file and return null
res.writeContent(null, download);
con.close();
return null;
}
}
@ -847,11 +878,16 @@ public final class httpc {
httpc.response res = con.POST(path, requestHeader, props, files);
//System.out.println("response=" + res.toString());
if (!(res.status.startsWith("2"))) return res.status.getBytes();
if (!(res.status.startsWith("2"))) {
byte[] status = res.status.getBytes();
con.close();
return status;
}
// read connection body and return body
serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null);
con.close();
return sbb.getBytes();
}
@ -993,15 +1029,11 @@ public final class httpc {
String realhost = url.getHost();
// start connection
httpc con = null;
con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc.response res = con.HEAD(path, requestHeader);
if (res.status.startsWith("2")) {
// success
return res.responseHeader;
}
// fail
return res.responseHeader;
httpHeader h = res.responseHeader;
con.close();
return h;
}
public static byte[] wput(

8
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -708,6 +708,8 @@ public final class httpdProxyHandler {
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
}
remote.close();
if (gzippedOut != null) {
gzippedOut.finish();
}
@ -954,11 +956,11 @@ public final class httpdProxyHandler {
// sending the server respond back to the client
httpd.sendRespondHeader(conProp,respond,httpVer,res.statusCode,res.statusText,res.responseHeader);
respond.flush();
remote.close();
} catch (Exception e) {
handleProxyException(e,remote,conProp,respond,url);
}
respond.flush();
}
public static void doPost(Properties conProp, httpHeader requestHeader, OutputStream respond, PushbackInputStream body) throws IOException {
@ -1144,12 +1146,14 @@ public final class httpdProxyHandler {
// replace connection details
host = switchboard.remoteProxyConfig.getProxyHost();
port = switchboard.remoteProxyConfig.getProxyPort();
remoteProxy.close();
// go on (see below)
} else {
// pass error response back to client
httpd.sendRespondHeader(conProp,clientOut,httpVersion,response.statusCode,response.statusText,response.responseHeader);
//respondHeader(clientOut, response.status, response.responseHeader);
forceConnectionClose(conProp);
remoteProxy.close();
return;
}
} catch (Exception e) {

2
source/de/anomic/plasma/crawler/http/CrawlWorker.java
Unescape View File

@ -271,10 +271,12 @@ public final class CrawlWorker extends AbstractCrawlWorker {
// we write the new cache entry to file system directly
byte[] cacheArray = null;
cacheArray = res.writeContent(fos,this.keepInMemory);
remote.close();
htCache.setCacheArray(cacheArray);
plasmaHTCache.writeFileAnnouncement(cacheFile);
} finally {
if (fos!=null)try{fos.close();}catch(Exception e){/* ignore this */}
remote.close();
}
// enQueue new entry with response header

1
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -409,6 +409,7 @@ public final class plasmaCrawlLURL {
remove(urlHash);
log.logInfo("UrlDB-Entry with urlHash '" + urlHash + "' removed\n\tURL: " + oldUrlStr + "\n\tConnection Status: " + res.status);
}
theHttpc.close();
}
} catch (Exception e) {
remove(urlHash);

1
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -932,6 +932,7 @@ public final class plasmaParser {
contentMimeType = res.responseHeader.mime();
charSet = res.responseHeader.getCharacterEncoding();
contentLength = res.responseHeader.contentLength();
remote.close();
}
if ((args.length >= 4)&&(args[2].equalsIgnoreCase("-m"))) {

2
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -791,9 +791,9 @@ public final class yacySeedDB {
throw new IOException("Server returned status: " + res.status);
}
// read byte array
byte[] content = serverFileUtils.read(res.getContentInputStream());
remote.close();
// uncompress it if it is gzipped
content = serverFileUtils.uncompressGZipArray(content);

1
source/yacy.java
Unescape View File

@ -521,6 +521,7 @@ public final class yacy {
con.close();
} else {
serverLog.logSevere("REMOTE-SHUTDOWN", "error response from YACY socket: " + res.status);
con.close();
System.exit(-1);
}
} catch (IOException e) {

Write Preview
Loading…
Cancel
Save