From c8a35a013077267da0cec88748b416355879d685 Mon Sep 17 00:00:00 2001 From: theli Date: Wed, 12 Oct 2005 08:17:43 +0000 Subject: [PATCH] *) Adding new connection tracking page (currently only for incoming connections) *) Displaying statistic for incoming connections on status page *) Bugfix for Loop-Access Bug when trying to access the yacy page while yacy is configured as proxy See: http://www.yacy-forum.de/viewtopic.php?p=6826 *) Bugfix for Referer Bug See: http://www.yacy-forum.de/viewtopic.php?p=11098#11098 *) Adding reverse Name lookup for yacy-domain names (used by the connection tracking page) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@916 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Connections_p.html | 37 ++++ htroot/Connections_p.java | 167 ++++++++++++++++++ htroot/Status.java | 10 ++ htroot/Status_p.inc | 118 +++++++++---- source/de/anomic/http/httpHeader.java | 33 +--- source/de/anomic/http/httpd.java | 85 ++++++++- .../de/anomic/plasma/plasmaCrawlStacker.java | 8 +- .../de/anomic/plasma/plasmaSwitchboard.java | 10 +- source/de/anomic/server/serverCore.java | 59 ++++++- source/de/anomic/yacy/yacyPeerActions.java | 5 + source/de/anomic/yacy/yacySeedDB.java | 107 +++++++++++ 11 files changed, 568 insertions(+), 71 deletions(-) create mode 100644 htroot/Connections_p.html create mode 100644 htroot/Connections_p.java diff --git a/htroot/Connections_p.html b/htroot/Connections_p.html new file mode 100644 index 000000000..c3941206f --- /dev/null +++ b/htroot/Connections_p.html @@ -0,0 +1,37 @@ + + + +YaCy '#[clientname]#': Connection Tracking +#[metas]# + + +#[header]# +

Connection Tracking

+

Incoming Connections

+

Showing #[numActiveRunning]# active connections, #[numActivePending]# pending connections from a max. of #[numMax]# allowed incoming connections.
+ + + + + + + + + +#{list}# + + + + + + + + +#{/list}# +
ProtocolDurationSource IP[:Port]Dest. IP[:Port]CommandUsed
#[proto]##(ms)##[duration]#::#[duration]# ms#(/ms)##[source]##[dest]##(running)#Waiting for new request nr. # #[reqNr]#::#[command]##(/running)##[used]#
+

+ +#[footer]# + + + \ No newline at end of file diff --git a/htroot/Connections_p.java b/htroot/Connections_p.java new file mode 100644 index 000000000..5805a2f40 --- /dev/null +++ b/htroot/Connections_p.java @@ -0,0 +1,167 @@ +//PerformaceQueues_p.java +//----------------------- +//part of YaCy +//(C) by Michael Peter Christen; mc@anomic.de +//first published on http://www.anomic.de +//Frankfurt, Germany, 2004, 2005 +//last major change: 16.02.2005 +// +//This program is free software; you can redistribute it and/or modify +//it under the terms of the GNU General Public License as published by +//the Free Software Foundation; either version 2 of the License, or +//(at your option) any later version. +// +//This program is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. +// +//You should have received a copy of the GNU General Public License +//along with this program; if not, write to the Free Software +//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +//Using this software in any meaning (reading, learning, copying, compiling, +//running) means that you agree that the Author(s) is (are) not responsible +//for cost, loss of data or any harm that may be caused directly or indirectly +//by usage of this softare or this documentation. The usage of this software +//is on your own risk. The installation and usage (starting/running) of this +//software may allow other people or application to access your computer and +//any attached devices and is highly dependent on the configuration of the +//software which must be done by the user of the software; the author(s) is +//(are) also not responsible for proper configuration and usage of the +//software, even if provoked by documentation provided together with +//the software. +// +//Any changes to this file according to the GPL as documented in the file +//gpl.txt aside this file in the shipment you received can be done to the +//lines that follows this copyright notice here, but changes must not be +//done inside the copyright notive above. A re-distribution must contain +//the intact and unchanged copyright notice. +//Contributions and changes to the program code must be marked as such. + +//You must compile this file with +//javac -classpath .:../classes Network.java +//if the shell's current path is HTROOT + +import java.net.InetAddress; +import java.util.Properties; + +import org.apache.commons.pool.impl.GenericObjectPool; + +import de.anomic.http.httpHeader; +import de.anomic.http.httpd; +import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.server.serverCore; +import de.anomic.server.serverDate; +import de.anomic.server.serverHandler; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; +import de.anomic.server.serverThread; +import de.anomic.server.serverCore.Session; +import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacySeed; +import de.anomic.yacy.yacySeedDB; + +public class Connections_p { + + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) { + // return variable that accumulates replacements + plasmaSwitchboard switchboard = (plasmaSwitchboard) sb; + serverObjects prop = new serverObjects(); + + String virtualHost = switchboard.getConfig("fileHost","localhost"); + + serverThread httpd = switchboard.getThread("10_httpd"); + ThreadGroup httpSessions = ((serverCore)httpd).getSessionThreadGroup(); + + GenericObjectPool.Config httpdPoolConfig = ((serverCore)httpd).getPoolConfig(); + + /* waiting for all threads to finish */ + int threadCount = httpSessions.activeCount(); + Thread[] threadList = new Thread[httpdPoolConfig.maxActive]; + threadCount = httpSessions.enumerate(threadList); + + int idx = 0, numActiveRunning = 0, numActivePending = 0, numMax = ((serverCore)httpd).getMaxSessionCount(); + boolean dark = true; + for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) { + Thread currentThread = threadList[currentThreadIdx]; + if ((currentThread != null) && (currentThread instanceof serverCore.Session) && (currentThread.isAlive())) { + // getting the session object + Session currentSession = ((Session)currentThread); + + // getting the session runtime + long sessionTime = currentSession.getTime(); + + // getting the request command line + boolean blockingRequest = false; + String commandLine = currentSession.getCommandLine(); + if (commandLine == null) blockingRequest = true; + int commandCount = currentSession.getCommandCount(); + + // getting the source ip address and port + InetAddress userAddress = currentSession.getUserAddress(); + int userPort = currentSession.getUserPort(); + if (userAddress == null) continue; + + serverHandler cmdObj = currentSession.getCommandObj(); + if (cmdObj instanceof httpd) { + + // getting the http command object + httpd currentHttpd = (httpd)cmdObj; + + // getting the connection properties of this session + Properties conProp = (Properties) currentHttpd.getConProp().clone(); + + // getting the destination host + String dest = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); + if (dest==null)continue; + if (dest.equals(virtualHost)) dest = yacyCore.seedDB.mySeed.getName() + ".yacy"; + + + + // determining if the source is a yacy host + yacySeed seed = yacyCore.seedDB.lookupByIP(userAddress,true,true,true); + if (seed != null) { + if ((seed.hash == yacyCore.seedDB.mySeed.hash) && + (!seed.get("Port","").equals(Integer.toString(userPort)))) { + seed = null; + } + } + + prop.put("list_" + idx + "_dark", ((dark) ? 1 : 0) ); dark=!dark; + prop.put("list_" + idx + "_sessionName",currentSession.getName()); + prop.put("list_" + idx + "_proto","http"); + if (sessionTime > 1000*60) { + prop.put("list_" + idx + "_ms",0); + prop.put("list_" + idx + "_ms_duration",serverDate.intervalToString(sessionTime)); + } else { + prop.put("list_" + idx + "_ms",1); + prop.put("list_" + idx + "_ms_duration",Long.toString(sessionTime)); + } + prop.put("list_" + idx + "_source",(seed!=null)?seed.getName()+".yacy":userAddress.getHostAddress()+":"+userPort); + prop.put("list_" + idx + "_dest",dest); + if (blockingRequest) { + prop.put("list_" + idx + "_running",0); + prop.put("list_" + idx + "_running_reqNr",Integer.toString(commandCount+1)); + numActivePending++; + } else { + prop.put("list_" + idx + "_running",1); + prop.put("list_" + idx + "_running_command",commandLine); + numActiveRunning++; + } + prop.put("list_" + idx + "_used",Integer.toString(commandCount)); + + idx++; + } + } + } + prop.put("list",idx); + + prop.put("numMax",Integer.toString(numMax)); + prop.put("numActiveRunning",Integer.toString(numActiveRunning)); + prop.put("numActivePending",Integer.toString(numActivePending)); + + // return rewrite values for templates + return prop; + } +} diff --git a/htroot/Status.java b/htroot/Status.java index 012fa9369..7c44c6705 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -58,6 +58,7 @@ import de.anomic.server.serverCore; import de.anomic.server.serverDate; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.server.serverThread; import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; @@ -227,6 +228,15 @@ public class Status { prop.put("trafficIn",bytesToString(httpdByteCountInputStream.getGlobalCount())); prop.put("trafficOut",bytesToString(httpdByteCountOutputStream.getGlobalCount())); + // connection information + serverCore httpd = (serverCore) env.getThread("10_httpd"); + int activeSessionCount = httpd.getActiveSessionCount(); + int idleSessionCount = httpd.getIdleSessionCount(); + int maxSessionCount = httpd.getMaxSessionCount(); + prop.put("connectionsActive",Integer.toString(activeSessionCount)); + prop.put("connectionsMax",Integer.toString(maxSessionCount)); + prop.put("connectionsIdle",Integer.toString(idleSessionCount)); + // Queue information final plasmaSwitchboard sb = (plasmaSwitchboard)env; prop.put("indexingQueueSize", Integer.toString(sb.getThread("80_indexing").getJobCount()+sb.indexingTasksInProcess.size())); diff --git a/htroot/Status_p.inc b/htroot/Status_p.inc index abd4ca54f..3268dbb28 100644 --- a/htroot/Status_p.inc +++ b/htroot/Status_p.inc @@ -1,38 +1,88 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Private System Properties
Protection -#(protection)# -Your settings are _not_ protected! Please go to the settings page immediately and set an administration password. -:: -Your settings are protected by a password. -#(/protection)# -
Proxy host#[host]#:#[port]#
Port forwarding host#(portForwarding)#not used::#[host]#:#[port]# (#(status)#broken::connected#(/status)#)#(/portForwarding)#
Remote proxy#(remoteProxy)#not used::#[host]#:#[port]##(/remoteProxy)#
Auto-popup on start-up -#(popup)# -Disabled. To enable this again please use the Settings page -:: -Enabled. To disable this please use the Settings page -#(/popup)# -
Memory Usage - -free: #[freeMemory]# | total: #[totalMemory]# | max: #[maxMemory]# -
Traffic -Out: #[trafficIn]# | In: #[trafficOut]# -
System Resources -Processors: #[processors]# -
Indexing Queue#[indexingQueueSize]# | #[indexingQueueMax]#
Loader Queue#[loaderQueueSize]# | #[loaderQueueMax]# #(loaderPaused)#::(paused)#(/loaderPaused)#
Local Crawler QueueEnqueued: #[localCrawlQueueSize]# | Pending: #[stackCrawlQueueSize]#
Remote Crawler Queue#[remoteCrawlQueueSize]#
Private System Properties
System ResourcesProcessors: #[processors]# 
Protection + #(protection)# + Your settings are _not_ protected! Please go to the settings page immediately and set an administration password. + :: + Your settings are protected by a password. + #(/protection)# +  
Proxy host#[host]#:#[port]# 
Port forwarding host#(portForwarding)#not used::#[host]#:#[port]# (#(status)#broken::connected#(/status)#)#(/portForwarding)# 
Remote proxy#(remoteProxy)#not used::#[host]#:#[port]##(/remoteProxy)# 
Auto-popup on start-up + #(popup)# + Disabled. To enable this again please use the Settings page + :: + Enabled. To disable this please use the Settings page + #(/popup)# +  
Memory Usage + + free: #[freeMemory]# | total: #[totalMemory]# | max: #[maxMemory]# +  
TrafficOut: #[trafficIn]# | In: #[trafficOut]# 
Connections IncomingActive: #[connectionsActive]# | Idle: #[connectionsIdle]# | Max: #[connectionsMax]#[Details]
Indexing Queue#[indexingQueueSize]# | #[indexingQueueMax]#[Details]
Loader Queue#[loaderQueueSize]# | #[loaderQueueMax]# #(loaderPaused)#::(paused)#(/loaderPaused)#[Details]
Local Crawler QueueEnqueued: #[localCrawlQueueSize]# | Pending: #[stackCrawlQueueSize]#[Details]
Remote Crawler Queue#[remoteCrawlQueueSize]#[Details]
\ No newline at end of file diff --git a/source/de/anomic/http/httpHeader.java b/source/de/anomic/http/httpHeader.java index d086c6915..e45d0c533 100644 --- a/source/de/anomic/http/httpHeader.java +++ b/source/de/anomic/http/httpHeader.java @@ -541,7 +541,8 @@ public final class httpHeader extends TreeMap implements Map { } else { // THIS IS THE "GOOD" CASE // a perfect formulated url - prop.setProperty(httpHeader.CONNECTION_PROP_HOST, args.substring(0, sep)); + String dstHostSocket = args.substring(0, sep); + prop.setProperty(httpHeader.CONNECTION_PROP_HOST, (httpd.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket)); prop.setProperty(httpHeader.CONNECTION_PROP_PATH, args.substring(sep)); // yes, including beginning "/" } } else { @@ -699,29 +700,11 @@ public final class httpHeader extends TreeMap implements Map { // if the transparent proxy support was disabled, we have nothing todo here ... if (!(isTransparentProxy && header.containsKey(HOST))) return; - try { - String dstHost, dstHostSocket = (String) header.get(HOST); - - int idx = dstHostSocket.indexOf(":"); - dstHost = (idx != -1) ? dstHostSocket.substring(0,idx).trim() : dstHostSocket.trim(); - Integer dstPort = (idx != -1) ? Integer.valueOf(dstHostSocket.substring(idx+1)) : new Integer(80); - - if (dstPort.intValue() == 80) { - if (dstHost.endsWith(".yacy")) { - // if this peer is accessed via its yacy domain name we need to set the - // host property to virtualHost to redirect the request to the yacy server - if (dstHost.endsWith(yacyCore.seedDB.mySeed.getName()+".yacy")) { - prop.setProperty(CONNECTION_PROP_HOST,virtualHost); - } else { - prop.setProperty(CONNECTION_PROP_HOST,dstHostSocket); - } - } else { - InetAddress dstHostAddress = InetAddress.getByName(dstHost); - if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) { - prop.setProperty(CONNECTION_PROP_HOST,dstHostSocket); - } - } - } - } catch (Exception e) {} + // we only need to do the transparent proxy support if the request URL didn't contain the hostname + // and therefor was set to virtualHost by function parseQuery() + if (!prop.getProperty(CONNECTION_PROP_HOST).equals(virtualHost)) return; + + String dstHostSocket = (String) header.get(httpHeader.HOST); + prop.setProperty(CONNECTION_PROP_HOST,(httpd.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket)); } } \ No newline at end of file diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index 0aab9bc74..c629f7a63 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -148,6 +148,10 @@ public final class httpd implements serverHandler { keepAliveSupport = Boolean.valueOf(switchboard.getConfig("connectionKeepAliveSupport","false")).booleanValue(); } + public Properties getConProp() { + return this.prop; + } + /** * Can be used to reset this {@link serverHandler} oject so that * it can be reused for further connections @@ -782,7 +786,7 @@ public final class httpd implements serverHandler { } // parsing post request bodies which are gzip content-encoded } else { - ByteArrayOutputStream bout = new ByteArrayOutputStream(); + serverByteBuffer bout = new serverByteBuffer(); serverFileUtils.copy(in,bout); buffer = bout.toByteArray(); bout.close(); bout = null; @@ -1310,6 +1314,85 @@ public final class httpd implements serverHandler { // httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS } + public static boolean isThisHostPortForwardingIP(String hostName) { + if ((hostName == null) || (hostName.length() == 0)) return false; + if ((!serverCore.portForwardingEnabled) || (serverCore.portForwarding == null)) return false; + + boolean isThisHostIP = false; + try { + InetAddress hostAddress = InetAddress.getByName(hostName); + InetAddress forwardingAddress = InetAddress.getByName(serverCore.portForwarding.getHost()); + + if (hostAddress.equals(forwardingAddress)) return true; + } catch (Exception e) {} + return isThisHostIP; + } + + public static boolean isThisHostIP(String hostName) { + if ((hostName == null) || (hostName.length() == 0)) return false; + + boolean isThisHostIP = false; + try { + final InetAddress clientAddress = InetAddress.getByName(hostName); + if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; + + final InetAddress[] localAddress = InetAddress.getAllByName(InetAddress.getLocalHost().getHostName()); + for (int i=0; i