From b3dca06bb165e608060dcb665ab8eaea07357913 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 22 Dec 2005 01:01:46 +0000 Subject: [PATCH] added location column to network pages. The location is computed from the userAgent string of connecting peers. Therefore this information is not available right after start-up. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1241 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.html | 4 ++ htroot/Network.java | 7 +++ htroot/yacy/hello.java | 2 + source/de/anomic/http/httpc.java | 58 +++++++++++----------- source/de/anomic/yacy/yacyPeerActions.java | 14 +++++- source/de/anomic/yacy/yacySeed.java | 19 +++++-- source/de/anomic/yacy/yacySeedDB.java | 1 - 7 files changed, 70 insertions(+), 35 deletions(-) diff --git a/htroot/Network.html b/htroot/Network.html index 775239bda..529fb8ce4 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -45,6 +45,7 @@ PPM
  Last
Seen
  < > UTC
Offset
  +Location

  Uptime
  < > #Links
  < > #RWIs
  < > @@ -59,6 +60,7 @@ Age
  #Seeds
  con/h

  +user agent

  #(/complete)# #{list}# @@ -74,6 +76,7 @@ #[ppm]# #[lastSeen]# #[utc]# +#[location]# #[uptime]# #[links]# #[words]# @@ -89,6 +92,7 @@ #[age]# #[seeds]# #[connects]# +#[userAgent]# #(/complete)# #{/list}# diff --git a/htroot/Network.java b/htroot/Network.java index 132b3c044..5b6d7e137 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -266,8 +266,10 @@ public class Network { case 3 : e = yacyCore.seedDB.seedsSortedPotential(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break; default: break; } + int p; String startURL; String wikiPage; + String userAgent, location; final StringBuffer alert = new StringBuffer(); int PPM; while (e.hasMoreElements() && conCount < maxCount) { @@ -305,6 +307,10 @@ public class Network { } prop.put(STR_TABLE_LIST + conCount + "_shortname", shortname); prop.put(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink")); + userAgent = yacyCore.peerActions.getUserAgent(seed.getIP()); + p = userAgent.lastIndexOf(';'); + location = (p > 0) ? userAgent.substring(p + 1, userAgent.length() - 1).trim(): ""; + prop.put(STR_TABLE_LIST + conCount + "_location", location); if (complete) { prop.put(STR_TABLE_LIST + conCount + "_complete", 1); prop.put(STR_TABLE_LIST + conCount + "_complete_ip", seed.get(yacySeed.IP, "-") ); @@ -315,6 +321,7 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_complete_CRTCnt", seed.get(yacySeed.CRTCNT, "0")); prop.put(STR_TABLE_LIST + conCount + "_complete_seeds", seed.get(yacySeed.SCOUNT, "-")); prop.put(STR_TABLE_LIST + conCount + "_complete_connects", groupDigits(seed.get(yacySeed.CCOUNT, "0"))); + prop.put(STR_TABLE_LIST + conCount + "_complete_userAgent", userAgent); } else { prop.put(STR_TABLE_LIST + conCount + "_complete", 0); } diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 46e87db0c..adb10efe8 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -86,6 +86,7 @@ public final class hello { // we easily know the caller's IP: final String clientip = (String) header.get("CLIENTIP", ""); // read an artificial header addendum + final String userAgent = (String) header.get(httpHeader.USER_AGENT, ""); final String reportedip = remoteSeed.get(yacySeed.IP, ""); final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); final float clientversion = remoteSeed.getVersion(); @@ -162,6 +163,7 @@ public final class hello { yacyCore.peerActions.peerPing(remoteSeed); } } + yacyCore.peerActions.setUserAgent(clientip, userAgent); if (!((String)prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) { yacyCore.log.logInfo("hello: changing remote peer '" + remoteSeed.getName() + "' [" + reportedip + diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index de148f99f..ad225aae4 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -152,21 +152,23 @@ public final class httpc { private boolean allowContentEncoding = true; static boolean useYacyReferer = true; public static boolean yacyDebugMode = false; - - static { - // set time-out of InetAddress.getByName cache ttl - java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); - java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0"); - } - + /** * Indicates if the current object was removed from pool because the maximum limit * was exceeded. */ boolean removedFromPool = false; - // Configuring the httpc object pool + static SSLSocketFactory theSSLSockFactory = null; + static { + // set time-out of InetAddress.getByName cache ttl + java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); + java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0"); + + + // Configuring the httpc object pool + // implementation of session thread pool GenericObjectPool.Config config = new GenericObjectPool.Config(); @@ -183,11 +185,10 @@ public final class httpc { config.minEvictableIdleTimeMillis = 30000; theHttpcPool = new httpcPool(new httpcFactory(),config); - } - - // initializing a dummy trustManager to enable https connections - static SSLSocketFactory theSSLSockFactory = null; - static { + + + // initializing a dummy trustManager to enable https connections + // Create a trust manager that does not validate certificate chains TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { public java.security.cert.X509Certificate[] getAcceptedIssuers() { @@ -220,8 +221,21 @@ public final class httpc { HttpsURLConnection.setDefaultHostnameVerifier(hv); } catch (Exception e) { } + + + // provide system information for client identification + String loc = System.getProperty("user.timezone", "nowhere"); + int p = loc.indexOf("/"); + if (p > 0) loc = loc.substring(0,p); + loc = loc + "/" + System.getProperty("user.language", "dumb"); + systemOST = + System.getProperty("os.arch", "no-os-arch") + " " + + System.getProperty("os.name", "no-os-name") + " " + + System.getProperty("os.version", "no-os-version") + "; " + + "java " + System.getProperty("java.version", "no-java-version") + "; " + loc; + userAgent = "yacy (www.yacy.net; v" + vDATE + "; " + systemOST + ")"; } - + /** * A reusable readline buffer * @see serverByteBuffer @@ -471,21 +485,7 @@ public final class httpc { return new GregorianCalendar(GMTTimeZone).getTime(); } - // FIXME: Why weren't all static parts put together? They are run one after - // each other on class loading? Hopefully. So why not put them into one - // static block? - static { - // provide system information for client identification - String loc = System.getProperty("user.timezone", "nowhere"); - int p = loc.indexOf("/"); - if (p > 0) loc = loc.substring(0,p); - loc = loc + "/" + System.getProperty("user.language", "dumb"); - systemOST = - System.getProperty("os.arch", "no-os-arch") + " " + System.getProperty("os.name", "no-os-arch") + " " + - System.getProperty("os.version", "no-os-version") + "; " + - "java " + System.getProperty("java.version", "no-java-version") + "; " + loc; - userAgent = "yacy (www.yacy.net; v" + vDATE + "; " + systemOST + ")"; - } + /** * Initialize the httpc-instance with the given data. This method is used, diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index 98e2c53fa..89707b8fb 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -51,6 +51,7 @@ import java.io.InputStreamReader; import java.net.URL; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import de.anomic.http.httpHeader; @@ -67,6 +68,7 @@ public class yacyPeerActions { private yacySeedDB seedDB; private plasmaSwitchboard sb; private HashSet actions; + private HashMap userAgents; private File superseedFile; private String superseedURL; public long juniorConnects; @@ -78,8 +80,9 @@ public class yacyPeerActions { this.seedDB = seedDB; this.sb = switchboard; this.actions = new HashSet(); + this.userAgents = new HashMap(); this.superseedFile = superseedFile; - this.superseedURL = superseedURL; + this.superseedURL = superseedURL; this.superseedURL = superseedURL; this.juniorConnects = 0; this.seniorConnects = 0; @@ -407,4 +410,13 @@ public class yacyPeerActions { Iterator i = actions.iterator(); while (i.hasNext()) ((yacyPeerAction) i.next()).processPeerPing(peer); } + + public void setUserAgent(String IP, String userAgent) { + userAgents.put(IP, userAgent); + } + + public String getUserAgent(String IP) { + String userAgent = (String) userAgents.get(IP); + return (userAgent == null) ? "" : userAgent; + } } diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 45405e807..33b07edcc 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -264,14 +264,25 @@ public class yacySeed { put(URL_IN, Integer.toString(Integer.parseInt(v) + count)); } - // 12 * 6 bit = 72 bit = 9 byte + // 12 * 6 bit = 72 bit = 18 characters hex-hash + public static String b64Hash2hexHash(String b64Hash) { + // the hash string represents 12 * 6 bit = 72 bits. This is too much for a long integer. + return serverCodings.encodeHex(serverCodings.enhancedCoder.decodeBase64(b64Hash)); + } + public static String hexHash2b64Hash(String hexHash) { return serverCodings.enhancedCoder.encodeBase64(serverCodings.decodeHex(hexHash)); } - public static String b64Hash2hexHash(String b64Hash) { - // the hash string represents 12 * 6 bit = 72 bits. This is too much for a long integer. - return serverCodings.encodeHex(serverCodings.enhancedCoder.decodeBase64(b64Hash)); + // 12 * 6 bit = 72 bit = 9 byte + public static byte[] b64Hash2b256Hash(String b64Hash) { + assert (b64Hash.length() == 12); + return serverCodings.enhancedCoder.decodeBase64(b64Hash); + } + + public static String b256Hash2b64Hash(byte[] b256Hash) { + assert (b256Hash.length == 9); + return serverCodings.enhancedCoder.encodeBase64(b256Hash); } public float getVersion() { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 2df99f4cd..cf0cbcc6a 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -95,7 +95,6 @@ public final class yacySeedDB { private final Hashtable nameLookupCache; private final Hashtable ipLookupCache; - public yacySeedDB(plasmaSwitchboard sb, File seedActiveDBFile, File seedPassiveDBFile,