From f175f9a2d33d4e1b9bdd412f76fe8b4759c12a75 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 24 Feb 2010 13:53:55 +0000 Subject: [PATCH] changed way how number of search requests are counted: so far only search requests at the remote search interface had been counted. This was done to protect the privacy of searchers, because counting was not done and published at the own search interface. This caused that no search requests of robinson peers had been counted, becuase they cannot be counted at remote peer. This change introduces a distinction of locally done search requests at the local search interface from search requests that are on the local interface but had been submitted from a remote IP without authentication. Now 3 counters are maintained: - partial count of remote searches - total count of local searches on robinson peers from non-authenticated clients - total count of local searches on robinson peers from localhost or authenticated clients In the global statistic of search requests now the first two counters of the three cases are added Because we habe a large number of robinson peers with a large number of remote non-authenticated requests the statistic should show at least three times of the number of search requests. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6696 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.html | 20 +++++++++--------- htroot/Network.java | 8 ++++---- htroot/Network.xml | 2 ++ htroot/yacy/search.java | 2 +- htroot/yacysearch.java | 15 ++++++++++++++ source/de/anomic/search/Switchboard.java | 26 +++++++++++++++++++----- 6 files changed, 53 insertions(+), 20 deletions(-) diff --git a/htroot/Network.html b/htroot/Network.html index 1860c65f6..ddd5e4ea9 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -233,12 +233,12 @@ To see a list of all APIs, please visit the #[my-name]# @@ -253,12 +253,12 @@ To see a list of all APIs, please visit the #[my-sU]# #[my-rI]# #[my-rU]# - #[my-ppm]# - #[my-qph]# - #[my-totalppm]# - #[my-totalqph]# #[my-seeds]# #[my-connects]# + #[my-ppm]# + #[my-qph]# + #[my-qph-publocal]# + #[my-qph-pubremote]# diff --git a/htroot/Network.java b/htroot/Network.java index 9ec7a158d..886a13eb4 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -126,8 +126,8 @@ public class Network { prop.put("table_my-rankingreceive", seed.getFlagAcceptCitationReference() ? 1 : 0); - myppm = seed.getPPM(); - myqph = 60d * seed.getQPM(); + myppm = sb.currentPPM(); + myqph = 60d * sb.averageQPM(); prop.put("table_my-version", seed.get(yacySeed.VERSION, "-")); prop.put("table_my-utc", seed.get(yacySeed.UTC, "-")); prop.put("table_my-uptime", DateFormatter.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "")))); @@ -140,8 +140,8 @@ public class Network { prop.putNum("table_my-rU", Long.parseLong(seed.get(yacySeed.URL_IN, "0"))); prop.putNum("table_my-ppm", myppm); prop.putNum("table_my-qph", Math.round(100d * myqph) / 100d); - prop.putNum("table_my-totalppm", sb.currentPPM()); - prop.putNum("table_my-totalqph", Math.round(6000d * sb.totalQPM) / 100d); + prop.putNum("table_my-qph-publocal", Math.round(6000d * sb.averageQPMPublicLocal()) / 100d); + prop.putNum("table_my-qph-pubremote", Math.round(6000d * sb.averageQPMGlobal()) / 100d); prop.putNum("table_my-seeds", Long.parseLong(seed.get(yacySeed.SCOUNT, "0"))); prop.putNum("table_my-connects", Double.parseDouble(seed.get(yacySeed.CCOUNT, "0"))); prop.put("table_my-url", seed.get(yacySeed.SEEDLIST, "")); diff --git a/htroot/Network.xml b/htroot/Network.xml index 1fe03d485..51b66e30d 100644 --- a/htroot/Network.xml +++ b/htroot/Network.xml @@ -74,6 +74,8 @@ #[my-rU]# #[my-ppm]# #[my-qph]# + #[my-qph-publocal]# + #[my-qph-pubremote]# #[my-seeds]# #[my-connects]# #[my-location]# diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index ded76570f..a2af194c9 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -332,7 +332,7 @@ public final class search { //yacyCore.log.logFine("DEBUG HASH SEARCH: " + indexabstract); } } - if (partitions > 0) sb.requestedQueries = sb.requestedQueries + 1d / partitions; // increase query counter + if (partitions > 0) sb.searchQueriesGlobal += 1d / partitions; // increase query counter // prepare reference hints final long timer = System.currentTimeMillis(); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index a3789c57f..e231e9953 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -202,6 +202,21 @@ public class yacysearch { //if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; } if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes + // increase search statistic counter + if (!global) { + // we count only searches on the local peer here, because global searches + // are counted on the target peer to preserve privacy of the searcher + if (authenticated) { + // local or authenticated search requests are counted separately + // because they are not part of a public available peer statistic + sb.searchQueriesRobinsonFromLocal++; + } else { + // robinson-searches from non-authenticated requests are public + // and may be part of the public available statistic + sb.searchQueriesRobinsonFromRemote++; + } + } + // find search domain final ContentDomain contentdom = ContentDomain.contentdomParser((post == null ? "text" : post.get("contentdom", "text"))); diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index 9c92c51bb..d7df31d16 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -208,8 +208,9 @@ public final class Switchboard extends serverSwitch { public ArrayList remoteSearches; // array of search result properties as HashMaps public ConcurrentHashMap> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time) public long indexedPages = 0; - public double requestedQueries = 0d; - public double totalQPM = 0d; + public int searchQueriesRobinsonFromLocal = 0; // absolute counter of all local queries submitted on this peer from a local or autheticated used + public int searchQueriesRobinsonFromRemote = 0; // absolute counter of all local queries submitted on this peer from a remote IP without authentication + public double searchQueriesGlobal = 0d; // partial counter of remote queries (1/number-of-requested-peers) public TreeMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used public URLLicense licensedURLs; public List networkWhitelist, networkBlacklist; @@ -1999,6 +2000,23 @@ public final class Switchboard extends serverSwitch { return EventTracker.countEvents("indexed", 20000) * 3; } + public double averageQPM() { + final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000; + return (this.searchQueriesRobinsonFromRemote + this.searchQueriesGlobal) * 60d / Math.max(uptime, 1d); + } + public double averageQPMGlobal() { + final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000; + return (this.searchQueriesGlobal) * 60d / Math.max(uptime, 1d); + } + public double averageQPMPrivateLocal() { + final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000; + return (this.searchQueriesRobinsonFromLocal) * 60d / Math.max(uptime, 1d); + } + public double averageQPMPublicLocal() { + final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000; + return (this.searchQueriesRobinsonFromRemote) * 60d / Math.max(uptime, 1d); + } + public String makeDefaultPeerName() { String name = myPublicIP() + "-" + yacyCore.speedKey + "dpn" + OS.infoKey() + (System.currentTimeMillis() & 99); name = name.replace('.', '-'); @@ -2018,9 +2036,7 @@ public final class Switchboard extends serverSwitch { //the speed of indexing (pages/minute) of the peer final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000; peers.mySeed().put(yacySeed.ISPEED, Integer.toString(currentPPM())); - totalQPM = requestedQueries * 60d / Math.max(uptime, 1d); - peers.mySeed().put(yacySeed.RSPEED, Double.toString(totalQPM /*Math.max((float) requestcdiff, 0f) * 60f / Math.max((float) uptimediff, 1f)*/ )); - + peers.mySeed().put(yacySeed.RSPEED, Double.toString(averageQPM())); peers.mySeed().put(yacySeed.UPTIME, Long.toString(uptime/60)); // the number of minutes that the peer is up in minutes/day (moving average MA30) peers.mySeed().put(yacySeed.LCOUNT, Long.toString(indexSegments.URLCount())); // the number of links that the peer has stored (LURL's) peers.mySeed().put(yacySeed.NCOUNT, Integer.toString(crawlQueues.noticeURL.size())); // the number of links that the peer has noticed, but not loaded (NURL's)