boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

changed way how number of search requests are counted:

Browse Source 
so far only search requests at the remote search interface had been counted.
This was done to protect the privacy of searchers, because counting was not done and published at the own search interface.
This caused that no search requests of robinson peers had been counted, becuase they cannot be counted at remote peer.
This change introduces a distinction of locally done search requests at the local search interface from search requests that are on the local interface but had been submitted from a remote IP without authentication.
Now 3 counters are maintained:
- partial count of remote searches
- total count of local searches on robinson peers from non-authenticated clients
- total count of local searches on robinson peers from localhost or authenticated clients
In the global statistic of search requests now the first two counters of the three cases are added
Because we habe a large number of robinson peers with a large number of remote non-authenticated requests the statistic should show at least three times of the number of search requests.


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6696 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 84222e3b4f
commit f175f9a2d3
6 changed files with 53 additions and 20 deletions
Show Stats Download Patch File Download Diff File

20
htroot/Network.html
Unescape View File

@ -233,12 +233,12 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<td>Sent<br />URLs</td>
<td>Received<br />Words</td>
<td>Received<br />URLs</td>
<td>PPM</td>
<td>QPH</td>
<td>total PPM</td>
<td>total QPH</td>
<td>#Seeds</td>
<td>#Connects<br />per hour</td>
<td>Known Seeds</td>
<td>Connects<br />per hour</td>
<td>Indexing PPM</td>
<td>QPH<br />(public&nbsp;local+remote)</td>
<td>QPH<br />(public&nbsp;local)</td>
<td>QPH<br />(remote)</td>
</tr>
<tr class="TableCellDark">
<td>#[my-name]#</td>
@ -253,12 +253,12 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<td align="right">#[my-sU]#</td>
<td align="right">#[my-rI]#</td>
<td align="right">#[my-rU]#</td>
<td align="right">#[my-ppm]#</td>
<td align="right">#[my-qph]#</td>
<td align="right">#[my-totalppm]#</td>
<td align="right">#[my-totalqph]#</td>
<td align="right">#[my-seeds]#</td>
<td align="right">#[my-connects]#</td>
<td align="right">#[my-ppm]#</td>
<td align="right">#[my-qph]#</td>
<td align="right">#[my-qph-publocal]#</td>
<td align="right">#[my-qph-pubremote]#</td>
</tr>
</table>
</div>

8
htroot/Network.java
Unescape View File

@ -126,8 +126,8 @@ public class Network {
prop.put("table_my-rankingreceive", seed.getFlagAcceptCitationReference() ? 1 : 0);
myppm = seed.getPPM();
myqph = 60d * seed.getQPM();
myppm = sb.currentPPM();
myqph = 60d * sb.averageQPM();
prop.put("table_my-version", seed.get(yacySeed.VERSION, "-"));
prop.put("table_my-utc", seed.get(yacySeed.UTC, "-"));
prop.put("table_my-uptime", DateFormatter.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, ""))));
@ -140,8 +140,8 @@ public class Network {
prop.putNum("table_my-rU", Long.parseLong(seed.get(yacySeed.URL_IN, "0")));
prop.putNum("table_my-ppm", myppm);
prop.putNum("table_my-qph", Math.round(100d * myqph) / 100d);
prop.putNum("table_my-totalppm", sb.currentPPM());
prop.putNum("table_my-totalqph", Math.round(6000d * sb.totalQPM) / 100d);
prop.putNum("table_my-qph-publocal", Math.round(6000d * sb.averageQPMPublicLocal()) / 100d);
prop.putNum("table_my-qph-pubremote", Math.round(6000d * sb.averageQPMGlobal()) / 100d);
prop.putNum("table_my-seeds", Long.parseLong(seed.get(yacySeed.SCOUNT, "0")));
prop.putNum("table_my-connects", Double.parseDouble(seed.get(yacySeed.CCOUNT, "0")));
prop.put("table_my-url", seed.get(yacySeed.SEEDLIST, ""));

2
htroot/Network.xml
Unescape View File

@ -74,6 +74,8 @@
<receivedurls>#[my-rU]#</receivedurls>
<ppm>#[my-ppm]#</ppm>
<qph>#[my-qph]#</qph>
<qph-publocal>#[my-qph-publocal]#</qph-publocal>
<qph-pubremote>#[my-qph-pubremote]#</qph-pubremote>
<seeds>#[my-seeds]#</seeds>
<connects>#[my-connects]#</connects>
<location>#[my-location]#</location>

2
htroot/yacy/search.java
Unescape View File

@ -332,7 +332,7 @@ public final class search {
//yacyCore.log.logFine("DEBUG HASH SEARCH: " + indexabstract);
}
}
if (partitions > 0) sb.requestedQueries = sb.requestedQueries + 1d / partitions; // increase query counter
if (partitions > 0) sb.searchQueriesGlobal += 1d / partitions; // increase query counter
// prepare reference hints
final long timer = System.currentTimeMillis();

15
htroot/yacysearch.java
Unescape View File

@ -202,6 +202,21 @@ public class yacysearch {
//if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; }
if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes
// increase search statistic counter
if (!global) {
// we count only searches on the local peer here, because global searches
// are counted on the target peer to preserve privacy of the searcher
if (authenticated) {
// local or authenticated search requests are counted separately
// because they are not part of a public available peer statistic
sb.searchQueriesRobinsonFromLocal++;
} else {
// robinson-searches from non-authenticated requests are public
// and may be part of the public available statistic
sb.searchQueriesRobinsonFromRemote++;
}
}
// find search domain
final ContentDomain contentdom = ContentDomain.contentdomParser((post == null ? "text" : post.get("contentdom", "text")));

26
source/de/anomic/search/Switchboard.java
Unescape View File

@ -208,8 +208,9 @@ public final class Switchboard extends serverSwitch {
public ArrayList<QueryParams> remoteSearches; // array of search result properties as HashMaps
public ConcurrentHashMap<String, TreeSet<Long>> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time)
public long indexedPages = 0;
public double requestedQueries = 0d;
public double totalQPM = 0d;
public int searchQueriesRobinsonFromLocal = 0; // absolute counter of all local queries submitted on this peer from a local or autheticated used
public int searchQueriesRobinsonFromRemote = 0; // absolute counter of all local queries submitted on this peer from a remote IP without authentication
public double searchQueriesGlobal = 0d; // partial counter of remote queries (1/number-of-requested-peers)
public TreeMap<byte[], String> clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used
public URLLicense licensedURLs;
public List<Pattern> networkWhitelist, networkBlacklist;
@ -1999,6 +2000,23 @@ public final class Switchboard extends serverSwitch {
return EventTracker.countEvents("indexed", 20000) * 3;
}
public double averageQPM() {
final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
return (this.searchQueriesRobinsonFromRemote + this.searchQueriesGlobal) * 60d / Math.max(uptime, 1d);
}
public double averageQPMGlobal() {
final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
return (this.searchQueriesGlobal) * 60d / Math.max(uptime, 1d);
}
public double averageQPMPrivateLocal() {
final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
return (this.searchQueriesRobinsonFromLocal) * 60d / Math.max(uptime, 1d);
}
public double averageQPMPublicLocal() {
final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
return (this.searchQueriesRobinsonFromRemote) * 60d / Math.max(uptime, 1d);
}
public String makeDefaultPeerName() {
String name = myPublicIP() + "-" + yacyCore.speedKey + "dpn" + OS.infoKey() + (System.currentTimeMillis() & 99);
name = name.replace('.', '-');
@ -2018,9 +2036,7 @@ public final class Switchboard extends serverSwitch {
//the speed of indexing (pages/minute) of the peer
final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
peers.mySeed().put(yacySeed.ISPEED, Integer.toString(currentPPM()));
totalQPM = requestedQueries * 60d / Math.max(uptime, 1d);
peers.mySeed().put(yacySeed.RSPEED, Double.toString(totalQPM /*Math.max((float) requestcdiff, 0f) * 60f / Math.max((float) uptimediff, 1f)*/ ));
peers.mySeed().put(yacySeed.RSPEED, Double.toString(averageQPM()));
peers.mySeed().put(yacySeed.UPTIME, Long.toString(uptime/60)); // the number of minutes that the peer is up in minutes/day (moving average MA30)
peers.mySeed().put(yacySeed.LCOUNT, Long.toString(indexSegments.URLCount())); // the number of links that the peer has stored (LURL's)
peers.mySeed().put(yacySeed.NCOUNT, Integer.toString(crawlQueues.noticeURL.size())); // the number of links that the peer has noticed, but not loaded (NURL's)

Write Preview
Loading…
Cancel
Save