added a monitor page for search requests

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3206 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
(no author) 18 years ago
parent 454f182ba3
commit fe72b772cf

@ -235,7 +235,7 @@ public class DetailedSearch {
plasmaSearchRankingProfile localRanking = new plasmaSearchRankingProfile("local", post.toString());
plasmaSearchTimingProfile localTiming = new plasmaSearchTimingProfile(4 * thisSearch.maximumTime / 10, thisSearch.wantedResults);
plasmaSearchTimingProfile remoteTiming = new plasmaSearchTimingProfile(6 * thisSearch.maximumTime / 10, thisSearch.wantedResults);
final serverObjects prop = sb.searchFromLocal(thisSearch, localRanking, localTiming, remoteTiming, postsort);
final serverObjects prop = sb.searchFromLocal(thisSearch, localRanking, localTiming, remoteTiming, postsort, (String) header.get("CLIENTIP"));
putRanking(prop, localRanking, "local");
// remember the last search expression

@ -0,0 +1,37 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Local Search Statistics</title>
#%env/templates/metas.template%#
</head>
<body id="LocalSearchStatistics">
#%env/templates/header.template%#
#%env/templates/submenuSearchStatistics.template%#
<h2>Search Statistics: Local Searches</h2>
<p>This is a list of searches that had been requested from this' peer search interface</p>
<p>Showing #[num]# entries from a total of #[total]# Requests.</p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader">
<td>Requesting Host</td>
<td>Date</td>
<td>Expected Results</td>
<td>Returned Results</td>
<td>Expected Time</td>
<td>Used Time</td>
<td>Search Words</td>
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#">
<td>#[host]#</td>
<td>#[date]#</td>
<td>#[querycount]#</td>
<td>#[resultcount]#</td>
<td>#[querytime]#</td>
<td>#[resulttime]#</td>
<td>#[querywords]#</td>
</tr>
#{/list}#
</table>
#%env/templates/footer.template%#
</body>
</html>

@ -0,0 +1,82 @@
// SearchStatisticsLocal_p.java
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 14.01.2007 on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class SearchStatisticsLocal_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
int maxCount = 100;
int entCount = 0;
boolean dark = true;
Iterator i = switchboard.localSearches.entrySet().iterator();
Map.Entry entry;
Long trackerHandle;
HashMap searchProfile;
while ((entCount < maxCount) && (i.hasNext())) {
entry = (Map.Entry) i.next();
trackerHandle = (Long) entry.getKey();
searchProfile = (HashMap) entry.getValue();
Iterator wi = ((Set) searchProfile.get("querywords")).iterator();
StringBuffer a = new StringBuffer();
while (wi.hasNext()) a.append((String) wi.next()).append(' ');
// put values in template
prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.put("list_" + entCount + "_host", (String) searchProfile.get("host"));
prop.put("list_" + entCount + "_date", (new Date(trackerHandle.longValue())).toString());
prop.put("list_" + entCount + "_querywords", new String(a));
prop.put("list_" + entCount + "_querycount", ((Integer) searchProfile.get("querycount")).toString());
prop.put("list_" + entCount + "_querytime", ((Long) searchProfile.get("querytime")).toString());
prop.put("list_" + entCount + "_resultcount", ((Integer) searchProfile.get("resultcount")).toString());
prop.put("list_" + entCount + "_resulttime", ((Long) searchProfile.get("resulttime")).toString());
// next
entCount++;
}
prop.put("list", entCount);
prop.put("num", entCount);
prop.put("total", switchboard.localSearches.size());
// return rewrite properties
return prop;
}
}

@ -0,0 +1,37 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Remote Search Statistics</title>
#%env/templates/metas.template%#
</head>
<body id="RemoteSearchStatistics">
#%env/templates/header.template%#
#%env/templates/submenuSearchStatistics.template%#
<h2>Search Statistics: Remote Searches</h2>
<p>This is a list of searches that had been requested from remote peer search interface</p>
<p>Showing #[num]# entries from a total of #[total]# Requests.</p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader">
<td>Requesting Host</td>
<td>Date</td>
<td>Expected Results</td>
<td>Returned Results</td>
<td>Expected Time</td>
<td>Used Time</td>
<td>Search Words</td>
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#">
<td>#[host]#</td>
<td>#[date]#</td>
<td>#[querycount]#</td>
<td>#[resultcount]#</td>
<td>#[querytime]#</td>
<td>#[resulttime]#</td>
<td>#[queryhashes]#</td>
</tr>
#{/list}#
</table>
#%env/templates/footer.template%#
</body>
</html>

@ -0,0 +1,80 @@
// SearchStatisticsRemote_p.java
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 14.01.2007 on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class SearchStatisticsRemote_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
int maxCount = 100;
int entCount = 0;
boolean dark = true;
Iterator i = switchboard.remoteSearches.entrySet().iterator();
Map.Entry entry;
Long trackerHandle;
HashMap searchProfile;
while ((entCount < maxCount) && (i.hasNext())) {
entry = (Map.Entry) i.next();
trackerHandle = (Long) entry.getKey();
searchProfile = (HashMap) entry.getValue();
// put values in template
prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.put("list_" + entCount + "_host", (String) searchProfile.get("host"));
prop.put("list_" + entCount + "_date", (new Date(trackerHandle.longValue())).toString());
prop.put("list_" + entCount + "_queryhashes", plasmaSearchQuery.hashSet2hashString((Set) searchProfile.get("queryhashes")));
prop.put("list_" + entCount + "_querycount", ((Integer) searchProfile.get("querycount")).toString());
prop.put("list_" + entCount + "_querytime", ((Long) searchProfile.get("querytime")).toString());
prop.put("list_" + entCount + "_resultcount", ((Integer) searchProfile.get("resultcount")).toString());
prop.put("list_" + entCount + "_resulttime", ((Long) searchProfile.get("resulttime")).toString());
// next
entCount++;
}
prop.put("list", entCount);
prop.put("num", entCount);
prop.put("total", switchboard.localSearches.size());
// return rewrite properties
return prop;
}
}

@ -10,6 +10,7 @@
<li><a href="/Surftips.html?display=1" class="MenuItemLink">Surftips</a></li>
<li><a href="/Bookmarks.html" class="MenuItemLink">Bookmarks</a></li>
<li><a href="/Help.html" class="MenuItemLink">Help</a></li>
<li><a href="/SearchStatisticsLocal_p.html" class="MenuItemLink lock">Search Statistics</a></li>
</ul>
</li>
<li class="menugroup" id="menugroupCrawlerControl">

@ -0,0 +1,7 @@
<div class="SubMenu">
<h3>Cookie Menu</h3>
<ul class="SubMenu">
<li><a href="/SearchStatisticsLocal_p.html" class="MenuItemLink lock">Local&nbsp;Searches</a></li>
<li><a href="/SearchStatisticsRemote_p.html" class="MenuItemLink lock">Remote&nbsp;Searches</a></li>
</ul>
</div>

@ -47,9 +47,11 @@
// javac -classpath .:../../Classes search.java
// if the shell's current path is htroot/yacy
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroBitfield;
@ -136,6 +138,7 @@ public final class search {
int joincount = 0;
plasmaSearchResult acc = null;
plasmaSearchQuery squery = null;
plasmaSearchEvent theSearch = null;
if ((query.length() == 0) && (abstractSet != null)) {
// this is _not_ a normal search, only a request for index abstracts
squery = new plasmaSearchQuery(abstractSet, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), count, duetime, filter, plasmaSearchQuery.catchall_constraint);
@ -147,7 +150,7 @@ public final class search {
plasmaSearchTimingProfile localTiming = new plasmaSearchTimingProfile(squery.maximumTime, squery.wantedResults);
plasmaSearchTimingProfile remoteTiming = null;
plasmaSearchEvent theSearch = new plasmaSearchEvent(squery, rankingProfile, localTiming, remoteTiming, true, yacyCore.log, sb.wordIndex, sb.wordIndex.loadedURL, sb.snippetCache);
theSearch = new plasmaSearchEvent(squery, rankingProfile, localTiming, remoteTiming, true, yacyCore.log, sb.wordIndex, sb.wordIndex.loadedURL, sb.snippetCache);
Map containers = theSearch.localSearchContainers(plasmaSearchQuery.hashes2Set(urls));
if (containers != null) {
Iterator ci = containers.entrySet().iterator();
@ -174,7 +177,7 @@ public final class search {
plasmaSearchTimingProfile localTiming = new plasmaSearchTimingProfile(squery.maximumTime, squery.wantedResults);
plasmaSearchTimingProfile remoteTiming = null;
plasmaSearchEvent theSearch = new plasmaSearchEvent(squery,
theSearch = new plasmaSearchEvent(squery,
rankingProfile, localTiming, remoteTiming, true,
yacyCore.log, sb.wordIndex, sb.wordIndex.loadedURL,
sb.snippetCache);
@ -242,6 +245,17 @@ public final class search {
}
prop.put("indexabstract", indexabstract.toString());
// prepare search statistics
Long trackerHandle = new Long(System.currentTimeMillis());
HashMap searchProfile = theSearch.resultProfile();
String client = (String) header.get("CLIENTIP");
searchProfile.put("host", client);
sb.remoteSearches.put(trackerHandle, searchProfile);
TreeSet handles = (TreeSet) sb.remoteSearchTracker.get(client);
if (handles == null) handles = new TreeSet();
handles.add(trackerHandle);
sb.remoteSearchTracker.put(client, handles);
// prepare result
if ((joincount == 0) || (acc == null)) {

@ -55,7 +55,6 @@ import java.util.TreeSet;
import de.anomic.data.wikiCode;
import de.anomic.htmlFilter.htmlFilterImageEntry;
import de.anomic.http.httpHeader;
import de.anomic.http.httpd;
import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroMSetTools;
@ -259,7 +258,7 @@ public class yacysearch {
plasmaSearchRankingProfile ranking = (sb.getConfig("rankingProfile", "").length() == 0) ? new plasmaSearchRankingProfile(contentdomString) : new plasmaSearchRankingProfile("", crypt.simpleDecode(sb.getConfig("rankingProfile", ""), null));
plasmaSearchTimingProfile localTiming = new plasmaSearchTimingProfile(4 * thisSearch.maximumTime / 10, thisSearch.wantedResults);
plasmaSearchTimingProfile remoteTiming = new plasmaSearchTimingProfile(6 * thisSearch.maximumTime / 10, thisSearch.wantedResults);
prop = sb.searchFromLocal(thisSearch, ranking, localTiming, remoteTiming, true);
prop = sb.searchFromLocal(thisSearch, ranking, localTiming, remoteTiming, true, (String) header.get("CLIENTIP"));
// remember the last search expression
env.setConfig("last-search", querystring + contentdomString);

@ -76,6 +76,8 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
private plasmaSearchTimingProfile profileLocal, profileGlobal;
private boolean postsort;
private yacySearch[] primarySearchThreads, secondarySearchThreads;
private long searchtime;
private int searchcount;
public plasmaSearchEvent(plasmaSearchQuery query,
plasmaSearchRankingProfile ranking,
@ -100,6 +102,8 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
this.postsort = postsort;
this.primarySearchThreads = null;
this.secondarySearchThreads = null;
this.searchtime = -1;
this.searchcount = -1;
}
public plasmaSearchQuery getQuery() {
@ -117,21 +121,30 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
return secondarySearchThreads;
}
public HashMap resultProfile() {
// generate statistics about search: query, time, etc
HashMap r = new HashMap();
r.put("queryhashes", query.queryHashes);
r.put("querywords", query.queryWords);
r.put("querycount", new Integer(query.wantedResults));
r.put("querytime", new Long(query.maximumTime));
r.put("resultcount", new Integer(this.searchcount));
r.put("resulttime", new Long(this.searchtime));
return r;
}
public plasmaSearchResult search() {
// combine all threads
// we synchronize with flushThreads to allow only one local search at a time,
// so all search tasks are queued
synchronized (flushThreads) {
long start = System.currentTimeMillis();
if (query.domType == plasmaSearchQuery.SEARCHDOM_GLOBALDHT) {
int fetchpeers = (int) (query.maximumTime / 500L); // number of target peers; means 10 peers in 10 seconds
if (fetchpeers > 50) fetchpeers = 50;
if (fetchpeers < 30) fetchpeers = 30;
// remember time
long start = System.currentTimeMillis();
// do a global search
// the result of the fetch is then in the rcGlobal
log.logFine("STARTING " + fetchpeers + " THREADS TO CATCH EACH " + profileGlobal.getTargetCount(plasmaSearchTimingProfile.PROCESS_POSTSORT) + " URLs WITHIN " + (profileGlobal.duetime() / 1000) + " SECONDS");
@ -206,6 +219,8 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
// return search result
log.logFine("SEARCHRESULT: " + profileLocal.reportToString());
lastEvent = this;
this.searchtime = System.currentTimeMillis() - start;
this.searchcount = result.filteredResults;
return result;
} else {
Map searchContainerMap = localSearchContainers(null);
@ -216,6 +231,8 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
// return search result
log.logFine("SEARCHRESULT: " + profileLocal.reportToString());
lastEvent = this;
this.searchtime = System.currentTimeMillis() - start;
this.searchcount = result.filteredResults;
return result;
}
}

@ -119,6 +119,7 @@ import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.data.blogBoard;
@ -239,6 +240,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public dbImportManager dbImportManager;
public plasmaDHTFlush transferIdxThread = null;
private plasmaDHTChunk dhtTransferChunk = null;
public TreeMap localSearches, remoteSearches;
public HashMap localSearchTracker, remoteSearchTracker;
/*
* Remote Proxy configuration
@ -556,6 +559,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.outgoingCookies = new HashMap();
this.incomingCookies = new HashMap();
// init search history trackers
this.localSearchTracker = new HashMap(); // String:TreeSet - IP:set of Long(accessTime)
this.remoteSearchTracker = new HashMap();
this.localSearches = new TreeMap(); // Long:HashMap - Long(accessTime):properties
this.remoteSearches = new TreeMap();
// init messages: clean up message symbol
File notifierSource = new File(getRootPath(), getConfig("htRootPath", "htroot") + "/env/grafics/empty.gif");
File notifierDest = new File(getConfig("htDocsPath", "DATA/HTDOCS"), "notifier.gif");
@ -2074,7 +2083,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
plasmaSearchRankingProfile ranking,
plasmaSearchTimingProfile localTiming,
plasmaSearchTimingProfile remoteTiming,
boolean postsort) {
boolean postsort,
String client) {
// tell all threads to do nothing for a specific time
intermissionAllThreads(2 * query.maximumTime);
@ -2243,6 +2253,18 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
prop.get("num-results_orderedcount", "0") + " links ordered, " +
prop.get("num-results_linkcount", "?") + " links selected, " +
((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
// prepare search statistics
Long trackerHandle = new Long(System.currentTimeMillis());
HashMap searchProfile = theSearch.resultProfile();
searchProfile.put("host", client);
this.localSearches.put(trackerHandle, searchProfile);
TreeSet handles = (TreeSet) this.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet();
handles.add(trackerHandle);
this.localSearchTracker.put(client, handles);
return prop;
} catch (IOException e) {
return null;

Loading…
Cancel
Save