boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- more control in access tracker: count number of returned search results (not only info how much is in the index)

Browse Source 
- extended query params for this
- enhanced cora

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7430 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent fafab7a8fe
commit 6c1b14c8e1
13 changed files with 59 additions and 30 deletions
Show Stats Download Patch File Download Diff File

3
htroot/AccessTracker_p.html
Unescape View File

@ -75,6 +75,7 @@
<td>Offset</td>
<td>Expected Results</td>
<td>Returned Results</td>
<td>Known Results</td>
<td>Used Time (ms)</td>
<td>URL fetch (ms)</td>
<td>Snippet comp (ms)</td>
@ -88,6 +89,7 @@
<td></td>
<td></td>
<td>&#216; #[querycount_avg]#</td>
<td>&#216; #[transmitcount_avg]#</td>
<td>&#216; #[resultcount_avg]#</td>
<td>&#216; #[resulttime_avg]#</td>
<td>&#216; #[urltime_avg]#</td>
@ -103,6 +105,7 @@
<td>#[date]#</td>
<td>#[offset]#</td>
<td>#[querycount]#</td>
<td>#[transmitcount]#</td>
<td>#[resultcount]#</td>
<td>#[resulttime]#</td>
<td>#[urltime]#</td>

23
htroot/AccessTracker_p.java
Unescape View File

@ -144,6 +144,7 @@ public class AccessTracker_p {
QueryParams query;
long qcountSum = 0;
long rcountSum = 0;
long tcountSum = 0;
long rcount = 0;
long utimeSum = 0;
long stimeSum = 0;
@ -175,6 +176,7 @@ public class AccessTracker_p {
prop.put("page_list_" + m + "_queryhashes", QueryParams.anonymizedQueryHashes(query.queryHashes));
}
prop.putNum("page_list_" + m + "_querycount", query.itemsPerPage);
prop.putNum("page_list_" + m + "_transmitcount", query.transmitcount);
prop.putNum("page_list_" + m + "_resultcount", query.resultcount);
prop.putNum("page_list_" + m + "_urltime", query.urlretrievaltime);
prop.putNum("page_list_" + m + "_snippettime", query.snippetcomputationtime);
@ -182,6 +184,7 @@ public class AccessTracker_p {
prop.putHTML("page_list_" + m + "_userAgent", query.userAgent);
qcountSum += query.itemsPerPage;
rcountSum += query.resultcount;
tcountSum += query.transmitcount;
utimeSum += query.urlretrievaltime;
stimeSum += query.snippetcomputationtime;
rtimeSum += query.searchtime;
@ -203,24 +206,25 @@ public class AccessTracker_p {
// return empty values to not break the table view
prop.put("page_list", 1);
prop.put("page_list_0_dark", 1 );
prop.put("page_list_0_host", "&nbsp;");
prop.put("page_list_0_date", "&nbsp;");
prop.put("page_list_0_timestamp", "&nbsp;");
prop.put("page_list_0_host", "");
prop.put("page_list_0_date", "");
prop.put("page_list_0_timestamp", "");
if (page == 2) {
// local search
prop.putNum("page_list_0_offset", "");
prop.put("page_list_0_querystring", "");
} else {
// remote search
prop.put("page_list_0_peername", "&nbsp;");
prop.put("page_list_0_queryhashes", "&nbsp;");
prop.put("page_list_0_peername", "");
prop.put("page_list_0_queryhashes", "");
}
prop.putNum("page_list_0_querycount", "");
prop.putNum("page_list_0_transmitcount", "");
prop.putNum("page_list_0_resultcount", "");
prop.putNum("page_list_0_urltime", "");
prop.putNum("page_list_0_snippettime", "");
prop.putNum("page_list_0_resulttime", "");
prop.put("page_list_0_userAgent", "&nbsp;");
prop.put("page_list_0_userAgent", "");
}
if (rcount == 0) rcount = -1;
prop.putNum("page_querycount_avg", (double) qcountSum / m);
@ -228,6 +232,7 @@ public class AccessTracker_p {
prop.putNum("page_urltime_avg", (double) utimeSum / m);
prop.putNum("page_snippettime_avg", (double) stimeSum / m);
prop.putNum("page_resulttime_avg", (double) rtimeSum / m);
prop.putNum("page_transmitcount_avg", (double) tcountSum / rcount);
prop.putNum("page_resultcount_avg1", (double) rcountSum / rcount);
prop.putNum("page_urltime_avg1", (double) utimeSum1 / rcount);
prop.putNum("page_snippettime_avg1", (double) stimeSum1 / rcount);
@ -274,12 +279,12 @@ public class AccessTracker_p {
// return empty values to not break the table view if no results can be listed
if (m==0) {
prop.put("page_list", 1);
prop.put("page_list_0_dates_0_date", "&nbsp;");
prop.put("page_list_0_dates_0_date", "");
prop.put("page_list_0_dates", 1);
prop.putNum("page_list_0_qph", "");
prop.put("page_list_0_dark", 1 );
prop.put("page_list_0_peername", "&nbsp;");
prop.put("page_list_0_host", "&nbsp;");
prop.put("page_list_0_peername", "");
prop.put("page_list_0_host", "");
prop.putNum("page_list_0_count", "");
} else {
prop.put("page_list", m);

1
htroot/AccessTracker_p.xml
Unescape View File

@ -25,6 +25,7 @@
<date timestamp="#[timestamp]#">#[date]#</date>
<offset>#[offset]#</offset>
<querycount>#[querycount]#</querycount>
<transmitcount>#[transmitcount]#</transmitcount>
<resultcount>#[resultcount]#</resultcount>
<resulttime>#[resulttime]#</resulttime>
<urltime>#[urltime]#</urltime>

1
htroot/yacy/search.java
Unescape View File

@ -382,6 +382,7 @@ public final class search {
links.append("resource").append(i).append('=').append(resource).append(serverCore.CRLF_STRING);
}
}
theQuery.transmitcount = accu.size() + 1;
prop.put("links", links.toString());
prop.put("linkcount", accu.size());
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.RESULTLIST, "", accu.size(), System.currentTimeMillis() - timer), false);

2
htroot/yacysearch.rss
Unescape View File

@ -22,11 +22,9 @@
<opensearch:itemsPerPage>#[num-results_itemsPerPage]#</opensearch:itemsPerPage>
<atom:link rel="related" href="opensearchdescription.xml" type="application/opensearchdescription+xml"/>
<opensearch:Query role="request" searchTerms="#[rss_queryenc]#" />
#{results}#
<!--#include virtual="yacysearchitem.xml?item=#[item]#&eventID=#[eventID]#" -->
#{/results}#
<!--#include virtual="yacysearchtrailer.xml?eventID=#[eventID]#" -->
</channel>
</rss>

2
htroot/yacysearch_location.java
Unescape View File

@ -93,7 +93,7 @@ public class yacysearch_location {
// get a queue of search results
String rssSearchServiceURL = "http://localhost:" + sb.getConfig("port", "8080") + "/yacysearch.rss";
BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>();
SearchSRURSS.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, false, false);
SearchSRURSS.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, false, false, null);
// take the results and compute some locations
RSSMessage message;

4
htroot/yacysearchitem.java
Unescape View File

@ -172,7 +172,7 @@ public class yacysearchitem {
} else {
prop.put("content_code", "");
}
theQuery.transmitcount = item + 1;
return prop;
}
@ -203,6 +203,7 @@ public class yacysearchitem {
prop.put("content_item_nl", (item == 0) ? 0 : 1);
prop.put("content_item", 1);
}
theQuery.transmitcount = item + 1;
return prop;
}
@ -232,6 +233,7 @@ public class yacysearchitem {
} else {
prop.put("content_items", "0");
}
theQuery.transmitcount = item + 1;
return prop;
}

3
source/de/anomic/search/QueryParams.java
Unescape View File

@ -99,6 +99,7 @@ public final class QueryParams {
public final Long time;
// values that are set after a search:
public int resultcount; // number of found results
public int transmitcount; // number of results that had been shown to the user
public long searchtime, urlretrievaltime, snippetcomputationtime; // time to perform the search, to get all the urls, and to compute the snippets
public boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options
public final String userAgent;
@ -152,6 +153,7 @@ public final class QueryParams {
this.navigators = "all";
this.indexSegment = indexSegment;
this.userAgent = userAgent;
this.transmitcount = 0;
}
public QueryParams(
@ -206,6 +208,7 @@ public final class QueryParams {
this.specialRights = specialRights;
this.indexSegment = indexSegment;
this.userAgent = userAgent;
this.transmitcount = 0;
}
public Segment getSegment() {

2
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -372,7 +372,7 @@ public final class yacyClient {
public static RSSFeed search(final yacySeed targetSeed, String query, boolean verify, boolean global, long timeout, int startRecord, int maximumRecords) throws IOException {
String address = (targetSeed == null || targetSeed == Switchboard.getSwitchboard().peers.mySeed()) ? "localhost:" + Switchboard.getSwitchboard().getConfig("port", "8080") : targetSeed.getClusterAddress();
String urlBase = "http://" + address + "/yacysearch.rss";
return SearchSRURSS.loadSRURSS(urlBase, query, timeout, startRecord, maximumRecords, verify, global);
return SearchSRURSS.loadSRURSS(urlBase, query, timeout, startRecord, maximumRecords, verify, global, null);
}
@SuppressWarnings("unchecked")

2
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -114,7 +114,7 @@ public class HTTPClient {
HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent);
}
private static HttpClient initConnectionManager() {
public static HttpClient initConnectionManager() {
// Create and initialize HTTP parameters
final HttpParams httpParams = new BasicHttpParams();
/**

16
source/net/yacy/cora/services/SearchHub.java
Unescape View File

@ -33,10 +33,13 @@ import net.yacy.cora.storage.ScoreMap;
public class SearchHub {
private static final String[] SRURSSServicesList = {
//"http://192.168.1.51:8000/yacysearch.rss"//,
"http://localhost:8008/yacysearch.rss"//,
/*
"http://yacy.dyndns.org:8000/yacysearch.rss",
"http://yacy.caloulinux.net:8085/yacysearch.rss",
"http://algire.dyndns.org:8085/yacysearch.rss",
"http://breyvogel.dyndns.org:8002/yacysearch.rss"
"http://breyvogel.dyndns.org:8002/yacysearch.rss"*/
};
public final static SearchHub EMPTY = new SearchHub("", 0);
@ -136,20 +139,23 @@ public class SearchHub {
* @param verify
* @param global
*/
public static void addSRURSSServices(SearchHub search, String[] rssServices, int count, boolean verify, boolean global) {
public static void addSRURSSServices(SearchHub search, String[] rssServices, int count, boolean verify, boolean global, String userAgent) {
for (String service: rssServices) {
SearchSRURSS accumulator = new SearchSRURSS(search, service, count, verify, global);
SearchSRURSS accumulator = new SearchSRURSS(search, service, count, verify, global, userAgent);
accumulator.start();
search.addAccumulator(accumulator);
}
}
public static void main(String[] args) {
HTTPClient.setDefaultUserAgent("searchhub");
HTTPClient.initConnectionManager();
StringBuilder sb = new StringBuilder();
for (String s: args) sb.append(s).append(' ');
String query = sb.toString().trim();
SearchHub search = new SearchHub(query, 10000);
addSRURSSServices(search, SRURSSServicesList, 100, false, false);
addSRURSSServices(search, SRURSSServicesList, 100, false, false, "searchhub");
try {Thread.sleep(100);} catch (InterruptedException e1) {}
search.waitTermination();
ScoreMap<String> result = search.getResults();
@ -159,6 +165,6 @@ public class SearchHub {
u = i.next();
System.out.println("[" + result.get(u) + "] " + u);
}
try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) {}
try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) { e.printStackTrace(); }
}
}

28
source/net/yacy/cora/services/SearchSRURSS.java
Unescape View File

@ -50,6 +50,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
final boolean verify;
final boolean global;
final Map<RSSMessage, List<Integer>> result;
final String userAgent;
private final BlockingQueue<RSSMessage> results;
@ -60,7 +61,8 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
final String urlBase,
final int maximumRecordsInit,
final boolean verify,
final boolean global) {
final boolean global,
final String userAgent) {
this.results = new LinkedBlockingQueue<RSSMessage>();
this.result = result;
this.query = query;
@ -69,6 +71,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
this.maximumRecordsInit = maximumRecordsInit;
this.verify = verify;
this.global = global;
this.userAgent = userAgent;
}
public SearchSRURSS(
@ -76,7 +79,8 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
final String urlBase,
final int maximumRecordsInit,
final boolean verify,
final boolean global) {
final boolean global,
final String userAgent) {
this.results = new LinkedBlockingQueue<RSSMessage>();
this.result = search.getAccumulation();
this.query = search.getQuery();
@ -85,10 +89,11 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
this.maximumRecordsInit = maximumRecordsInit;
this.verify = verify;
this.global = global;
this.userAgent = userAgent;
}
public void run() {
searchSRURSS(results, urlBase, query, timeoutInit, maximumRecordsInit, verify, global);
searchSRURSS(results, urlBase, query, timeoutInit, maximumRecordsInit, verify, global, userAgent);
int p = 1;
RSSMessage message;
try {
@ -111,7 +116,8 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
final long timeoutInit,
final int maximumRecordsInit,
final boolean verify,
final boolean global) {
final boolean global,
final String userAgent) {
Thread job = new Thread() {
public void run() {
int startRecord = 0;
@ -122,8 +128,9 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
long st = System.currentTimeMillis();
RSSFeed feed;
try {
feed = loadSRURSS(urlBase, query, timeout, startRecord, recordsPerSession, verify, global);
feed = loadSRURSS(urlBase, query, timeout, startRecord, recordsPerSession, verify, global, userAgent);
} catch (IOException e1) {
e1.printStackTrace();
break mainloop;
}
if (feed == null || feed.isEmpty()) break mainloop;
@ -134,13 +141,14 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
try {
queue.put(message);
} catch (InterruptedException e) {
e.printStackTrace();
break innerloop;
}
}
startRecord += recordsPerSession;
timeout -= System.currentTimeMillis() - st;
}
try { queue.put(RSSMessage.POISON); } catch (InterruptedException e) {}
try { queue.put(RSSMessage.POISON); } catch (InterruptedException e) { e.printStackTrace(); }
}
};
job.start();
@ -165,7 +173,8 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
int startRecord,
int maximumRecords,
boolean verify,
boolean global) throws IOException {
boolean global,
String userAgent) throws IOException {
MultiProtocolURI uri = null;
try {
uri = new MultiProtocolURI(rssSearchServiceURL);
@ -181,8 +190,9 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
parts.put("maximumRecords", new StringBody(Long.toString(maximumRecords)));
parts.put("verify", new StringBody(verify ? "true" : "false"));
parts.put("resource", new StringBody(global ? "global" : "local"));
final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);
//String debug = new String(result); System.out.println("*** DEBUG: " + debug);
parts.put("nav", new StringBody("none"));
final byte[] result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);
String debug = new String(result); System.out.println("*** DEBUG: " + debug);
final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
if (reader == null) {
throw new IOException("cora.Search failed asking peer '" + uri.getHost() + "': probably bad response from remote peer (1), reader == null");

2
source/net/yacy/repository/LoaderDispatcher.java
Unescape View File

@ -265,7 +265,7 @@ public final class LoaderDispatcher {
if (protocol.equals("ftp")) response = ftpLoader.load(request, true);
if (protocol.equals("smb")) response = smbLoader.load(request, true);
if (protocol.equals("file")) response = fileLoader.load(request, true);
if (response != null) {
if (response != null && response.getContent() != null) {
// we got something. Now check if we want to store that to the cache
// first check looks if we want to store the content to the cache
if (!crawlProfile.storeHTCache()) {

Write Preview
Loading…
Cancel
Save