From 09412ea3a4719a525f7ade03b1285a6befef03c0 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Thu, 12 Dec 2013 03:37:19 +0100 Subject: [PATCH] counting search requests in solr interface --- htroot/solr/select.java | 6 +++++ htroot/yacy/seedlist.java | 3 ++- source/net/yacy/peers/Protocol.java | 3 ++- source/net/yacy/peers/RemoteSearch.java | 25 +++++++++++-------- source/net/yacy/peers/SeedDB.java | 6 ++--- source/net/yacy/search/query/SearchEvent.java | 6 ++--- 6 files changed, 30 insertions(+), 19 deletions(-) diff --git a/htroot/solr/select.java b/htroot/solr/select.java index 5c0e497aa..77c99a592 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -152,6 +152,12 @@ public class select { } sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time + // count remote searches if this was part of a p2p search + if (post.containsKey("partitions")) { + final int partitions = post.getInt("partitions", 30); + sb.searchQueriesGlobal += 1.0f / partitions; // increase query counter + } + // get the ranking profile id int profileNr = post.getInt("profileNr", 0); diff --git a/htroot/yacy/seedlist.java b/htroot/yacy/seedlist.java index 8cf4d7971..6821bf8c2 100644 --- a/htroot/yacy/seedlist.java +++ b/htroot/yacy/seedlist.java @@ -46,10 +46,11 @@ public final class seedlist { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE)); + float minversion = Math.min(LISTMAX, post == null ? 0.0f : post.getFloat("minversion", 0.0f)); boolean nodeonly = post == null || !post.containsKey("node") ? false : post.getBoolean("node"); boolean includeme = post == null || !post.containsKey("me") ? true : post.getBoolean("me"); boolean addressonly = post == null || !post.containsKey("address") ? false : post.getBoolean("address"); - final ArrayList v = sb.peers.getSeedlist(maxcount, includeme, nodeonly); + final ArrayList v = sb.peers.getSeedlist(maxcount, includeme, nodeonly, minversion); final serverObjects prop = new serverObjects(); // write simple-encoded seed lines or json diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 3e702d0c4..4d7534c5a 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -1006,13 +1006,14 @@ public final class Protocol { final int offset, final int count, Seed target, + final int partitions, final Blacklist blacklist) { if (event.query.getQueryGoal().getOriginalQueryString(false) == null || event.query.getQueryGoal().getOriginalQueryString(false).length() == 0) { return -1; // we cannot query solr only with word hashes, there is no clear text string } event.addExpectedRemoteReferences(count); - + if (partitions > 0) solrQuery.set("partitions", partitions); solrQuery.setStart(offset); solrQuery.setRows(count); diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java index 9fbbe2f85..a41b43639 100644 --- a/source/net/yacy/peers/RemoteSearch.java +++ b/source/net/yacy/peers/RemoteSearch.java @@ -165,29 +165,30 @@ public class RemoteSearch extends Thread { nodePeers.add(s); } } - - // start solr searches + + if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false)) { + dhtPeers.clear(); + dhtPeers.add(event.peers.mySeed()); + } + if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) { nodePeers.clear(); nodePeers.add(event.peers.mySeed()); } + + // start solr searches + final int targets = dhtPeers.size() + nodePeers.size(); if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) { final SolrQuery solrQuery = event.query.solrQuery(event.getQuery().contentdom, start == 0, event.excludeintext_image); for (Seed s: nodePeers) { - Thread t = solrRemoteSearch(event, solrQuery, start, count, s, blacklist); + Thread t = solrRemoteSearch(event, solrQuery, start, count, s, targets, blacklist); event.nodeSearchThreads.add(t); } } - - if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false)) { - dhtPeers.clear(); - dhtPeers.add(event.peers.mySeed()); - } + // start search to YaCy DHT peers if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false)) { - final int targets = dhtPeers.size(); - if (targets == 0) return; - for (int i = 0; i < targets; i++) { + for (int i = 0; i < dhtPeers.size(); i++) { if (dhtPeers.get(i) == null || dhtPeers.get(i).hash == null) continue; try { RemoteSearch rs = new RemoteSearch( @@ -271,6 +272,7 @@ public class RemoteSearch extends Thread { final int start, final int count, final Seed targetPeer, + final int partitions, final Blacklist blacklist) { assert solrQuery != null; @@ -290,6 +292,7 @@ public class RemoteSearch extends Thread { start, count, targetPeer, + partitions, blacklist); if (urls >= 0) { // urls is an array of url hashes. this is only used for log output diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 96b3b4f5f..2714a7356 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -717,7 +717,7 @@ public final class SeedDB implements AlternativeDomainNames { try { pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile))); - List seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed, false); + List seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed, false, 0.0f); String line; for (Seed seed: seedlist) { line = seed.genSeedStr(null); @@ -731,7 +731,7 @@ public final class SeedDB implements AlternativeDomainNames { return v; } - public ArrayList getSeedlist(int maxcount, boolean addMySeed, boolean nodeonly) { + public ArrayList getSeedlist(int maxcount, boolean addMySeed, boolean nodeonly, float minversion) { final ArrayList v = new ArrayList(this.seedActiveDB.size() + 1000); // store own peer seed @@ -739,7 +739,7 @@ public final class SeedDB implements AlternativeDomainNames { // store active peer seeds Seed ys; - Iterator se = this.seedsConnected(true, false, null, (float) 0.0); + Iterator se = this.seedsConnected(true, false, null, minversion); while (se.hasNext() && v.size() < maxcount) { ys = se.next(); if (ys != null && (!nodeonly || ys.getFlagRootNode())) v.add(ys); diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index e4be9ec6e..483b1aab1 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -287,7 +287,7 @@ public final class SearchEvent { // start a local solr search if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { - this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist); + this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), 0, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist); } this.localsolroffset = this.query.itemsPerPage; @@ -1375,7 +1375,7 @@ public final class SearchEvent { int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded. if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}} if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { - this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist); + this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, nextitems, null /*this peer*/, 0, Switchboard.urlBlacklist); } this.localsolroffset += nextitems; } @@ -1396,7 +1396,7 @@ public final class SearchEvent { if (this.localsolrsearch == null || !this.localsolrsearch.isAlive() && this.local_solr_stored.get() > this.localsolroffset && (item + 1) % this.query.itemsPerPage == 0) { // at the end of a list, trigger a next solr search if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { - this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist); + this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist); } this.localsolroffset += this.query.itemsPerPage; }