boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

local/global fixes in search, better abstraction

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8054 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 13 years ago
parent aca0f33f08
commit 84c3fc9d97
4 changed files with 39 additions and 42 deletions
Show Stats Download Patch File Download Diff File

8
htroot/yacy/search.java
Unescape View File

@ -56,10 +56,10 @@ import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.ISO639; import net.yacy.kelondro.util.ISO639;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.peers.Seed;
import net.yacy.peers.EventChannel; import net.yacy.peers.EventChannel;
import net.yacy.peers.Protocol;
import net.yacy.peers.Network; import net.yacy.peers.Network;
import net.yacy.peers.Protocol;
import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
@ -230,7 +230,7 @@ public final class search {
count, count,
0, 0,
filter, filter,
QueryParams.SEARCHDOM_LOCAL, QueryParams.Searchdom.LOCAL,
-1, -1,
null, null,
false, false,
@ -290,7 +290,7 @@ public final class search {
count, count,
0, 0,
filter, filter,
QueryParams.SEARCHDOM_LOCAL, QueryParams.Searchdom.LOCAL,
-1, -1,
constraint, constraint,
false, false,

4
htroot/yacysearch.java
Unescape View File

@ -584,8 +584,8 @@ public class yacysearch {
itemsPerPage, itemsPerPage,
offset, offset,
urlmask, urlmask,
(clustersearch && globalsearch) ? QueryParams.SEARCHDOM_CLUSTERALL : (clustersearch && globalsearch) ? QueryParams.Searchdom.CLUSTER :
((globalsearch) ? QueryParams.SEARCHDOM_GLOBALDHT : QueryParams.SEARCHDOM_LOCAL), ((globalsearch) ? QueryParams.Searchdom.GLOBAL : QueryParams.Searchdom.LOCAL),
20, 20,
constraint, constraint,
true, true,

18
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -61,11 +61,9 @@ import net.yacy.search.snippet.ContentDomain;
public final class QueryParams { public final class QueryParams {
public static final int SEARCHDOM_LOCAL = 0; public enum Searchdom {
public static final int SEARCHDOM_CLUSTERDHT = 1; LOCAL, CLUSTER, GLOBAL;
public static final int SEARCHDOM_CLUSTERALL = 2; }
public static final int SEARCHDOM_GLOBALDHT = 3;
public static final int SEARCHDOM_GLOBALALL = 4;
private static final String ampersand = "&"; private static final String ampersand = "&";
@ -89,7 +87,7 @@ public final class QueryParams {
public final ContentDomain contentdom; public final ContentDomain contentdom;
public final String targetlang; public final String targetlang;
public final String navigators; public final String navigators;
public final int domType; public final Searchdom domType;
public final int zonecode; public final int zonecode;
public final int domMaxTargets; public final int domMaxTargets;
public final int maxDistance; public final int maxDistance;
@ -148,7 +146,7 @@ public final class QueryParams {
this.itemsPerPage = itemsPerPage; this.itemsPerPage = itemsPerPage;
this.offset = 0; this.offset = 0;
this.targetlang = "en"; this.targetlang = "en";
this.domType = SEARCHDOM_LOCAL; this.domType = Searchdom.LOCAL;
this.zonecode = DigestURI.TLD_any_zone_filter; this.zonecode = DigestURI.TLD_any_zone_filter;
this.domMaxTargets = 0; this.domMaxTargets = 0;
this.constraint = constraint; this.constraint = constraint;
@ -178,7 +176,7 @@ public final class QueryParams {
final String navigators, final String navigators,
final CacheStrategy snippetCacheStrategy, final CacheStrategy snippetCacheStrategy,
final int itemsPerPage, final int offset, final String urlMask, final int itemsPerPage, final int offset, final String urlMask,
final int domType, final int domMaxTargets, final Searchdom domType, final int domMaxTargets,
final Bitfield constraint, final boolean allofconstraint, final Bitfield constraint, final boolean allofconstraint,
final String site, final String site,
final String authorhash, final String authorhash,
@ -257,7 +255,7 @@ public final class QueryParams {
} }
public boolean isLocal() { public boolean isLocal() {
return this.domType == SEARCHDOM_LOCAL; return this.domType == Searchdom.LOCAL;
} }
public static HandleSet hashes2Set(final String query) { public static HandleSet hashes2Set(final String query) {
@ -471,7 +469,7 @@ public final class QueryParams {
context.append(asterisk); context.append(asterisk);
context.append(this.maxDistance); context.append(this.maxDistance);
context.append(asterisk); context.append(asterisk);
context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()); context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());
if (anonymized) { if (anonymized) {
this.idCacheAnon = context.toString(); this.idCacheAnon = context.toString();
} else { } else {

51
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -103,7 +103,7 @@ public final class SearchEvent {
this.peers = peers; this.peers = peers;
this.workTables = workTables; this.workTables = workTables;
this.query = query; this.query = query;
this.secondarySearchSuperviser = (query.queryHashes.size() > 1) ? new SecondarySearchSuperviser() : null; // generate abstracts only for combined searches this.secondarySearchSuperviser = (this.query.queryHashes.size() > 1) ? new SecondarySearchSuperviser() : null; // generate abstracts only for combined searches
if (this.secondarySearchSuperviser != null) this.secondarySearchSuperviser.start(); if (this.secondarySearchSuperviser != null) this.secondarySearchSuperviser.start();
this.primarySearchThreads = null; this.primarySearchThreads = null;
this.secondarySearchThreads = null; this.secondarySearchThreads = null;
@ -114,9 +114,8 @@ public final class SearchEvent {
this.IAmaxcounthash = null; this.IAmaxcounthash = null;
this.IAneardhthash = null; this.IAneardhthash = null;
this.localSearchThread = null; this.localSearchThread = null;
this.order = new ReferenceOrder(query.ranking, UTF8.getBytes(query.targetlang)); this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang));
boolean remote = (query.domType == QueryParams.SEARCHDOM_GLOBALDHT) || (query.domType == QueryParams.SEARCHDOM_CLUSTERALL); final boolean remote = (this.query.domType == QueryParams.Searchdom.GLOBAL || this.query.domType == QueryParams.Searchdom.CLUSTER) && peers.sizeConnected() > 0 && peers.mySeed().getFlagAcceptRemoteIndex();
if (remote && peers.sizeConnected() == 0) remote = false;
final long start = System.currentTimeMillis(); final long start = System.currentTimeMillis();
if (remote) { if (remote) {
// initialize a ranking process that is the target for data // initialize a ranking process that is the target for data
@ -128,32 +127,32 @@ public final class SearchEvent {
// start global searches // start global searches
final long timer = System.currentTimeMillis(); final long timer = System.currentTimeMillis();
this.primarySearchThreads = (query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches( this.primarySearchThreads = (this.query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches(
QueryParams.hashSet2hashString(query.queryHashes), QueryParams.hashSet2hashString(this.query.queryHashes),
QueryParams.hashSet2hashString(query.excludeHashes), QueryParams.hashSet2hashString(this.query.excludeHashes),
query.prefer, this.query.prefer,
query.urlMask, this.query.urlMask,
query.snippetMatcher, this.query.snippetMatcher,
query.targetlang == null ? "" : query.targetlang, this.query.targetlang == null ? "" : this.query.targetlang,
query.sitehash == null ? "" : query.sitehash, this.query.sitehash == null ? "" : this.query.sitehash,
query.authorhash == null ? "" : query.authorhash, this.query.authorhash == null ? "" : this.query.authorhash,
remote_maxcount, remote_maxcount,
remote_maxtime, remote_maxtime,
query.maxDistance, this.query.maxDistance,
query.getSegment(), this.query.getSegment(),
peers, peers,
this.rankingProcess, this.rankingProcess,
this.secondarySearchSuperviser, this.secondarySearchSuperviser,
Switchboard.urlBlacklist, Switchboard.urlBlacklist,
query.ranking, this.query.ranking,
query.constraint, this.query.constraint,
(query.domType == QueryParams.SEARCHDOM_GLOBALDHT) ? null : preselectedPeerHashes, (this.query.domType == QueryParams.Searchdom.GLOBAL) ? null : preselectedPeerHashes,
burstRobinsonPercent, burstRobinsonPercent,
burstMultiwordPercent); burstMultiwordPercent);
if (this.primarySearchThreads != null) { if (this.primarySearchThreads != null) {
Log.logFine("SEARCH_EVENT", "STARTING " + this.primarySearchThreads.length + " THREADS TO CATCH EACH " + remote_maxcount + " URLs"); Log.logFine("SEARCH_EVENT", "STARTING " + this.primarySearchThreads.length + " THREADS TO CATCH EACH " + remote_maxcount + " URLs");
this.rankingProcess.moreFeeders(this.primarySearchThreads.length); this.rankingProcess.moreFeeders(this.primarySearchThreads.length);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false);
// finished searching // finished searching
Log.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + this.primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); Log.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + this.primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
} else { } else {
@ -162,7 +161,7 @@ public final class SearchEvent {
} }
// start worker threads to fetch urls and snippets // start worker threads to fetch urls and snippets
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, query, this.peers, this.workTables, 3000, deleteIfSnippetFail); this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 3000, deleteIfSnippetFail);
} else { } else {
// do a local search // do a local search
this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation); this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation);
@ -192,7 +191,7 @@ public final class SearchEvent {
this.IACount.put(wordhash, LargeNumberCache.valueOf(container.size())); this.IACount.put(wordhash, LargeNumberCache.valueOf(container.size()));
this.IAResults.put(wordhash, WordReferenceFactory.compressIndex(container, null, 1000).toString()); this.IAResults.put(wordhash, WordReferenceFactory.compressIndex(container, null, 1000).toString());
} }
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false);
} else { } else {
this.rankingProcess.start(); // start concurrently this.rankingProcess.start(); // start concurrently
// but give process time to accumulate a certain amount of data // but give process time to accumulate a certain amount of data
@ -206,16 +205,16 @@ public final class SearchEvent {
} }
// start worker threads to fetch urls and snippets // start worker threads to fetch urls and snippets
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, query, this.peers, this.workTables, 500, deleteIfSnippetFail); this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 500, deleteIfSnippetFail);
} }
// clean up events // clean up events
SearchEventCache.cleanupEvents(false); SearchEventCache.cleanupEvents(false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.CLEANUP, "", 0, 0), false); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.CLEANUP, "", 0, 0), false);
// store this search to a cache so it can be re-used // store this search to a cache so it can be re-used
if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true); if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true);
SearchEventCache.put(query.id(false), this); SearchEventCache.put(this.query.id(false), this);
} }
@ -372,8 +371,8 @@ public final class SearchEvent {
} }
public ResultEntry oneResult(final int item, final long timeout) { public ResultEntry oneResult(final int item, final long timeout) {
if ((this.query.domType == QueryParams.SEARCHDOM_GLOBALDHT) || if ((this.query.domType == QueryParams.Searchdom.GLOBAL) ||
(this.query.domType == QueryParams.SEARCHDOM_CLUSTERALL)) { (this.query.domType == QueryParams.Searchdom.CLUSTER)) {
// this is a search using remote search threads. Also the local // this is a search using remote search threads. Also the local
// search thread is started as background process // search thread is started as background process
if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) { if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) {

Write Preview
Loading…
Cancel
Save