From 84c3fc9d9725485cfda30853fe319c0cd2d0c273 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 17 Nov 2011 01:05:45 +0000 Subject: [PATCH] local/global fixes in search, better abstraction git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8054 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacy/search.java | 8 +-- htroot/yacysearch.java | 4 +- source/net/yacy/search/query/QueryParams.java | 18 +++---- source/net/yacy/search/query/SearchEvent.java | 51 +++++++++---------- 4 files changed, 39 insertions(+), 42 deletions(-) diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 0be2946ad..ab72d7f3a 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -56,10 +56,10 @@ import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.ISO639; import net.yacy.kelondro.util.MemoryControl; -import net.yacy.peers.Seed; import net.yacy.peers.EventChannel; -import net.yacy.peers.Protocol; import net.yacy.peers.Network; +import net.yacy.peers.Protocol; +import net.yacy.peers.Seed; import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -230,7 +230,7 @@ public final class search { count, 0, filter, - QueryParams.SEARCHDOM_LOCAL, + QueryParams.Searchdom.LOCAL, -1, null, false, @@ -290,7 +290,7 @@ public final class search { count, 0, filter, - QueryParams.SEARCHDOM_LOCAL, + QueryParams.Searchdom.LOCAL, -1, constraint, false, diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 2d884f5db..056b3b1f6 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -584,8 +584,8 @@ public class yacysearch { itemsPerPage, offset, urlmask, - (clustersearch && globalsearch) ? QueryParams.SEARCHDOM_CLUSTERALL : - ((globalsearch) ? QueryParams.SEARCHDOM_GLOBALDHT : QueryParams.SEARCHDOM_LOCAL), + (clustersearch && globalsearch) ? QueryParams.Searchdom.CLUSTER : + ((globalsearch) ? QueryParams.Searchdom.GLOBAL : QueryParams.Searchdom.LOCAL), 20, constraint, true, diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 349675fee..c75f4d3a3 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -61,11 +61,9 @@ import net.yacy.search.snippet.ContentDomain; public final class QueryParams { - public static final int SEARCHDOM_LOCAL = 0; - public static final int SEARCHDOM_CLUSTERDHT = 1; - public static final int SEARCHDOM_CLUSTERALL = 2; - public static final int SEARCHDOM_GLOBALDHT = 3; - public static final int SEARCHDOM_GLOBALALL = 4; + public enum Searchdom { + LOCAL, CLUSTER, GLOBAL; + } private static final String ampersand = "&"; @@ -89,7 +87,7 @@ public final class QueryParams { public final ContentDomain contentdom; public final String targetlang; public final String navigators; - public final int domType; + public final Searchdom domType; public final int zonecode; public final int domMaxTargets; public final int maxDistance; @@ -148,7 +146,7 @@ public final class QueryParams { this.itemsPerPage = itemsPerPage; this.offset = 0; this.targetlang = "en"; - this.domType = SEARCHDOM_LOCAL; + this.domType = Searchdom.LOCAL; this.zonecode = DigestURI.TLD_any_zone_filter; this.domMaxTargets = 0; this.constraint = constraint; @@ -178,7 +176,7 @@ public final class QueryParams { final String navigators, final CacheStrategy snippetCacheStrategy, final int itemsPerPage, final int offset, final String urlMask, - final int domType, final int domMaxTargets, + final Searchdom domType, final int domMaxTargets, final Bitfield constraint, final boolean allofconstraint, final String site, final String authorhash, @@ -257,7 +255,7 @@ public final class QueryParams { } public boolean isLocal() { - return this.domType == SEARCHDOM_LOCAL; + return this.domType == Searchdom.LOCAL; } public static HandleSet hashes2Set(final String query) { @@ -471,7 +469,7 @@ public final class QueryParams { context.append(asterisk); context.append(this.maxDistance); context.append(asterisk); - context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()); + context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()); if (anonymized) { this.idCacheAnon = context.toString(); } else { diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 3460a5c14..8638ab4f5 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -103,7 +103,7 @@ public final class SearchEvent { this.peers = peers; this.workTables = workTables; this.query = query; - this.secondarySearchSuperviser = (query.queryHashes.size() > 1) ? new SecondarySearchSuperviser() : null; // generate abstracts only for combined searches + this.secondarySearchSuperviser = (this.query.queryHashes.size() > 1) ? new SecondarySearchSuperviser() : null; // generate abstracts only for combined searches if (this.secondarySearchSuperviser != null) this.secondarySearchSuperviser.start(); this.primarySearchThreads = null; this.secondarySearchThreads = null; @@ -114,9 +114,8 @@ public final class SearchEvent { this.IAmaxcounthash = null; this.IAneardhthash = null; this.localSearchThread = null; - this.order = new ReferenceOrder(query.ranking, UTF8.getBytes(query.targetlang)); - boolean remote = (query.domType == QueryParams.SEARCHDOM_GLOBALDHT) || (query.domType == QueryParams.SEARCHDOM_CLUSTERALL); - if (remote && peers.sizeConnected() == 0) remote = false; + this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang)); + final boolean remote = (this.query.domType == QueryParams.Searchdom.GLOBAL || this.query.domType == QueryParams.Searchdom.CLUSTER) && peers.sizeConnected() > 0 && peers.mySeed().getFlagAcceptRemoteIndex(); final long start = System.currentTimeMillis(); if (remote) { // initialize a ranking process that is the target for data @@ -128,32 +127,32 @@ public final class SearchEvent { // start global searches final long timer = System.currentTimeMillis(); - this.primarySearchThreads = (query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches( - QueryParams.hashSet2hashString(query.queryHashes), - QueryParams.hashSet2hashString(query.excludeHashes), - query.prefer, - query.urlMask, - query.snippetMatcher, - query.targetlang == null ? "" : query.targetlang, - query.sitehash == null ? "" : query.sitehash, - query.authorhash == null ? "" : query.authorhash, + this.primarySearchThreads = (this.query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches( + QueryParams.hashSet2hashString(this.query.queryHashes), + QueryParams.hashSet2hashString(this.query.excludeHashes), + this.query.prefer, + this.query.urlMask, + this.query.snippetMatcher, + this.query.targetlang == null ? "" : this.query.targetlang, + this.query.sitehash == null ? "" : this.query.sitehash, + this.query.authorhash == null ? "" : this.query.authorhash, remote_maxcount, remote_maxtime, - query.maxDistance, - query.getSegment(), + this.query.maxDistance, + this.query.getSegment(), peers, this.rankingProcess, this.secondarySearchSuperviser, Switchboard.urlBlacklist, - query.ranking, - query.constraint, - (query.domType == QueryParams.SEARCHDOM_GLOBALDHT) ? null : preselectedPeerHashes, + this.query.ranking, + this.query.constraint, + (this.query.domType == QueryParams.Searchdom.GLOBAL) ? null : preselectedPeerHashes, burstRobinsonPercent, burstMultiwordPercent); if (this.primarySearchThreads != null) { Log.logFine("SEARCH_EVENT", "STARTING " + this.primarySearchThreads.length + " THREADS TO CATCH EACH " + remote_maxcount + " URLs"); this.rankingProcess.moreFeeders(this.primarySearchThreads.length); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false); // finished searching Log.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + this.primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); } else { @@ -162,7 +161,7 @@ public final class SearchEvent { } // start worker threads to fetch urls and snippets - this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, query, this.peers, this.workTables, 3000, deleteIfSnippetFail); + this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 3000, deleteIfSnippetFail); } else { // do a local search this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation); @@ -192,7 +191,7 @@ public final class SearchEvent { this.IACount.put(wordhash, LargeNumberCache.valueOf(container.size())); this.IAResults.put(wordhash, WordReferenceFactory.compressIndex(container, null, 1000).toString()); } - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false); } else { this.rankingProcess.start(); // start concurrently // but give process time to accumulate a certain amount of data @@ -206,16 +205,16 @@ public final class SearchEvent { } // start worker threads to fetch urls and snippets - this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, query, this.peers, this.workTables, 500, deleteIfSnippetFail); + this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 500, deleteIfSnippetFail); } // clean up events SearchEventCache.cleanupEvents(false); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.CLEANUP, "", 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.CLEANUP, "", 0, 0), false); // store this search to a cache so it can be re-used if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true); - SearchEventCache.put(query.id(false), this); + SearchEventCache.put(this.query.id(false), this); } @@ -372,8 +371,8 @@ public final class SearchEvent { } public ResultEntry oneResult(final int item, final long timeout) { - if ((this.query.domType == QueryParams.SEARCHDOM_GLOBALDHT) || - (this.query.domType == QueryParams.SEARCHDOM_CLUSTERALL)) { + if ((this.query.domType == QueryParams.Searchdom.GLOBAL) || + (this.query.domType == QueryParams.Searchdom.CLUSTER)) { // this is a search using remote search threads. Also the local // search thread is started as background process if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) {