diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 0a913b540..0c788b31f 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -629,8 +629,8 @@ public class yacysearch { // log Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " + "local-unfiltered(" + theSearch.getRankingResult().getLocalIndexCount() + "), " + - "-local_miss(" + theSearch.getRankingResult().getMissCount() + "), " + - "-local_sortout(" + theSearch.getRankingResult().getSortOutCount() + "), " + + "local_miss(" + theSearch.getRankingResult().getMissCount() + "), " + + "local_sortout(" + theSearch.getRankingResult().getSortOutCount() + "), " + "remote(" + theSearch.getRankingResult().getRemoteResourceSize() + ") links found, " + (System.currentTimeMillis() - timestamp) + " ms"); diff --git a/source/net/yacy/search/query/RWIProcess.java b/source/net/yacy/search/query/RWIProcess.java index 74afb784e..ce2b98224 100644 --- a/source/net/yacy/search/query/RWIProcess.java +++ b/source/net/yacy/search/query/RWIProcess.java @@ -153,11 +153,9 @@ public final class RWIProcess extends Thread { this.localSearchInclusion = search.inclusion(); final ReferenceContainer index = search.joined(); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.JOIN, this.query.queryString, index.size(), System.currentTimeMillis() - timer), false); - if (index.isEmpty()) { - return; + if (!index.isEmpty()) { + add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true); } - - add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true); } catch (final Exception e) { Log.logException(e); } finally { @@ -299,7 +297,7 @@ public final class RWIProcess extends Thread { } public boolean feedingIsFinished() { - return this.feeders.get() == 0; + return this.feeders.get() <= 0; } private boolean testFlags(final WordReference ientry) { diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 7f224738d..c087f915d 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -72,8 +72,8 @@ public final class SearchEvent { private QueryParams query; private final SeedDB peers; private final WorkTables workTables; - private RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container - private SnippetProcess resultFetcher; + private final RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container + private final SnippetProcess resultFetcher; private final SecondarySearchSuperviser secondarySearchSuperviser; @@ -117,14 +117,15 @@ public final class SearchEvent { this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang)); final boolean remote = peers.sizeConnected() > 0 && (this.query.domType == QueryParams.Searchdom.CLUSTER || (this.query.domType == QueryParams.Searchdom.GLOBAL && peers.mySeed().getFlagAcceptRemoteIndex())); final long start = System.currentTimeMillis(); - if (remote) { - // initialize a ranking process that is the target for data - // that is generated concurrently from local and global search threads - this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation); - // start a local search concurrently - this.rankingProcess.start(); + // initialize a ranking process that is the target for data + // that is generated concurrently from local and global search threads + this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation); + + // start a local search concurrently + this.rankingProcess.start(); + if (remote) { // start global searches final long timer = System.currentTimeMillis(); this.primarySearchThreads = (this.query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches( @@ -159,15 +160,13 @@ public final class SearchEvent { // no search since query is empty, user might have entered no data or filters have removed all search words Log.logFine("SEARCH_EVENT", "NO SEARCH STARTED DUE TO EMPTY SEARCH REQUEST."); } - - // start worker threads to fetch urls and snippets - this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 3000, deleteIfSnippetFail); } else { - // do a local search - this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation); - if (generateAbstracts) { - this.rankingProcess.run(); // this is not started concurrently here on purpose! + // we need the results now + try { + this.rankingProcess.join(); + } catch (final Throwable e) { + } // compute index abstracts final long timer = System.currentTimeMillis(); int maxcount = -1; @@ -193,21 +192,20 @@ public final class SearchEvent { } EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false); } else { - this.rankingProcess.start(); // start concurrently - // but give process time to accumulate a certain amount of data + // give process time to accumulate a certain amount of data // before a reading process wants to get results from it - for (int i = 0; i < 10; i++) { - if (!this.rankingProcess.isAlive()) break; - try {Thread.sleep(10);} catch (final InterruptedException e) {} + try { + this.rankingProcess.join(100); + } catch (final Throwable e) { } // this will reduce the maximum waiting time until results are available to 100 milliseconds // while we always get a good set of ranked data } - - // start worker threads to fetch urls and snippets - this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 500, deleteIfSnippetFail); } + // start worker threads to fetch urls and snippets + this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 5000, deleteIfSnippetFail); + // clean up events SearchEventCache.cleanupEvents(false); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.CLEANUP, "", 0, 0), false); diff --git a/source/net/yacy/search/query/SnippetProcess.java b/source/net/yacy/search/query/SnippetProcess.java index d200a8620..f130dfc26 100644 --- a/source/net/yacy/search/query/SnippetProcess.java +++ b/source/net/yacy/search/query/SnippetProcess.java @@ -165,7 +165,11 @@ public class SnippetProcess { // finally wait until enough results are there produced from the snippet fetch process WeakPriorityBlockingQueue.Element entry = null; while (System.currentTimeMillis() < finishTime) { - if (this.result.sizeAvailable() + this.rankingProcess.sizeQueue() <= item && this.rankingProcess.feedingIsFinished()) break; // the fail case + + if (!anyWorkerAlive() && !this.rankingProcess.isAlive() && this.result.sizeAvailable() + this.rankingProcess.sizeQueue() <= item && this.rankingProcess.feedingIsFinished()) { + //Log.logInfo("SnippetProcess", "interrupted result fetching; item = " + item + "; this.result.sizeAvailable() = " + this.result.sizeAvailable() + ", this.rankingProcess.sizeQueue() = " + this.rankingProcess.sizeQueue()); + break; // the fail case + } // deploy worker to get more results if (!anyWorkerAlive()) { @@ -189,7 +193,7 @@ public class SnippetProcess { private int resultCounter = 0; public ResultEntry nextResult() { - final ResultEntry re = oneResult(this.resultCounter, 1000); + final ResultEntry re = oneResult(this.resultCounter, 3000); this.resultCounter++; return re; } @@ -331,7 +335,7 @@ public class SnippetProcess { for (final Worker workerThread : this.workerThreads) { if ((workerThread != null) && (workerThread.isAlive()) && - (workerThread.busytime() < 1000)) return true; + (workerThread.busytime() < 10000)) return true; } } return false;