boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

bug fixes and speed enhancements for search

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8085 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 13 years ago
parent 507c9d478d
commit 1019c36dad
4 changed files with 33 additions and 33 deletions
Show Stats Download Patch File Download Diff File

4
htroot/yacysearch.java
Unescape View File

@ -629,8 +629,8 @@ public class yacysearch {
// log
Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
"local-unfiltered(" + theSearch.getRankingResult().getLocalIndexCount() + "), " +
"-local_miss(" + theSearch.getRankingResult().getMissCount() + "), " +
"-local_sortout(" + theSearch.getRankingResult().getSortOutCount() + "), " +
"local_miss(" + theSearch.getRankingResult().getMissCount() + "), " +
"local_sortout(" + theSearch.getRankingResult().getSortOutCount() + "), " +
"remote(" + theSearch.getRankingResult().getRemoteResourceSize() + ") links found, " +
(System.currentTimeMillis() - timestamp) + " ms");

8
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -153,11 +153,9 @@ public final class RWIProcess extends Thread {
this.localSearchInclusion = search.inclusion();
final ReferenceContainer<WordReference> index = search.joined();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.JOIN, this.query.queryString, index.size(), System.currentTimeMillis() - timer), false);
if (index.isEmpty()) {
return;
if (!index.isEmpty()) {
add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true);
}
add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true);
} catch (final Exception e) {
Log.logException(e);
} finally {
@ -299,7 +297,7 @@ public final class RWIProcess extends Thread {
}
public boolean feedingIsFinished() {
return this.feeders.get() == 0;
return this.feeders.get() <= 0;
}
private boolean testFlags(final WordReference ientry) {

44
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -72,8 +72,8 @@ public final class SearchEvent {
private QueryParams query;
private final SeedDB peers;
private final WorkTables workTables;
private RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
private SnippetProcess resultFetcher;
private final RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
private final SnippetProcess resultFetcher;
private final SecondarySearchSuperviser secondarySearchSuperviser;
@ -117,14 +117,15 @@ public final class SearchEvent {
this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang));
final boolean remote = peers.sizeConnected() > 0 && (this.query.domType == QueryParams.Searchdom.CLUSTER || (this.query.domType == QueryParams.Searchdom.GLOBAL && peers.mySeed().getFlagAcceptRemoteIndex()));
final long start = System.currentTimeMillis();
if (remote) {
// initialize a ranking process that is the target for data
// that is generated concurrently from local and global search threads
this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation);
// start a local search concurrently
this.rankingProcess.start();
// initialize a ranking process that is the target for data
// that is generated concurrently from local and global search threads
this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation);
// start a local search concurrently
this.rankingProcess.start();
if (remote) {
// start global searches
final long timer = System.currentTimeMillis();
this.primarySearchThreads = (this.query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches(
@ -159,15 +160,13 @@ public final class SearchEvent {
// no search since query is empty, user might have entered no data or filters have removed all search words
Log.logFine("SEARCH_EVENT", "NO SEARCH STARTED DUE TO EMPTY SEARCH REQUEST.");
}
// start worker threads to fetch urls and snippets
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 3000, deleteIfSnippetFail);
} else {
// do a local search
this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation);
if (generateAbstracts) {
this.rankingProcess.run(); // this is not started concurrently here on purpose!
// we need the results now
try {
this.rankingProcess.join();
} catch (final Throwable e) {
}
// compute index abstracts
final long timer = System.currentTimeMillis();
int maxcount = -1;
@ -193,21 +192,20 @@ public final class SearchEvent {
}
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false);
} else {
this.rankingProcess.start(); // start concurrently
// but give process time to accumulate a certain amount of data
// give process time to accumulate a certain amount of data
// before a reading process wants to get results from it
for (int i = 0; i < 10; i++) {
if (!this.rankingProcess.isAlive()) break;
try {Thread.sleep(10);} catch (final InterruptedException e) {}
try {
this.rankingProcess.join(100);
} catch (final Throwable e) {
}
// this will reduce the maximum waiting time until results are available to 100 milliseconds
// while we always get a good set of ranked data
}
// start worker threads to fetch urls and snippets
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 500, deleteIfSnippetFail);
}
// start worker threads to fetch urls and snippets
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 5000, deleteIfSnippetFail);
// clean up events
SearchEventCache.cleanupEvents(false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.CLEANUP, "", 0, 0), false);

10
source/net/yacy/search/query/SnippetProcess.java
Unescape View File

@ -165,7 +165,11 @@ public class SnippetProcess {
// finally wait until enough results are there produced from the snippet fetch process
WeakPriorityBlockingQueue.Element<ResultEntry> entry = null;
while (System.currentTimeMillis() < finishTime) {
if (this.result.sizeAvailable() + this.rankingProcess.sizeQueue() <= item && this.rankingProcess.feedingIsFinished()) break; // the fail case
if (!anyWorkerAlive() && !this.rankingProcess.isAlive() && this.result.sizeAvailable() + this.rankingProcess.sizeQueue() <= item && this.rankingProcess.feedingIsFinished()) {
//Log.logInfo("SnippetProcess", "interrupted result fetching; item = " + item + "; this.result.sizeAvailable() = " + this.result.sizeAvailable() + ", this.rankingProcess.sizeQueue() = " + this.rankingProcess.sizeQueue());
break; // the fail case
}
// deploy worker to get more results
if (!anyWorkerAlive()) {
@ -189,7 +193,7 @@ public class SnippetProcess {
private int resultCounter = 0;
public ResultEntry nextResult() {
final ResultEntry re = oneResult(this.resultCounter, 1000);
final ResultEntry re = oneResult(this.resultCounter, 3000);
this.resultCounter++;
return re;
}
@ -331,7 +335,7 @@ public class SnippetProcess {
for (final Worker workerThread : this.workerThreads) {
if ((workerThread != null) &&
(workerThread.isAlive()) &&
(workerThread.busytime() < 1000)) return true;
(workerThread.busytime() < 10000)) return true;
}
}
return false;

Write Preview
Loading…
Cancel
Save