boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

latest bugfixes to search and indexing function after test of demo presentation

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7223 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent c3bf17a3a1
commit 6e6994e328
3 changed files with 50 additions and 48 deletions
Show Stats Download Patch File Download Diff File

1
htroot/Load_RSS_p.html
Unescape View File

@ -35,7 +35,6 @@
<dd><input type="submit" name="showrss" value="Show RSS Items" /></dd>
<dt>Indexing</dt>
<dd>#(showload)#Available after successful loading of rss feed in preview::
<input type="hidden" name="url" value="#[rss]#" />
<input type="submit" name="indexAllItemContent" value="Add All Items to Index (full content of url)" />
<dl>
<dt>once<input type="radio" name="repeat" value="off" checked="checked"/></dt>

91
source/de/anomic/search/RankingProcess.java
Unescape View File

@ -286,17 +286,39 @@ public final class RankingProcess extends Thread {
// returns from the current RWI list the best entry and removes this entry from the list
WeakPriorityBlockingQueue<WordReferenceVars> m;
WeakPriorityBlockingQueue.Element<WordReferenceVars> rwi = null;
try {
// check if the doubleDomCache is filled
/*
boolean doubleDomCacheFilled = false;
synchronized (this.doubleDomCache) {
final Iterator<WeakPriorityBlockingQueue<WordReferenceVars>> i = this.doubleDomCache.values().iterator();
while (i.hasNext()) {
try {
m = i.next();
} catch (ConcurrentModificationException e) {
Log.logException(e);
break; // not the best solution...
}
if (m == null) continue;
if (m.isEmpty()) continue;
doubleDomCacheFilled = true;
break;
}
}
*/
// take one entry from the stack if there are entries on that stack or the feeding is not yet finished
if (!feedingIsFinished() || stack.sizeQueue() > 0) try {
//System.out.println("stack.poll: feeders = " + this.feeders + ", stack.sizeQueue = " + stack.sizeQueue());
int loops = 0; // a loop counter to terminate the reading if all the results are from the same domain
long timeout = System.currentTimeMillis() + waitingtime;
while (this.query.itemsPerPage < 1 || loops++ < this.query.itemsPerPage) {
if (waitingtime <= 0) {
rwi = stack.poll();
} else while (System.currentTimeMillis() < timeout) {
} else timeoutloop:while (System.currentTimeMillis() < timeout) {
if (feedingIsFinished() && stack.sizeQueue() == 0) break timeoutloop;
rwi = stack.poll(50);
if (rwi != null) break;
if (feedingIsFinished() && stack.sizeQueue() == 0) break;
if (rwi != null) break timeoutloop;
}
if (rwi == null) break;
if (!skipDoubleDom) {
@ -306,24 +328,20 @@ public final class RankingProcess extends Thread {
// check doubledom
final String domhash = new String(rwi.getElement().metadataHash(), 6, 6);
m = this.doubleDomCache.get(domhash);
if (m == null) {
// first appearance of dom
m = new WeakPriorityBlockingQueue<WordReferenceVars>((query.specialRights) ? maxDoubleDomSpecial : maxDoubleDomAll);
this.doubleDomCache.put(domhash, m);
//System.out.println("m == null");
return rwi;
synchronized (this.doubleDomCache) {
m = this.doubleDomCache.get(domhash);
if (m == null) {
// first appearance of dom. we create an entry to signal that one of that domain was already returned
m = new WeakPriorityBlockingQueue<WordReferenceVars>((query.specialRights) ? maxDoubleDomSpecial : maxDoubleDomAll);
this.doubleDomCache.put(domhash, m);
return rwi;
}
// second appearances of dom
m.put(rwi);
}
// second appearances of dom
m.put(rwi);
}
} catch (InterruptedException e1) {
}
if (this.doubleDomCache.size() == 0) {
//System.out.println("this.doubleDomCache.size() == 0");
return null;
}
} catch (InterruptedException e1) {}
if (this.doubleDomCache.size() == 0) return null;
// no more entries in sorted RWI entries. Now take Elements from the doubleDomCache
// find best entry from all caches
@ -336,41 +354,26 @@ public final class RankingProcess extends Thread {
m = i.next();
} catch (ConcurrentModificationException e) {
Log.logException(e);
break; // not the best solution...
}
if (m == null) {
//System.out.println("m == null");
continue;
}
if (m.isEmpty()) {
//System.out.println("m.isEmpty()");
continue;
continue; // not the best solution...
}
if (m == null) continue;
if (m.isEmpty()) continue;
if (bestEntry == null) {
bestEntry = m.peek();
//System.out.println("bestEntry = m.peek() = " + bestEntry);
continue;
}
o = m.peek();
if (o == null) {
//System.out.println("o == null");
continue;
}
if (o == null) continue;
if (o.getWeight() < bestEntry.getWeight()) {
bestEntry = o;
}
}
if (bestEntry == null) return null;
// finally remove the best entry from the doubledom cache
m = this.doubleDomCache.get(new String(bestEntry.getElement().metadataHash()).substring(6));
bestEntry = m.poll();
}
if (bestEntry == null) {
//System.out.println("bestEntry == null");
return null;
}
// finally remove the best entry from the doubledom cache
m = this.doubleDomCache.get(new String(bestEntry.getElement().metadataHash()).substring(6));
o = m.poll();
//assert o == null || o.element.metadataHash().equals(bestEntry.element.metadataHash()) : "bestEntry.element.metadataHash() = " + bestEntry.element.metadataHash() + ", o.element.metadataHash() = " + o.element.metadataHash();
//System.out.println("return bestEntry");
return bestEntry;
}

6
source/de/anomic/search/ResultFetcher.java
Unescape View File

@ -72,7 +72,7 @@ public class ResultFetcher {
final QueryParams query,
final yacySeedDB peers,
final int taketimeout) {
assert query != null;
this.loader = loader;
this.rankingProcess = rankedCache;
this.query = query;
@ -102,7 +102,7 @@ public class ResultFetcher {
// start worker threads to fetch urls and snippets
this.workerThreads = null;
deployWorker(Math.min(10, query.itemsPerPage), query.neededResults());
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), SearchEvent.Type.SNIPPETFETCH_START, this.workerThreads.length + " online snippet fetch threads started", 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), SearchEvent.Type.SNIPPETFETCH_START, ((this.workerThreads == null) ? "no" : this.workerThreads.length) + " online snippet fetch threads started", 0, 0), false);
}
public void deployWorker(int deployCount, int neededResults) {
@ -111,7 +111,7 @@ public class ResultFetcher {
this.workerThreads = new Worker[/*(query.snippetCacheStrategy.mustBeOffline()) ? 1 : */deployCount];
synchronized(this.workerThreads) {
for (int i = 0; i < workerThreads.length; i++) {
Worker worker = new Worker(i, 1000, query.snippetCacheStrategy, neededResults);
Worker worker = new Worker(i, 10000, query.snippetCacheStrategy, neededResults);
worker.start();
this.workerThreads[i] = worker;
}

Write Preview
Loading…
Cancel
Save