boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- no more search time-out!

Browse Source 
- fixed a bug with last commit

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4430 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent cd3e0d6f03
commit 7404256997
15 changed files with 35 additions and 64 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5
# Release Configuration
releaseVersion=0.565
releaseVersion=0.566
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

15
htroot/index.html
Unescape View File

@ -35,7 +35,6 @@
<input type="hidden" name="count" value="10" />
<input type="hidden" name="offset" value="0" />
<input type="hidden" name="resource" value="global" />
<input type="hidden" name="time" value="#[searchtime]#" />
<input type="hidden" name="urlmaskfilter" value=".*" />
<input type="hidden" name="prefermaskfilter" value="" />
<input type="hidden" name="indexof" value="off" />
@ -72,20 +71,6 @@
</td>
</tr>
<tr>
<td><label for="time">Max. search time (seconds)</label>:</td>
<td>
<select id="time" name="time">
<option #(time-1)#::selected="selected"#(/time-1)#>1</option>
<option #(time-2)#::selected="selected"#(/time-2)#>2</option>
<option #(time-3)#::selected="selected"#(/time-3)#>3</option>
<option #(time-4)#::selected="selected"#(/time-4)#>4</option>
<option #(time-6)#::selected="selected"#(/time-6)#>6</option>
<option #(time-8)#::selected="selected"#(/time-8)#>8</option>
<option #(time-10)#::selected="selected"#(/time-10)#>10</option>
</select>
</td>
</tr>
<tr>
<td><label for="urlmaskfilter">URL mask</label>:</td>
<td>#(urlmaskoptions)#
<input id="urlmaskfilter" name="urlmaskfilter" type="text" size="12" maxlength="80" value="#[urlmaskfilter]#" />

9
htroot/index.java
Unescape View File

@ -60,7 +60,6 @@ public class index {
final int searchoptions = (post == null) ? 0 : post.getInt("searchoptions", 0);
final String former = (post == null) ? "" : post.get("former", "");
final int count = Math.min(100, (post == null) ? 10 : post.getInt("count", 10));
final int time = Math.min(60, (post == null) ? (int) sb.getConfigLong("network.unit.search.time", 3) : post.getInt("time", (int) sb.getConfigLong("network.unit.search.time", 3)));
final String urlmaskfilter = (post == null) ? ".*" : post.get("urlmaskfilter", ".*");
final String prefermaskfilter = (post == null) ? "" : post.get("prefermaskfilter", "");
final String constraint = (post == null) ? "" : post.get("constraint", "");
@ -119,14 +118,6 @@ public class index {
prop.put("searchoptions_resource-global-disabled", (indexReceiveGranted && indexDistributeGranted) ? "0" : "1");
prop.put("searchoptions_resource-global-disabled_reason", (indexReceiveGranted) ? "0" : (indexDistributeGranted ? "1" : "2"));
prop.put("searchoptions_resource-local", global ? "0" : "1");
prop.put("searchoptions_searchtime", time);
prop.put("searchoptions_time-1", (time == 1) ? "1" : "0");
prop.put("searchoptions_time-2", (time == 2) ? "1" : "0");
prop.put("searchoptions_time-3", (time == 3) ? "1" : "0");
prop.put("searchoptions_time-4", (time == 4) ? "1" : "0");
prop.put("searchoptions_time-6", (time == 6) ? "1" : "0");
prop.put("searchoptions_time-8", (time == 8) ? "1" : "0");
prop.put("searchoptions_time-10", (time == 10) ? "1" : "0");
prop.put("searchoptions_urlmaskoptions", "0");
prop.putHTML("searchoptions_urlmaskoptions_urlmaskfilter", urlmaskfilter);
prop.put("searchoptions_prefermaskoptions", "0");

9
htroot/yacy/search.java
Unescape View File

@ -77,7 +77,6 @@ public final class search {
String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
// final String fwdep = post.get("fwdep", ""); // forward depth. if "0" then peer may NOT ask another peer for more results
// final String fwden = post.get("fwden", ""); // forward deny, a list of seed hashes. They may NOT be target of forward hopping
final long duetime= Math.min(60000, post.getLong("duetime", 3000));
final int count = Math.min(100, post.getInt("count", 10)); // maximum number of wanted results
final int maxdist= post.getInt("maxdist", Integer.MAX_VALUE);
final String prefer = post.get("prefer", "");
@ -119,7 +118,7 @@ public final class search {
}
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(2 * duetime);
sb.intermissionAllThreads(3000);
TreeSet<String> abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts);
@ -147,7 +146,7 @@ public final class search {
long urlRetrievalAllTime = 0, snippetComputationAllTime = 0;
if ((query.length() == 0) && (abstractSet != null)) {
// this is _not_ a normal search, only a request for index abstracts
theQuery = new plasmaSearchQuery(null, abstractSet, new TreeSet<String>(kelondroBase64Order.enhancedComparator), rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, duetime, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, null, false);
theQuery = new plasmaSearchQuery(null, abstractSet, new TreeSet<String>(kelondroBase64Order.enhancedComparator), rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, null, false);
theQuery.domType = plasmaSearchQuery.SEARCHDOM_LOCAL;
yacyCore.log.logInfo("INIT HASH SEARCH (abstracts only): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
@ -173,7 +172,7 @@ public final class search {
} else {
// retrieve index containers from search request
theQuery = new plasmaSearchQuery(null, queryhashes, excludehashes, rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, duetime, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, constraint, false);
theQuery = new plasmaSearchQuery(null, queryhashes, excludehashes, rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, constraint, false);
theQuery.domType = plasmaSearchQuery.SEARCHDOM_LOCAL;
yacyCore.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
@ -213,7 +212,7 @@ public final class search {
} else {
joincount = theSearch.getRankingResult().getLocalResourceSize();
prop.put("joincount", Integer.toString(joincount));
accu = theSearch.completeResults(duetime);
accu = theSearch.completeResults(3000);
}
// generate compressed index for maxcounthash

11
htroot/yacysearch.java
Unescape View File

@ -120,7 +120,6 @@ public class yacysearch {
prop.put("input_count", "10");
prop.put("input_offset", "0");
prop.put("input_resource", "global");
prop.put("input_time", sb.getConfigLong("network.unit.search.time", 3));
prop.put("input_urlmaskfilter", ".*");
prop.put("input_prefermaskfilter", "");
prop.put("input_indexof", "off");
@ -154,7 +153,6 @@ public class yacysearch {
int offset = post.getInt("offset", 0);
boolean global = (post == null) ? true : post.get("resource", "global").equals("global");
final boolean indexof = post.get("indexof","").equals("on");
final long searchtime = 1000 * post.getLong("time", (int) sb.getConfigLong("network.unit.search.time", 3));
String urlmask = "";
if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) {
urlmask = ".*";
@ -267,7 +265,6 @@ public class yacysearch {
true,
itemsPerPage,
offset,
searchtime,
urlmask,
(clustersearch && globalsearch) ? plasmaSearchQuery.SEARCHDOM_CLUSTERALL :
((globalsearch) ? plasmaSearchQuery.SEARCHDOM_GLOBALDHT : plasmaSearchQuery.SEARCHDOM_LOCAL),
@ -279,13 +276,13 @@ public class yacysearch {
String client = (String) header.get("CLIENTIP"); // the search client who initiated the search
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(2 * theQuery.maximumTime);
sb.intermissionAllThreads(10000);
// filter out words that appear in bluelist
theQuery.filterOut(plasmaSwitchboard.blueList);
// log
serverLog.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + theQuery.queryHashes + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed, " + (theQuery.maximumTime / 1000) + " seconds");
serverLog.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + theQuery.queryHashes + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed");
long timestamp = System.currentTimeMillis();
// create a new search event
@ -300,8 +297,6 @@ public class yacysearch {
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER RESULT PREPARATION: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
// calc some more cross-reference
long remainingTime = theQuery.maximumTime - (System.currentTimeMillis() - timestamp);
if (remainingTime < 0) remainingTime = 1000;
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER XREF PREPARATION: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
// log
@ -406,7 +401,6 @@ public class yacysearch {
prop.put("input_count", itemsPerPage);
prop.put("input_offset", offset);
prop.put("input_resource", global ? "global" : "local");
prop.put("input_time", searchtime / 1000);
prop.putHTML("input_urlmaskfilter", urlmask);
prop.putHTML("input_prefermaskfilter", prefermask);
prop.put("input_indexof", (indexof) ? "on" : "off");
@ -435,7 +429,6 @@ public class yacysearch {
"&amp;count="+ theQuery.displayResults() +
"&amp;offset=" + (page * theQuery.displayResults()) +
"&amp;resource=" + theQuery.searchdom() +
"&amp;time=" + (theQuery.maximumTime / 1000) +
"&amp;urlmaskfilter=" + theQuery.urlMask +
"&amp;prefermaskfilter=" + theQuery.prefer +
"&amp;cat=href&amp;constraint=" + ((theQuery.constraint == null) ? "" : theQuery.constraint.exportB64()) +

1
htroot/yacysearchitem.java
Unescape View File

@ -151,7 +151,6 @@ public class yacysearchitem {
prop.put("references_words_" + hintcount + "_offset", "0");
prop.put("references_words_" + hintcount + "_contentdom", theQuery.contentdom());
prop.put("references_words_" + hintcount + "_resource", theQuery.searchdom());
prop.put("references_words_" + hintcount + "_time", (theQuery.maximumTime / 1000));
}
prop.put("references_words", hintcount);
if (hintcount++ > MAX_TOPWORDS) {

2
source/de/anomic/index/indexCollectionRI.java
Unescape View File

@ -169,7 +169,7 @@ public class indexCollectionRI implements indexRI {
}
}
public void addEntries(indexContainer newEntries, long creationTime, boolean dhtCase) {
public void addEntries(indexContainer newEntries) {
try {
collectionIndex.merge(newEntries);
} catch (kelondroOutOfLimitsException e) {

4
source/de/anomic/index/indexRAMRI.java
Unescape View File

@ -481,7 +481,7 @@ public final class indexRAMRI implements indexRI {
return delCount;
}
public synchronized void addEntries(indexContainer container, long updateTime, boolean dhtCase) {
public synchronized void addEntries(indexContainer container) {
// this puts the entries into the cache, not into the assortment directly
int added = 0;
if ((container == null) || (container.size() == 0)) return;
@ -498,7 +498,7 @@ public final class indexRAMRI implements indexRI {
if (added > 0) {
cache.put(wordHash, entries);
hashScore.addScore(wordHash, added);
hashDate.setScore(wordHash, intTime(updateTime));
hashDate.setScore(wordHash, intTime(System.currentTimeMillis()));
}
entries = null;
}

2
source/de/anomic/index/indexRI.java
Unescape View File

@ -44,7 +44,7 @@ public interface indexRI {
public indexContainer deleteContainer(String wordHash);
public boolean removeEntry(String wordHash, String urlHash);
public int removeEntries(String wordHash, Set<String> urlHashes);
public void addEntries(indexContainer newEntries, long creationTime, boolean dhtCase);
public void addEntries(indexContainer newEntries);
public void close();
}

8
source/de/anomic/kelondro/kelondroEcoFS.java
Unescape View File

@ -225,12 +225,12 @@ public class kelondroEcoFS {
assert b.length - start >= this.recordsize;
if (index > size()) throw new IndexOutOfBoundsException("kelondroEcoFS.put(" + index + ") outside bounds (" + this.size() + ")");
// check if this is an empty entry
/*
if (isClean(b , start, this.recordsize)) {
clean(index);
return;
}
*/
// check if index is inside of cache
int p = inCache(index);
int q = (p >= 0) ? -1 : inBuffer(index);
@ -306,7 +306,7 @@ public class kelondroEcoFS {
assert false;
return false;
}
/*
public synchronized void clean(long index, byte[] b, int start) throws IOException {
// removes an entry by cleaning (writing zero bytes to the file)
// the entry that had been at the specific place before is copied to the given array b
@ -378,7 +378,7 @@ public class kelondroEcoFS {
raf.seek((long) index * (long) this.recordsize);
raf.write(zero, 0, this.recordsize);
}
*/
public synchronized void cleanLast(byte[] b, int start) throws IOException {
cleanLast0(b, start);
long i;

2
source/de/anomic/plasma/dbImport/plasmaDbImporter.java
Unescape View File

@ -219,7 +219,7 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter {
if (isAborted()) break;
// importing entity container to home db
if (newContainer.size() > 0) { homeWordIndex.addEntries(newContainer, System.currentTimeMillis(), false); }
if (newContainer.size() > 0) { homeWordIndex.addEntries(newContainer); }
// delete complete index entity file
this.importWordIndex.deleteContainer(this.wordHash);

10
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -122,9 +122,7 @@ public final class plasmaSearchEvent {
// do a global search
this.rankedCache = new plasmaSearchRankingProcess(wordIndex, query, 2, max_results_preparation);
int fetchpeers = (int) (query.maximumTime / 500L); // number of target peers; means 10 peers in 10 seconds
if (fetchpeers > 50) fetchpeers = 50;
if (fetchpeers < 30) fetchpeers = 30;
int fetchpeers = 30;
// the result of the fetch is then in the rcGlobal
long timer = System.currentTimeMillis();
@ -194,7 +192,7 @@ public final class plasmaSearchEvent {
// start worker threads to fetch urls and snippets
this.workerThreads = new resultWorker[workerThreadCount];
for (int i = 0; i < workerThreadCount; i++) {
this.workerThreads[i] = new resultWorker(i, query.maximumTime * 3);
this.workerThreads[i] = new resultWorker(i, 10000);
this.workerThreads[i].start();
}
} else {
@ -475,7 +473,7 @@ public final class plasmaSearchEvent {
// start worker threads to fetch urls and snippets
event.workerThreads = new resultWorker[workerThreadCount];
for (int i = 0; i < workerThreadCount; i++) {
event.workerThreads[i] = event.deployWorker(i, 3 * query.maximumTime);
event.workerThreads[i] = event.deployWorker(i, 10000);
}
}
@ -619,7 +617,7 @@ public final class plasmaSearchEvent {
public ArrayList<ResultEntry> completeResults(long waitingtime) {
long timeout = System.currentTimeMillis() + waitingtime;
while ((this.resultList.size() < query.neededResults()) && (anyWorkerAlive()) && (System.currentTimeMillis() < timeout)) {
try {Thread.sleep(200);} catch (InterruptedException e) {}
try {Thread.sleep(100);} catch (InterruptedException e) {}
//System.out.println("+++DEBUG-completeResults+++ sleeping " + 200);
}
return this.resultList;

8
source/de/anomic/plasma/plasmaSearchQuery.java
Unescape View File

@ -78,7 +78,6 @@ public final class plasmaSearchQuery {
private int linesPerPage, offset;
public String prefer;
public int contentdom;
public long maximumTime;
public String urlMask;
public int domType;
public String domGroupName;
@ -110,7 +109,6 @@ public final class plasmaSearchQuery {
this.contentdom = CONTENTDOM_ALL;
this.linesPerPage = lines;
this.offset = 0;
this.maximumTime = 3000;
this.urlMask = ".*";
this.domType = SEARCHDOM_LOCAL;
this.domGroupName = "";
@ -125,7 +123,7 @@ public plasmaSearchQuery(
plasmaSearchRankingProfile ranking,
int maxDistance, String prefer, int contentdom,
boolean onlineSnippetFetch,
int lines, int offset, long maximumTime, String urlMask,
int lines, int offset, String urlMask,
int domType, String domGroupName, int domMaxTargets,
kelondroBitfield constraint, boolean allofconstraint) {
this.queryString = queryString;
@ -137,7 +135,7 @@ public plasmaSearchQuery(
this.contentdom = contentdom;
this.linesPerPage = lines;
this.offset = offset;
this.maximumTime = Math.min(6000, maximumTime);
//this.maximumTime = Math.min(6000, maximumTime);
this.urlMask = urlMask;
this.domType = domType;
this.domGroupName = domGroupName;
@ -288,7 +286,7 @@ public plasmaSearchQuery(
r.put("queryhashes", queryHashes);
r.put("querystring", queryString);
r.put("querycount", new Integer(linesPerPage));
r.put("querytime", new Long(maximumTime));
//r.put("querytime", new Long(maximumTime));
r.put("resultcount", new Integer(searchcount));
r.put("resulttime", new Long(searchtime));
r.put("resulturltime", new Long(urlretrieval));

13
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -187,7 +187,11 @@ public final class plasmaWordIndex implements indexRI {
}
}
public void addEntries(indexContainer entries, long updateTime, boolean dhtInCase) {
public void addEntries(indexContainer entries) {
addEntries(entries, false);
}
public void addEntries(indexContainer entries, boolean dhtInCase) {
assert (entries.row().objectsize == indexRWIRowEntry.urlEntryRow.objectsize);
// set dhtInCase depending on wordHash
@ -195,10 +199,10 @@ public final class plasmaWordIndex implements indexRI {
// add the entry
if (dhtInCase) {
dhtInCache.addEntries(entries, updateTime, true);
dhtInCache.addEntries(entries);
dhtFlushControl(this.dhtInCache);
} else {
dhtOutCache.addEntries(entries, updateTime, false);
dhtOutCache.addEntries(entries);
dhtFlushControl(this.dhtOutCache);
}
}
@ -244,6 +248,9 @@ public final class plasmaWordIndex implements indexRI {
if (c != null) containerList.add(c);
}
// flush the containers
for (int i = 0; i < containerList.size(); i++) {
collections.addEntries((indexContainer) containerList.get(i));
}
collections.addMultipleEntries(containerList);
//System.out.println("DEBUG-Finished flush of " + count + " entries from RAM to DB in " + (System.currentTimeMillis() - start) + " milliseconds");
busyCacheFlush = false;

3
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -417,6 +417,7 @@ public final class yacyClient {
post.put("partitions", partitions);
post.put("query", wordhashes);
post.put("exclude", excludehashes);
post.put("duetime", 1000);
post.put("urls", urlhashes);
post.put("prefer", prefer);
post.put("filter", filter);
@ -573,7 +574,7 @@ public final class yacyClient {
// insert the containers to the index
for (int m = 0; m < words; m++) {
wordIndex.addEntries(container[m], System.currentTimeMillis(), true);
wordIndex.addEntries(container[m], true);
}
// read index abstract

Write Preview
Loading…
Cancel
Save