boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

smaller bug fixes for search behavior; should produce less unnecessary removals and an exact number of results as shown in counter

Browse Source 
should also be a little bit faster

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8057 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 13 years ago
parent a3eebfdcba
commit 804e48888b
9 changed files with 33 additions and 41 deletions
Show Stats Download Patch File Download Diff File

24
htroot/ConfigPortal.java
Unescape View File

@ -1,4 +1,4 @@
// ConfigPortal.java
// ConfigPortal.java
// -----------------------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
@ -10,7 +10,7 @@
//$LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -38,14 +38,14 @@ public class ConfigPortal {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
if (post != null) {
// AUTHENTICATE
if (!header.containsKey(RequestHeader.AUTHORIZATION)) {
prop.putHTML("AUTHENTICATE","log-in");
return prop;
}
if (post.containsKey("popup")) {
final String popup = post.get("popup", "status");
if ("front".equals(popup)) {
@ -62,7 +62,7 @@ public class ConfigPortal {
final String newGreeting = post.get(SwitchboardConstants.GREETING, "");
// store this call as api call
sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));
sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, ""));
@ -80,7 +80,7 @@ public class ConfigPortal {
sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser", false));
sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures", false));
sb.setConfig("search.verify", post.get("search.verify", "ifexist"));
sb.setConfig("search.verify.delete", post.getBoolean("search.verify.delete", false));
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete", false));
// construct navigation String
String nav = "";
if (post.getBoolean("search.navigation.hosts", false)) nav += "hosts,";
@ -109,8 +109,8 @@ public class ConfigPortal {
sb.setConfig("search.result.show.parser", true);
sb.setConfig("search.result.show.pictures", true);
sb.setConfig("search.verify", "iffresh");
sb.setConfig("search.verify.delete", "true");
}
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "true");
}
}
prop.putHTML(SwitchboardConstants.GREETING, sb.getConfig(SwitchboardConstants.GREETING, ""));
@ -121,7 +121,7 @@ public class ConfigPortal {
prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0);
prop.put("publicSearchpage", sb.getConfigBool("publicSearchpage", false) ? 1 : 0);
prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);
prop.put("search.result.show.date", sb.getConfigBool("search.result.show.date", false) ? 1 : 0);
prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);
prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);
@ -138,7 +138,7 @@ public class ConfigPortal {
prop.put("search.verify.ifexist", sb.getConfig("search.verify", "").equals("ifexist") ? 1 : 0);
prop.put("search.verify.cacheonly", sb.getConfig("search.verify", "").equals("cacheonly") ? 1 : 0);
prop.put("search.verify.false", sb.getConfig("search.verify", "").equals("false") ? 1 : 0);
prop.put("search.verify.delete", sb.getConfigBool("search.verify.delete", true) ? 1 : 0);
prop.put("search.verify.delete", sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true) ? 1 : 0);
final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
prop.put("popupFront", 0);
@ -156,14 +156,14 @@ public class ConfigPortal {
}
prop.put("maximumRecords", sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10));
final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET, "_self");
prop.put("selected_blank", "_blank".equals(target) ? 1 : 0);
prop.put("selected_self", "_self".equals(target) ? 1 : 0);
prop.put("selected_parent", "_parent".equals(target) ? 1 : 0);
prop.put("selected_top", "_top".equals(target) ? 1 : 0);
prop.put("selected_searchresult", "searchresult".equals(target) ? 1 : 0);
String myaddress = (sb.peers == null) ? null : sb.peers.mySeed() == null ? null : sb.peers.mySeed().getPublicAddress();
if (myaddress == null) {
myaddress = "localhost:" + sb.getConfig("port", "8090");

2
htroot/yacysearch.java
Unescape View File

@ -597,7 +597,7 @@ public class yacysearch {
indexSegment,
ranking,
header.get(RequestHeader.USER_AGENT, ""),
sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false);

5
source/net/yacy/cora/date/GenericFormatter.java
Unescape View File

@ -133,10 +133,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
if (UTCOffset == null || UTCOffset.length() == 0) { return new Date(); }
try {
return new Date(this.dateFormat.parse(timeString).getTime() - UTCDiff() + UTCDiff(UTCOffset));
} catch (final java.text.ParseException e) {
//serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]");
return new Date();
} catch (final java.lang.NumberFormatException e) {
} catch (final Throwable e) {
//serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]");
return new Date();
}

11
source/net/yacy/document/parser/images/genericImageParser.java
Unescape View File

@ -28,7 +28,6 @@ package net.yacy.document.parser.images;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@ -227,14 +226,8 @@ public class genericImageParser extends AbstractParser implements Parser {
try {
ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM
image = ImageIO.read(sourceStream);
} catch (final EOFException e) {
Log.logException(e);
throw new Parser.Failure(e.getMessage(), location);
} catch (final IOException e) {
Log.logException(e);
throw new Parser.Failure(e.getMessage(), location);
} catch (final Throwable e) { // may appear in case of an OutOfMemoryError
Log.logException(e);
} catch (final Throwable e) {
//Log.logException(e);
throw new Parser.Failure(e.getMessage(), location);
}
if (image == null) throw new Parser.Failure("ImageIO returned NULL", location);

1
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -437,6 +437,7 @@ public final class SwitchboardConstants {
public static final String SEARCH_ITEMS = "search.items";
public static final String SEARCH_TARGET = "search.target";
public static final String SEARCH_VERIFY_DELETE = "search.verify.delete";
/**
* system tray

13
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -35,6 +35,7 @@ import java.util.SortedMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import net.yacy.cora.document.ASCII;
@ -79,7 +80,7 @@ public final class RWIProcess extends Thread {
private int remote_resourceSize, remote_indexCount, remote_peerCount;
private int local_indexCount;
private final WeakPriorityBlockingQueue<WordReferenceVars> stack;
private int feeders;
private final AtomicInteger feeders;
private final ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack
//private final HandleSet handover; // key = urlhash; used for double-check of urls that had been handed over to search process
@ -123,7 +124,7 @@ public final class RWIProcess extends Thread {
this.protocolNavigator = new ConcurrentScoreMap<String>();
this.filetypeNavigator = new ConcurrentScoreMap<String>();
this.ref = new ConcurrentScoreMap<String>();
this.feeders = 1;
this.feeders = new AtomicInteger(1);
this.startTime = System.currentTimeMillis();
}
@ -288,16 +289,16 @@ public final class RWIProcess extends Thread {
* method to signal the incoming stack that one feeder has terminated
*/
public void oneFeederTerminated() {
this.feeders--;
assert this.feeders >= 0 : "feeders = " + this.feeders;
final int c = this.feeders.decrementAndGet();
assert c >= 0 : "feeders = " + c;
}
public void moreFeeders(final int countMoreFeeders) {
this.feeders += countMoreFeeders;
this.feeders.addAndGet(countMoreFeeders);
}
public boolean feedingIsFinished() {
return System.currentTimeMillis() - this.startTime > 50 && this.feeders == 0;
return this.feeders.get() == 0;
}
private boolean testFlags(final WordReference ientry) {

5
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -371,14 +371,13 @@ public final class SearchEvent {
}
public ResultEntry oneResult(final int item, final long timeout) {
if ((this.query.domType == QueryParams.Searchdom.GLOBAL) ||
(this.query.domType == QueryParams.Searchdom.CLUSTER)) {
if ((this.query.domType == QueryParams.Searchdom.GLOBAL) || (this.query.domType == QueryParams.Searchdom.CLUSTER)) {
// this is a search using remote search threads. Also the local
// search thread is started as background process
if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) {
// in case that the local search takes longer than some other
// remote search requests, wait that the local process terminates first
try {this.localSearchThread.join();} catch (final InterruptedException e) {}
try {this.localSearchThread.join(300);} catch (final InterruptedException e) {}
}
}
return this.resultFetcher.oneResult(item, timeout);

3
source/net/yacy/search/query/SearchEventCache.java
Unescape View File

@ -37,6 +37,7 @@ import net.yacy.kelondro.util.MemoryControl;
import net.yacy.peers.SeedDB;
import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment;
import net.yacy.search.ranking.RankingProfile;
import net.yacy.search.snippet.ContentDomain;
@ -178,7 +179,7 @@ public class SearchEventCache {
Log.logInfo("SearchEventCache", "getEvent: " + SearchEventCache.lastEvents.size() + " in cache; " + countAliveThreads() + " alive");
// start a new event
final boolean delete = Switchboard.getSwitchboard() == null | Switchboard.getSwitchboard().getConfigBool("search.verify.delete", true);
final boolean delete = Switchboard.getSwitchboard() == null || Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true);
event = new SearchEvent(query, peers, workTables, preselectedPeerHashes, generateAbstracts, loader, remote_maxcount, remote_maxtime, burstRobinsonPercent, burstMultiwordPercent, delete);
MemoryControl.request(100 * 1024 * 1024, false); // this may trigger a short memory status which causes a reducing of cache space of other threads
}

10
source/net/yacy/search/query/SnippetProcess.java
Unescape View File

@ -147,7 +147,8 @@ public class SnippetProcess {
final long waittimeout = System.currentTimeMillis() + 300;
while (
item == 0 &&
this.result.sizeAvailable() < this.query.neededResults() + this.query.itemsPerPage &&
(!this.rankingProcess.feedingIsFinished() || this.rankingProcess.sizeQueue() > 0) &&
this.result.sizeAvailable() < this.query.neededResults() /* + this.query.itemsPerPage */ &&
System.currentTimeMillis() < waittimeout &&
anyWorkerAlive()
) {
@ -324,7 +325,7 @@ public class SnippetProcess {
}
private boolean anyWorkerAlive() {
if (this.workerThreads == null) return false;
if (this.workerThreads == null || this.workerThreads.length == 0) return false;
synchronized(this.workerThreads) {
for (final Worker workerThread : this.workerThreads) {
if ((workerThread != null) &&
@ -377,7 +378,7 @@ public class SnippetProcess {
// check if we have enough
if (SnippetProcess.this.result.sizeAvailable() >= this.neededResults) {
//Log.logWarning("ResultFetcher", ResultFetcher.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
//Log.logWarning("ResultFetcher", SnippetProcess.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
break;
}
@ -391,7 +392,7 @@ public class SnippetProcess {
page = SnippetProcess.this.rankingProcess.takeURL(true, Math.min(100, this.timeout - System.currentTimeMillis()));
//if (page == null) page = rankedCache.takeURL(false, this.timeout - System.currentTimeMillis());
if (page == null) {
//System.out.println("page == null");
//Log.logWarning("ResultFetcher", "page == null");
break; // no more available
}
if (SnippetProcess.this.query.filterfailurls && SnippetProcess.this.workTables.failURLsContains(page.hash())) continue;
@ -405,7 +406,6 @@ public class SnippetProcess {
if (sd != null) solrContent = this.solr.getScheme().solrGetText(sd);
}
loops++;
resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0
if (resultEntry == null) continue; // the entry had some problems, cannot be used

Write Preview
Loading…
Cancel
Save