diff --git a/htroot/yacysearch_location.html b/htroot/yacysearch_location.html index 4734db426..d579b6fba 100644 --- a/htroot/yacysearch_location.html +++ b/htroot/yacysearch_location.html @@ -5,6 +5,12 @@ #%env/templates/metas.template%# + @@ -92,6 +105,9 @@ To see a list of all APIs, please visit the
-
+
+ \ No newline at end of file diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index e92d37edb..592a733c3 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -29,6 +29,7 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.services.SearchSRURSS; import net.yacy.document.LibraryProvider; import net.yacy.document.geolocalization.Location; +import de.anomic.crawler.CrawlProfile; import de.anomic.search.Switchboard; import de.anomic.search.SwitchboardConstants; import de.anomic.server.serverCore; @@ -62,8 +63,8 @@ public class yacysearch_location { boolean search_publisher = alltext || post.get("dom", "").indexOf("publisher") >= 0; boolean search_creator = alltext || post.get("dom", "").indexOf("creator") >= 0; boolean search_subject = alltext || post.get("dom", "").indexOf("subject") >= 0; - long maximumTime = post.getLong("maximumTime", 3000); - int maximumRecords = post.getInt("maximumRecords", 200); + long maximumTime = post.getLong("maximumTime", 5000); + int maximumRecords = post.getInt("maximumRecords", 3000); //i.e. http://localhost:8090/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100 int placemarkCounter = 0; @@ -93,7 +94,7 @@ public class yacysearch_location { // get a queue of search results String rssSearchServiceURL = "http://127.0.0.1:" + sb.getConfig("port", "8090") + "/yacysearch.rss"; BlockingQueue results = new LinkedBlockingQueue(); - SearchSRURSS.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, false, false, null); + SearchSRURSS.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, CrawlProfile.CacheStrategy.NOCACHE, false, null); // take the results and compute some locations RSSMessage message; diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index 7fb8910fe..ea73a89b2 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -319,7 +319,7 @@ public class ResultFetcher { // check if we have enough if (result.sizeAvailable() >= this.neededResults) { - Log.logWarning("ResultFetcher", result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults); + //Log.logWarning("ResultFetcher", result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults); break; } diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index a0fb3318f..3103147d0 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -1345,7 +1345,7 @@ public final class Switchboard extends serverSwitch { } public void processSurrogate(final InputStream is, String name) throws IOException { - SurrogateReader reader = new SurrogateReader(is, 3); + SurrogateReader reader = new SurrogateReader(is, 100); Thread readerThread = new Thread(reader, name); readerThread.start(); DCEntry surrogate; diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 1d65b2d86..9ce93d55d 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -80,6 +80,7 @@ import net.yacy.repository.Blacklist; import org.apache.http.entity.mime.content.ContentBody; +import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.ResultURLs; import de.anomic.crawler.ResultURLs.EventOrigin; import de.anomic.search.ContentDomain; @@ -369,7 +370,7 @@ public final class yacyClient { } } - public static RSSFeed search(final yacySeed targetSeed, String query, boolean verify, boolean global, long timeout, int startRecord, int maximumRecords) throws IOException { + public static RSSFeed search(final yacySeed targetSeed, String query, CrawlProfile.CacheStrategy verify, boolean global, long timeout, int startRecord, int maximumRecords) throws IOException { String address = (targetSeed == null || targetSeed == Switchboard.getSwitchboard().peers.mySeed()) ? "localhost:" + Switchboard.getSwitchboard().getConfig("port", "8090") : targetSeed.getClusterAddress(); String urlBase = "http://" + address + "/yacysearch.rss"; return SearchSRURSS.loadSRURSS(urlBase, query, timeout, startRecord, maximumRecords, verify, global, null); diff --git a/source/net/yacy/cora/services/SearchHub.java b/source/net/yacy/cora/services/SearchHub.java index 4e56ae53d..38e2b3a57 100644 --- a/source/net/yacy/cora/services/SearchHub.java +++ b/source/net/yacy/cora/services/SearchHub.java @@ -30,6 +30,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import de.anomic.crawler.CrawlProfile; + import net.yacy.cora.document.RSSMessage; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.storage.ConcurrentScoreMap; @@ -145,7 +147,7 @@ public class SearchHub { * @param verify * @param global */ - public static void addSRURSSServices(SearchHub search, String[] rssServices, int count, boolean verify, boolean global, String userAgent) { + public static void addSRURSSServices(SearchHub search, String[] rssServices, int count, CrawlProfile.CacheStrategy verify, boolean global, String userAgent) { for (String service: rssServices) { SearchSRURSS accumulator = new SearchSRURSS(search, service, count, verify, global, userAgent); accumulator.start(); @@ -161,7 +163,7 @@ public class SearchHub { for (String s: args) sb.append(s).append(' '); String query = sb.toString().trim(); SearchHub search = new SearchHub(query, 10000); - addSRURSSServices(search, SRURSSServicesList, 100, false, false, "searchhub"); + addSRURSSServices(search, SRURSSServicesList, 100, CrawlProfile.CacheStrategy.CACHEONLY, false, "searchhub"); try {Thread.sleep(100);} catch (InterruptedException e1) {} search.waitTermination(); ScoreMap result = search.getResults(); diff --git a/source/net/yacy/cora/services/SearchSRURSS.java b/source/net/yacy/cora/services/SearchSRURSS.java index a732e5a8c..652abb954 100644 --- a/source/net/yacy/cora/services/SearchSRURSS.java +++ b/source/net/yacy/cora/services/SearchSRURSS.java @@ -36,6 +36,8 @@ import java.util.concurrent.TimeUnit; import org.apache.http.entity.mime.content.ContentBody; +import de.anomic.crawler.CrawlProfile; + import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.RSSFeed; import net.yacy.cora.document.RSSMessage; @@ -45,13 +47,13 @@ import net.yacy.cora.protocol.http.HTTPConnector; public class SearchSRURSS extends Thread implements SearchAccumulator { - private final static int recordsPerSession = 10; + private final static int recordsPerSession = 100; final String urlBase; final String query; final long timeoutInit; final int maximumRecordsInit; - final boolean verify; + final CrawlProfile.CacheStrategy verify; final boolean global; final Map> result; final String userAgent; @@ -64,7 +66,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator { final long timeoutInit, final String urlBase, final int maximumRecordsInit, - final boolean verify, + final CrawlProfile.CacheStrategy verify, final boolean global, final String userAgent) { this.results = new LinkedBlockingQueue(); @@ -82,7 +84,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator { final SearchHub search, final String urlBase, final int maximumRecordsInit, - final boolean verify, + final CrawlProfile.CacheStrategy verify, final boolean global, final String userAgent) { this.results = new LinkedBlockingQueue(); @@ -120,7 +122,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator { final String query, final long timeoutInit, final int maximumRecordsInit, - final boolean verify, + final CrawlProfile.CacheStrategy verify, final boolean global, final String userAgent) { Thread job = new Thread() { @@ -178,7 +180,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator { long timeout, int startRecord, int maximumRecords, - boolean verify, + CrawlProfile.CacheStrategy cacheStrategy, boolean global, String userAgent) throws IOException { MultiProtocolURI uri = null; @@ -195,7 +197,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator { parts.put("query", UTF8.StringBody(query)); parts.put("startRecord", UTF8.StringBody(Integer.toString(startRecord))); parts.put("maximumRecords", UTF8.StringBody(Long.toString(maximumRecords))); - parts.put("verify", UTF8.StringBody(verify ? "true" : "false")); + parts.put("verify", UTF8.StringBody(cacheStrategy.toName())); parts.put("resource", UTF8.StringBody(global ? "global" : "local")); parts.put("nav", UTF8.StringBody("none")); result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);