diff --git a/htroot/api/timeline.java b/htroot/api/timeline.java index d6fbba266..07c10a8b8 100644 --- a/htroot/api/timeline.java +++ b/htroot/api/timeline.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Date; import java.util.Iterator; +import java.util.List; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; @@ -68,7 +69,7 @@ public final class timeline { language = (agent == null) ? "en" : ISO639.userAgentLanguageDetection(agent); if (language == null) language = "en"; } - final Collection[] query = QueryParams.cleanQuery(querystring); // converts also umlaute + final List[] query = QueryParams.cleanQuery(querystring); // converts also umlaute HandleSet q = Word.words2hashesHandles(query[0]); // tell all threads to do nothing for a specific time diff --git a/htroot/gsa/searchresult.java b/htroot/gsa/searchresult.java index 328f58a7b..5a39fbc98 100644 --- a/htroot/gsa/searchresult.java +++ b/htroot/gsa/searchresult.java @@ -24,6 +24,7 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -110,7 +111,7 @@ public class searchresult { post.put("originalQuery", q); // get a solr query string - Collection[] cq = QueryParams.cleanQuery(q); + List[] cq = QueryParams.cleanQuery(q); q = QueryParams.solrQueryString(cq[0], cq[1], sb.index.fulltext().getSolrScheme()).toString(); post.put(CommonParams.ROWS, post.remove("num")); diff --git a/htroot/index.html b/htroot/index.html index fde513383..fe1e398c2 100644 --- a/htroot/index.html +++ b/htroot/index.html @@ -71,7 +71,6 @@ #(searchoptions)# - @@ -96,10 +95,6 @@ #(/resource-select)# - - : - - : @@ -189,7 +184,7 @@
browser integration
after searching, click-open on the default search engine in the upper right search field of your browser and select 'Add "YaCy Search.."'
search as rss feed
-
click on the red icon in the upper right after a search. this works good in combination with the '/date' ranking modifier. See an example.
+
click on the red icon in the upper right after a search. this works good in combination with the '/date' ranking modifier. See an example.
json search results
for ajax developers: get the search rss feed and replace the '.rss' extension in the search result url with '.json'
diff --git a/htroot/index.java b/htroot/index.java index 79ad68e05..8304a12e9 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -74,7 +74,6 @@ public class index { final String former = (post == null) ? "" : post.get("former", ""); final int count = Math.min(100, (post == null) ? 10 : post.getInt("count", 10)); final int maximumRecords = sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10); - final String urlmaskfilter = (post == null) ? ".*" : post.get("urlmaskfilter", ".*"); final String prefermaskfilter = (post == null) ? "" : post.get("prefermaskfilter", ""); final String constraint = (post == null) ? "" : post.get("constraint", ""); final String cat = (post == null) ? "href" : post.get("cat", "href"); @@ -114,7 +113,6 @@ public class index { prop.put("searchoptions_resource-select_global", global ? "1" : "0"); prop.put("searchoptions_resource-select_global-disabled", indexReceiveGranted ? "0" : "1"); prop.put("searchoptions_resource-select_local", global ? "0" : "1"); - prop.putHTML("searchoptions_urlmaskfilter", urlmaskfilter); prop.put("searchoptions_prefermaskoptions", "0"); prop.putHTML("searchoptions_prefermaskoptions_prefermaskfilter", prefermaskfilter); prop.put("searchoptions_indexofChecked", ""); diff --git a/htroot/yacysearch.html b/htroot/yacysearch.html index 24a7db77a..80b206005 100644 --- a/htroot/yacysearch.html +++ b/htroot/yacysearch.html @@ -120,7 +120,7 @@ $(function() { #(searchvideo)#::  #(/searchvideo)# #(searchapp)#::#(/searchapp)#   - more options + more options #(/searchdomswitches)# @@ -129,7 +129,6 @@ $(function() { - diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 7dc4a8b3f..9d814ea44 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -34,6 +34,7 @@ import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; @@ -217,13 +218,6 @@ public class yacysearch { boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0; final boolean indexof = (post != null && post.get("indexof", "").equals("on")); - final String originalUrlMask; - if ( post.containsKey("urlmaskfilter") ) { - originalUrlMask = post.get("urlmaskfilter", ".*"); - } else { - originalUrlMask = ".*"; - } - String prefermask = (post == null) ? "" : post.get("prefermaskfilter", ""); if ( !prefermask.isEmpty() && prefermask.indexOf(".*", 0) < 0 ) { prefermask = ".*" + prefermask + ".*"; @@ -578,7 +572,7 @@ public class yacysearch { } } if ( urlmask == null || urlmask.isEmpty() ) { - urlmask = originalUrlMask; + urlmask = ".*"; } //if no urlmask was given // read the language from the language-restrict option 'lr' @@ -600,7 +594,7 @@ public class yacysearch { } // the query - final Collection[] query = QueryParams.cleanQuery(querystring.trim()); // converts also umlaute + final List[] query = QueryParams.cleanQuery(querystring.trim()); // converts also umlaute final int maxDistance = (querystring.indexOf('"', 0) >= 0) ? query.length - 1 : Integer.MAX_VALUE; @@ -863,8 +857,7 @@ public class yacysearch { "html", 0, theQuery, - suggestion, - originalUrlMask.toString()).toString()); + suggestion).toString()); prop.put("didYouMean_suggestions_" + meanCount + "_sep", "|"); meanCount++; } catch (ConcurrentModificationException e) {break meanCollect;} @@ -936,12 +929,7 @@ public class yacysearch { .append("\"arrowleft\" "); } else { resnav.append("\"arrowleft\" "); } @@ -956,9 +944,7 @@ public class yacysearch { resnav.append("\" width=\"16\" height=\"16\" /> "); } else { resnav.append("\"page");"); } else { resnav.append("\"arrowright\""); + resnav.append(QueryParams.navurl("html", thispage + 1, theQuery, null).toString()); + resnav.append("\">\"arrowright\""); } final String resnavs = resnav.toString(); prop.put("num-results_resnav", resnavs); @@ -1035,7 +1015,6 @@ public class yacysearch { prop.put("count", itemsPerPage); prop.put("offset", startRecord); prop.put("resource", global ? "global" : "local"); - prop.putHTML("urlmaskfilter", originalUrlMask); prop.putHTML("prefermaskfilter", prefermask); prop.put("indexof", (indexof) ? "on" : "off"); prop.put("constraint", (constraint == null) ? "" : constraint.exportB64()); diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index ef698d340..d04611b45 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -100,7 +100,7 @@ public class yacysearchitem { prop.put("remoteResourceSize", Formatter.number(theSearch.query.remote_stored.get(), true)); prop.put("remoteIndexCount", Formatter.number(theSearch.query.remote_available.get(), true)); prop.put("remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true)); - prop.put("navurlBase", QueryParams.navurlBase("html", theSearch.query, null, theSearch.query.urlMask.toString()).toString()); + prop.put("navurlBase", QueryParams.navurlBase("html", theSearch.query, null)); final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, ""); if (theSearch.query.contentdom == Classification.ContentDomain.TEXT || theSearch.query.contentdom == Classification.ContentDomain.ALL) { diff --git a/htroot/yacysearchlatestinfo.java b/htroot/yacysearchlatestinfo.java index 116d49941..faa871256 100644 --- a/htroot/yacysearchlatestinfo.java +++ b/htroot/yacysearchlatestinfo.java @@ -42,7 +42,7 @@ public class yacysearchlatestinfo { prop.put("remoteResourceSize", Formatter.number(theSearch.query.remote_stored.get(), true)); prop.put("remoteIndexCount", Formatter.number(theSearch.query.remote_available.get(), true)); prop.put("remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true)); - prop.putJSON("navurlBase", QueryParams.navurlBase("html", theSearch.query, null, theSearch.query.urlMask.toString()).toString()); + prop.putJSON("navurlBase", QueryParams.navurlBase("html", theSearch.query, null).toString()); return prop; } diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index a4d551f80..b41e8d054 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -93,7 +93,7 @@ public class yacysearchtrailer { queryStringForUrl = (queryStringForUrl.substring(0, p) + queryStringForUrl.substring(p + nav.length())).trim(); } prop.put(fileType, "nav-namespace_element_" + i + "_name", name); - prop.put(fileType, "nav-namespace_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString()).toString()); + prop.put(fileType, "nav-namespace_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl).toString()); prop.put("nav-namespace_element_" + i + "_count", count); prop.put("nav-namespace_element_" + i + "_nl", 1); i++; @@ -133,7 +133,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-domains_element_" + i + "_name", name); - prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString()).toString()); + prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl).toString()); prop.put("nav-domains_element_" + i + "_count", count); prop.put("nav-domains_element_" + i + "_nl", 1); i++; @@ -172,7 +172,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-authors_element_" + i + "_name", name); - prop.put(fileType, "nav-authors_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString()).toString()); + prop.put(fileType, "nav-authors_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl).toString()); prop.put("nav-authors_element_" + i + "_count", count); prop.put("nav-authors_element_" + i + "_nl", 1); i++; @@ -203,7 +203,7 @@ public class yacysearchtrailer { prop.put("nav-topics_element_" + i + "_on", 1); prop.put(fileType, "nav-topics_element_" + i + "_modifier", name); prop.put(fileType, "nav-topics_element_" + i + "_name", name); - prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl + "+" + name, theSearch.query.urlMask.toString()).toString()); + prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl + "+" + name).toString()); prop.put("nav-topics_element_" + i + "_count", count); prop.put("nav-topics_element_" + i + "_nl", 1); i++; @@ -241,7 +241,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-protocols_element_" + i + "_name", name); - prop.put(fileType, "nav-protocols_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, (p >= 0 && theSearch.query.urlMask.toString().startsWith(name)) ? ".*" : theSearch.query.urlMask.toString()).toString()); + prop.put(fileType, "nav-protocols_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl).toString()); prop.put("nav-protocols_element_" + i + "_count", count); prop.put("nav-protocols_element_" + i + "_nl", 1); i++; @@ -280,7 +280,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-filetypes_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-filetypes_element_" + i + "_name", name); - prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, (p >= 0 && theSearch.query.urlMask.toString().endsWith(name)) ? ".*" : theSearch.query.urlMask.toString()).toString()); + prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl).toString()); prop.put("nav-filetypes_element_" + i + "_count", count); prop.put("nav-filetypes_element_" + i + "_nl", 1); i++; @@ -322,7 +322,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_name", name); - prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString()).toString()); + prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl).toString()); prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_count", count); prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_nl", 1); i++; diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 6250a029b..3916a9d0b 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -1081,7 +1081,7 @@ public final class Protocol docList = rsp.getResults(); // no need to close this here because that sends a commit to remote solr which is not wanted here } catch (IOException e) { - Network.log.logInfo("SEARCH failed (solr), Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")", e); + Network.log.logInfo("SEARCH failed (solr), Peer: " +target.getName() + "/" + target.getPublicAddress() + " (" + e.getMessage() + ")", e); return -1; } } diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index d35a4837c..6c914c776 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -34,6 +34,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; @@ -114,7 +115,7 @@ public final class QueryParams { public final String queryString; public final HandleSet query_include_hashes, query_exclude_hashes, query_all_hashes; - private final Collection query_include_words, query_exclude_words, query_all_words; + private final List query_include_words, query_exclude_words, query_all_words; public int itemsPerPage; public int offset; public final Pattern urlMask, prefer; @@ -183,7 +184,7 @@ public final class QueryParams { } } else { this.queryString = queryString; - final Collection[] cq = cleanQuery(queryString); + final List[] cq = cleanQuery(queryString); this.query_include_words = cq[0]; this.query_exclude_words = cq[1]; this.query_all_words = cq[2]; @@ -237,9 +238,9 @@ public final class QueryParams { public QueryParams( final String queryString, - final Collection queryWords, - final Collection excludeWords, - final Collection fullqueryWords, + final List queryWords, + final List excludeWords, + final List fullqueryWords, final HandleSet queryHashes, final HandleSet excludeHashes, final HandleSet fullqueryHashes, @@ -445,11 +446,11 @@ public final class QueryParams { private static String seps = "'.,/&_"; static {seps += '"';} @SuppressWarnings("unchecked") - public static Collection[] cleanQuery(String querystring) { + public static List[] cleanQuery(String querystring) { // returns three sets: a query set, an exclude set and a full query set - final Collection query_include_words = new ArrayList(); - final Collection query_exclude_words = new ArrayList(); - final Collection query_all_words = new ArrayList(); + final List query_include_words = new ArrayList(); + final List query_exclude_words = new ArrayList(); + final List query_all_words = new ArrayList(); if ((querystring != null) && (!querystring.isEmpty())) { @@ -484,7 +485,7 @@ public final class QueryParams { } } } - return new Collection[]{query_include_words, query_exclude_words, query_all_words}; + return new List[]{query_include_words, query_exclude_words, query_all_words}; } public String queryString(final boolean encodeHTML) { @@ -533,29 +534,41 @@ public final class QueryParams { else q.append(" AND ").append(YaCySchema.host_id_s.getSolrFieldName()).append(":\"").append(this.nav_sitehash).append('\"'); } - String urlMaskPattern = this.urlMask.pattern(); - - // translate filetype navigation - int extm = urlMaskPattern.indexOf(".*\\."); - if (extm >= 0) { - String ext = urlMaskPattern.substring(extm + 4); - q.append(" AND ").append(YaCySchema.url_file_ext_s.getSolrFieldName()).append(':').append(ext); - } - - // translate protocol navigation - if (urlMaskPattern.startsWith("http://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("http"); - else if (urlMaskPattern.startsWith("https://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("https"); - else if (urlMaskPattern.startsWith("ftp://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("ftp"); - else if (urlMaskPattern.startsWith("smb://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("smb"); - else if (urlMaskPattern.startsWith("file://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("file"); // construct query final SolrQuery params = new SolrQuery(); - params.setQuery(q.toString()); params.setStart(this.offset); params.setRows(this.itemsPerPage); params.setFacet(false); + if (!this.urlMask_isCatchall) { + String urlMaskPattern = this.urlMask.pattern(); + + // translate filetype navigation + int extm = urlMaskPattern.indexOf(".*\\."); + if (extm >= 0) { + String ext = urlMaskPattern.substring(extm + 4); + q.append(" AND ").append(YaCySchema.url_file_ext_s.getSolrFieldName()).append(':').append(ext); + } + + // translate protocol navigation + if (urlMaskPattern.startsWith("http://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("http"); + else if (urlMaskPattern.startsWith("https://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("https"); + else if (urlMaskPattern.startsWith("ftp://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("ftp"); + else if (urlMaskPattern.startsWith("smb://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("smb"); + else if (urlMaskPattern.startsWith("file://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("file"); + + // add a filter query on urls + // solr doesn't like slashes, backslashes or doublepoints; remove them + int p; + while ((p = urlMaskPattern.indexOf("\\")) >= 0) urlMaskPattern = urlMaskPattern.substring(0, p) + "." + urlMaskPattern.substring(p + 2); + while ((p = urlMaskPattern.indexOf(':')) >= 0) urlMaskPattern = urlMaskPattern.substring(0, p) + "." + urlMaskPattern.substring(p + 1); + while ((p = urlMaskPattern.indexOf('/')) >= 0) urlMaskPattern = urlMaskPattern.substring(0, p) + "." + urlMaskPattern.substring(p + 1); + params.setFilterQueries(YaCySchema.sku.getSolrFieldName() + ":/" + urlMaskPattern + "/"); + } + + params.setQuery(q.toString()); + if (this.radius > 0.0d && this.lat != 0.0d && this.lon != 0.0d) { // localtion search, no special ranking // try http://localhost:8090/solr/select?q=*:*&fq={!bbox sfield=coordinate_p pt=50.17,8.65 d=1} @@ -579,9 +592,15 @@ public final class QueryParams { return params; } - public static StringBuilder solrQueryString(Collection include, Collection exclude, SolrConfiguration configuration) { + public static StringBuilder solrQueryString(List include, List exclude, SolrConfiguration configuration) { final StringBuilder q = new StringBuilder(80); + // parse special requests + if (include.size() == 1 && exclude.size() == 0) { + String w = include.get(0); + if (Segment.catchallString.equals(w)) return new StringBuilder("*:*"); + } + // add text query int wc = 0; StringBuilder w = new StringBuilder(80); @@ -628,7 +647,7 @@ public final class QueryParams { } } - public Collection[] queryWords() { + public List[] queryWords() { return cleanQuery(this.queryString); } @@ -719,11 +738,9 @@ public final class QueryParams { * @param addToQuery * @return */ - public static StringBuilder navurl( - final String ext, final int page, final QueryParams theQuery, - final String newQueryString, final String originalUrlMask) { + public static StringBuilder navurl(final String ext, final int page, final QueryParams theQuery, final String newQueryString) { - final StringBuilder sb = navurlBase(ext, theQuery, newQueryString, originalUrlMask); + final StringBuilder sb = navurlBase(ext, theQuery, newQueryString); sb.append(ampersand); sb.append("startRecord="); @@ -732,9 +749,7 @@ public final class QueryParams { return sb; } - public static StringBuilder navurlBase( - final String ext, final QueryParams theQuery, - final String newQueryString, final String originalUrlMask) { + public static StringBuilder navurlBase(final String ext, final QueryParams theQuery, final String newQueryString) { final StringBuilder sb = new StringBuilder(120); sb.append("/yacysearch."); @@ -754,10 +769,6 @@ public final class QueryParams { sb.append("verify="); sb.append(theQuery.snippetCacheStrategy == null ? "false" : theQuery.snippetCacheStrategy.toName()); - sb.append(ampersand); - sb.append("urlmaskfilter="); - sb.append(originalUrlMask); - sb.append(ampersand); sb.append("prefermaskfilter="); sb.append(theQuery.prefer);