From 57484eb1cc63e529270e24cdce605878bbaf3d0b Mon Sep 17 00:00:00 2001 From: Michael Christen Date: Tue, 18 Feb 2020 14:40:50 +0100 Subject: [PATCH] xss protection --- htroot/yacysearch.java | 297 +++++++++++++++++----------------- htroot/yacysearchtrailer.java | 146 ++++++++--------- 2 files changed, 222 insertions(+), 221 deletions(-) diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 94d768ab0..c25afd85b 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -112,24 +112,24 @@ public class yacysearch { final boolean searchAllowed = sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true) || adminAuthenticated; boolean extendedSearchRights = adminAuthenticated; - + if(adminAuthenticated) { - authenticatedUserName = sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"); + authenticatedUserName = sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"); } else { - final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null; - if(user != null) { + final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null; + if(user != null) { extendedSearchRights = user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT); authenticatedUserName = user.getUserName(); - } + } } - + final boolean localhostAccess = header.accessFromLocalhost(); final String promoteSearchPageGreeting = (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? env.getConfig( "network.unit.description", "") : env.getConfig(SwitchboardConstants.GREETING, ""); final String client = header.getRemoteAddr(); // the search client who initiated the search - + // in case that the crawler is running and the search user is the peer admin, we expect that the user wants to check recently crawled document // to ensure that recent crawl results are inside the search results, we do a soft commit here. This is also important for live demos! if (extendedSearchRights && sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount() > 0) { @@ -137,13 +137,14 @@ public class yacysearch { } final boolean focus = (post == null) ? true : post.get("focus", "1").equals("1"); // get query - final String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); + String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); + originalquerystring = originalquerystring.replace('<', ' ').replace('>', ' '); // light xss protection String querystring = originalquerystring; CacheStrategy snippetFetchStrategy = (post == null) ? null : CacheStrategy.parse(post.get("verify", sb.getConfig("search.verify", ""))); - + final servletProperties prop = new servletProperties(); prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0); - prop.put("authSearch", authenticatedUserName != null); + prop.put("authSearch", authenticatedUserName != null); // produce vocabulary navigation sidebars Collection vocabularies = LibraryProvider.autotagging.getVocabularies(); @@ -161,7 +162,7 @@ public class yacysearch { final boolean rss = "rss.atom".contains(EXT); final boolean json = EXT.equals("json"); prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting); - + // adding some additional properties needed for the rss feed String peerContext = YaCyDefaultServlet.getContext(header, sb); prop.put("searchBaseURL", peerContext + "/yacysearch.html"); @@ -172,7 +173,7 @@ public class yacysearch { boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted; boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode); boolean stealthmode = p2pmode && !global; - + if ( post == null || indexSegment == null || env == null || !searchAllowed ) { if (indexSegment == null) ConcurrentLog.info("yacysearch", "indexSegment == null"); // we create empty entries for template strings @@ -214,18 +215,18 @@ public class yacysearch { return prop; } - if (post.containsKey("auth") && authenticatedUserName == null) { - /* - * Access to authentication protected features is explicitely requested here - * but no authentication is provided : ask now for authentication. + if (post.containsKey("auth") && authenticatedUserName == null) { + /* + * Access to authentication protected features is explicitely requested here + * but no authentication is provided : ask now for authentication. * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information * and as this page is not private, protected features would simply be hidden without asking browser again for authentication. * (see mantis 766 : http://mantis.tokeek.de/view.php?id=766) * - */ - prop.authenticationRequired(); - return prop; - } - + */ + prop.authenticationRequired(); + return prop; + } + // check for JSONP if ( post.containsKey("callback") ) { final String jsonp = post.get("callback") + "(["; @@ -245,7 +246,7 @@ public class yacysearch { // time zone int timezoneOffset = post.getInt("timezoneOffset", 0); - + // collect search attributes // check an determine items per page (max of [100 or configured default]} @@ -295,18 +296,18 @@ public class yacysearch { // find search domain final Classification.ContentDomain contentdom = post == null || !post.containsKey("contentdom") ? ContentDomain.ALL : ContentDomain.contentdomParser(post.get("contentdom", "all")); - + // Strict/extended content domain constraint : configured setting may be overriden by request param - final boolean strictContentDom = !Boolean.FALSE.toString().equalsIgnoreCase(post.get("strictContentDom", - sb.getConfig(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, - String.valueOf(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT)))); - - /* Maximum number of suggestions to display in the first results page */ + final boolean strictContentDom = !Boolean.FALSE.toString().equalsIgnoreCase(post.get("strictContentDom", + sb.getConfig(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, + String.valueOf(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT)))); + + /* Maximum number of suggestions to display in the first results page */ final int meanMax = post.getInt("meanCount", 0); - - boolean jsResort = global - && (contentdom == ContentDomain.ALL || contentdom == ContentDomain.TEXT) // For now JavaScript resorting can only be applied for text search - && sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT); + + boolean jsResort = global + && (contentdom == ContentDomain.ALL || contentdom == ContentDomain.TEXT) // For now JavaScript resorting can only be applied for text search + && sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT); // check the search tracker TreeSet trackerHandles = sb.localSearchTracker.get(client); @@ -328,16 +329,16 @@ public class yacysearch { ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM " + client + " gets no permission to search"); - if (!"html".equals(EXT)) { - /* API request : return the relevant HTTP status */ - throw new TemplateProcessingException("You are not allowed to search the web with this peer.", - HttpStatus.SC_FORBIDDEN); - } + if (!"html".equals(EXT)) { + /* API request : return the relevant HTTP status */ + throw new TemplateProcessingException("You are not allowed to search the web with this peer.", + HttpStatus.SC_FORBIDDEN); + } } else if ( !extendedSearchRights && !localhostAccess && !intranetMode ) { // in case that we do a global search or we want to fetch snippets, we check for DoS cases - final int accInThreeSeconds; - final int accInOneMinute; - final int accInTenMinutes; + final int accInThreeSeconds; + final int accInOneMinute; + final int accInTenMinutes; synchronized ( trackerHandles ) { accInThreeSeconds = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size(); @@ -348,14 +349,14 @@ public class yacysearch { } // protections against too strong YaCy network load, reduces remote search if ( global ) { - if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getDefaultValue()) - || accInOneMinute >= sb.getConfigInt( - SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getDefaultValue()) - || accInThreeSeconds >= sb.getConfigInt( - SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getDefaultValue())) { + if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getDefaultValue()) + || accInOneMinute >= sb.getConfigInt( + SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getDefaultValue()) + || accInThreeSeconds >= sb.getConfigInt( + SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getDefaultValue())) { global = false; jsResort = false; ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " @@ -369,13 +370,13 @@ public class yacysearch { + "/600s, " + " requests, disallowed global search"); } else if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_10MN.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_10MN.getDefaultValue()) - || accInOneMinute >= sb.getConfigInt( - SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_1MN.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_1MN.getDefaultValue()) - || accInThreeSeconds >= sb.getConfigInt( - SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_3S.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_3S.getDefaultValue())) { + SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_10MN.getDefaultValue()) + || accInOneMinute >= sb.getConfigInt( + SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_1MN.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_1MN.getDefaultValue()) + || accInThreeSeconds >= sb.getConfigInt( + SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_3S.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_3S.getDefaultValue())) { jsResort = false; ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client @@ -391,15 +392,15 @@ public class yacysearch { } // protection against too many remote server snippet loads (protects traffic on server) if ( snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ) { - if (accInTenMinutes >= sb.getConfigInt( - SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getDefaultValue()) - || accInOneMinute >= sb.getConfigInt( - SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getDefaultValue()) - || accInThreeSeconds >= sb.getConfigInt( - SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getDefaultValue())) { + if (accInTenMinutes >= sb.getConfigInt( + SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getDefaultValue()) + || accInOneMinute >= sb.getConfigInt( + SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getDefaultValue()) + || accInThreeSeconds >= sb.getConfigInt( + SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getDefaultValue())) { snippetFetchStrategy = CacheStrategy.CACHEONLY; ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client @@ -415,23 +416,23 @@ public class yacysearch { } // general load protection String timePeriodMsg = ""; - if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())) { - block = true; - timePeriodMsg = "ten minutes"; - prop.put("num-results_blockReason", 2); - } else if (accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())) { - block = true; - timePeriodMsg = "one minute"; - prop.put("num-results_blockReason", 3); - } else if (accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(), - SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) { - block = true; - timePeriodMsg = "three seconds"; - prop.put("num-results_blockReason", 4); - } - if(block) { + if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())) { + block = true; + timePeriodMsg = "ten minutes"; + prop.put("num-results_blockReason", 2); + } else if (accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())) { + block = true; + timePeriodMsg = "one minute"; + prop.put("num-results_blockReason", 3); + } else if (accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(), + SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) { + block = true; + timePeriodMsg = "three seconds"; + prop.put("num-results_blockReason", 4); + } + if(block) { ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client + ": " @@ -442,21 +443,21 @@ public class yacysearch { + accInTenMinutes + "/600s, " + " requests, disallowed search"); - if (!"html".equals(EXT)) { - /* - * API request : return the relevant HTTP status (429 - Too Many Requests - see - * https://tools.ietf.org/html/rfc6585#section-4) - */ - throw new TemplateProcessingException( - "You have reached the maximum allowed number of accesses to this search service within " - + timePeriodMsg + ". Please try again later or log in as administrator or as a user with extended search right.", - 429); - } + if (!"html".equals(EXT)) { + /* + * API request : return the relevant HTTP status (429 - Too Many Requests - see + * https://tools.ietf.org/html/rfc6585#section-4) + */ + throw new TemplateProcessingException( + "You have reached the maximum allowed number of accesses to this search service within " + + timePeriodMsg + ". Please try again later or log in as administrator or as a user with extended search right.", + 429); + } } } if (block) { - prop.put("num-results", 5); + prop.put("num-results", 5); } else { String urlmask = (post == null) ? ".*" : post.get("urlmaskfilter", ".*"); // the expression must be a subset of the java Match syntax described in http://lucene.apache.org/core/4_4_0/core/org/apache/lucene/util/automaton/RegExp.html String tld = null; @@ -475,7 +476,7 @@ public class yacysearch { // read collection modifier.collection = post.get("collection", modifier.collection); // post arguments may overrule parsed collection values - + int stp = querystring.indexOf('*'); if (stp >= 0) { // if the star appears as a single entry, use the catchallstring @@ -554,7 +555,7 @@ public class yacysearch { if (mt != null) { metatags.add(mt); } else { - + } } } @@ -587,13 +588,13 @@ public class yacysearch { querystring = querystring.replace("/heuristic", ""); modifier.add("/heuristic"); } - + final String tldModifierPrefix = "tld:"; final int tldp = querystring.indexOf(tldModifierPrefix, 0); if (tldp >= 0) { int ftb = querystring.indexOf(' ', tldp); if (ftb == -1) { - ftb = querystring.length(); + ftb = querystring.length(); } tld = querystring.substring(tldp + tldModifierPrefix.length(), ftb); querystring = querystring.replace(tldModifierPrefix + tld, ""); @@ -602,20 +603,20 @@ public class yacysearch { tld = tld.substring(1); } if (tld.length() == 0) { - tld = null; + tld = null; } else { - try { - /* Convert to the same lower case ASCII Compatible Encoding that is used in normalized URLs */ - tld = IDN.toASCII(tld, 0); - } catch(final IllegalArgumentException e){ - ConcurrentLog.warn("LOCAL_SEARCH", "Failed to convert tld modifier value " + tld + "to ASCII Compatible Encoding (ACE)", e); - } - + try { + /* Convert to the same lower case ASCII Compatible Encoding that is used in normalized URLs */ + tld = IDN.toASCII(tld, 0); + } catch(final IllegalArgumentException e){ + ConcurrentLog.warn("LOCAL_SEARCH", "Failed to convert tld modifier value " + tld + "to ASCII Compatible Encoding (ACE)", e); + } + /* Domain name in an URL is case insensitive : convert now modifier to lower case for further processing over normalized URLs */ tld = tld.toLowerCase(Locale.ROOT); } } - + if (urlmask == null || urlmask.isEmpty()) urlmask = ".*"; //if no urlmask was given // read the language from the language-restrict option 'lr' @@ -644,12 +645,12 @@ public class yacysearch { // filter out stopwords final SortedSet filtered = SetTools.joinConstructiveByTest(qg.getIncludeWords(), Switchboard.stopwords); //find matching stopwords qg.removeIncludeWords(filtered); - + // if a minus-button was hit, remove a special reference first if ( post != null && post.containsKey("deleteref") ) { try { if ( !sb.verifyAuthentication(header) ) { - prop.authenticationRequired(); + prop.authenticationRequired(); return prop; } @@ -680,7 +681,7 @@ public class yacysearch { // if a plus-button was hit, create new voting message if ( post != null && post.containsKey("recommendref") ) { if ( !sb.verifyAuthentication(header) ) { - prop.authenticationRequired(); + prop.authenticationRequired(); return prop; } final String recommendHash = post.get("recommendref", ""); // urlhash @@ -783,10 +784,10 @@ public class yacysearch { sb.getConfigSet("search.navigation")); theQuery.setStrictContentDom(strictContentDom); theQuery.setMaxSuggestions(meanMax); - theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, - QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)); - theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, - QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT)); + theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, + QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)); + theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, + QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT)); EventTracker.delete(EventTracker.EClass.SEARCH); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch( theQuery.id(true), @@ -837,9 +838,9 @@ public class yacysearch { sb.getConfigLong( SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000))); - + if(post.getBoolean("resortCachedResults") && cachedEvent == theSearch) { - theSearch.resortCachedResults(); + theSearch.resortCachedResults(); } if ( startRecord == 0 && extendedSearchRights && !stealthmode ) { @@ -888,9 +889,9 @@ public class yacysearch { try { suggestion = meanIt.next().toString(); prop.put("didYouMean_suggestions_" + meanCount + "_word", suggestion); - prop.put("didYouMean_suggestions_" + meanCount + "_url", - QueryParams.navUrlWithNewQueryString(RequestHeader.FileType.HTML, 0, theQuery, - suggestion, authenticatedUserName != null)); + prop.put("didYouMean_suggestions_" + meanCount + "_url", + QueryParams.navUrlWithNewQueryString(RequestHeader.FileType.HTML, 0, theQuery, + suggestion, authenticatedUserName != null)); prop.put("didYouMean_suggestions_" + meanCount + "_sep", "|"); meanCount++; } catch (final ConcurrentModificationException e) { @@ -926,7 +927,7 @@ public class yacysearch { prop.put("geoinfo_loc", i); prop.put("geoinfo", "1"); } - + // update the search tracker try { synchronized ( trackerHandles ) { @@ -957,17 +958,17 @@ public class yacysearch { prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.remote_rwi_stored.get() + theSearch.remote_solr_stored.get(), true)); prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true)); prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true)); - - prop.put("jsResort", jsResort); + + prop.put("jsResort", jsResort); prop.put("num-results_jsResort", jsResort); - - /* In p2p mode only and if JavaScript resorting is not enabled, add a link allowing user to resort already drained results, - * eventually including fetched results with higher ranks from the Solr and RWI stacks */ - prop.put("resortEnabled", !jsResort && global && !stealthmode && theSearch.resortCacheAllowed.availablePermits() > 0 ? 1 : 0); - prop.put("resortEnabled_url", - QueryParams.navurlBase(RequestHeader.FileType.HTML, theQuery, null, true, authenticatedUserName != null) - .append("&startRecord=").append(startRecord).append("&resortCachedResults=true") - .toString()); + + /* In p2p mode only and if JavaScript resorting is not enabled, add a link allowing user to resort already drained results, + * eventually including fetched results with higher ranks from the Solr and RWI stacks */ + prop.put("resortEnabled", !jsResort && global && !stealthmode && theSearch.resortCacheAllowed.availablePermits() > 0 ? 1 : 0); + prop.put("resortEnabled_url", + QueryParams.navurlBase(RequestHeader.FileType.HTML, theQuery, null, true, authenticatedUserName != null) + .append("&startRecord=").append(startRecord).append("&resortCachedResults=true") + .toString()); // generate the search result lines; the content will be produced by another servlet for ( int i = 0; i < theQuery.itemsPerPage(); i++ ) { @@ -1005,36 +1006,36 @@ public class yacysearch { prop.put("depth", "0"); prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0"); prop.put("jsResort_localQuery", theSearch.query.isLocal() ? "1" : "0"); - + final boolean showLogin = sb.getConfigBool(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN, - SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN_DEFAULT); + SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN_DEFAULT); if(showLogin) { - if(authenticatedUserName != null) { - /* Show the name of the authenticated user */ - prop.put("showLogin", 1); - prop.put("showLogin_userName", authenticatedUserName); - } else { - /* Show a login link */ - prop.put("showLogin", 2); - prop.put("showLogin_loginURL", - QueryParams.navurlBase(RequestHeader.FileType.HTML, theQuery, null, true, true).toString()); - } + if(authenticatedUserName != null) { + /* Show the name of the authenticated user */ + prop.put("showLogin", 1); + prop.put("showLogin_userName", authenticatedUserName); + } else { + /* Show a login link */ + prop.put("showLogin", 2); + prop.put("showLogin_loginURL", + QueryParams.navurlBase(RequestHeader.FileType.HTML, theQuery, null, true, true).toString()); + } } else { - prop.put("showLogin", 0); + prop.put("showLogin", 0); } } - + prop.put("focus", focus ? 1 : 0); // focus search field prop.put("searchagain", global ? "1" : "0"); - String former = originalquerystring.replaceAll(Segment.catchallString, "*"); + String former = originalquerystring.replaceAll(Segment.catchallString, "*"); // hide catchallString in output prop.putHTML("former", former); try { - prop.put("formerEncoded", URLEncoder.encode(former, StandardCharsets.UTF_8.name())); - } catch (UnsupportedEncodingException e) { - ConcurrentLog.warn("LOCAL_SEARCH", "Unsupported UTF-8 encoding!"); - prop.put("formerEncoded", former); - } + prop.put("formerEncoded", URLEncoder.encode(former, StandardCharsets.UTF_8.name())); + } catch (UnsupportedEncodingException e) { + ConcurrentLog.warn("LOCAL_SEARCH", "Unsupported UTF-8 encoding!"); + prop.put("formerEncoded", former); + } prop.put("count", itemsPerPage); prop.put("offset", startRecord); prop.put("resource", global ? "global" : "local"); diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 79bebe3d4..c32fdaecb 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -64,31 +64,31 @@ public class yacysearchtrailer { @SuppressWarnings({ }) public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - if (post == null) { - throw new TemplateMissingParameterException("The eventID parameter is required"); - } - + if (post == null) { + throw new TemplateMissingParameterException("The eventID parameter is required"); + } + final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; final String eventID = post.get("eventID", ""); - + final boolean adminAuthenticated = sb.verifyAuthentication(header); - + final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null; - final boolean authenticated = adminAuthenticated || user != null; - - if (post.containsKey("auth") && !authenticated) { - /* - * Authenticated search is explicitely requested here - * but no authentication is provided : ask now for authentication. + final boolean authenticated = adminAuthenticated || user != null; + + if (post.containsKey("auth") && !authenticated) { + /* + * Authenticated search is explicitely requested here + * but no authentication is provided : ask now for authentication. * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information * and as this page is not private, protected features would simply be hidden without asking browser again for authentication. * (see mantis 766 : http://mantis.tokeek.de/view.php?id=766) * - */ - prop.authenticationRequired(); - return prop; - } - + */ + prop.authenticationRequired(); + return prop; + } + // find search event final SearchEvent theSearch = SearchEventCache.getEvent(eventID); if (theSearch == null) { @@ -96,14 +96,13 @@ public class yacysearchtrailer { return prop; } final RequestHeader.FileType fileType = header.fileType(); - - + final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER); final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch; boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted; boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode); boolean stealthmode = p2pmode && !global; - + // compose search navigation ContentDomain contentdom = theSearch.getQuery().contentdom; prop.put("searchdomswitches", @@ -112,35 +111,36 @@ public class yacysearchtrailer { || sb.getConfigBool("search.video", true) || sb.getConfigBool("search.image", true) || sb.getConfigBool("search.app", true) ? 1 : 0); - - final String originalquerystring = post.get("query", post.get("search", "")).trim(); + + String originalquerystring = post.get("query", post.get("search", "")).trim(); + originalquerystring = originalquerystring.replace('<', ' ').replace('>', ' '); // light xss protection final String former = originalquerystring.replaceAll(Segment.catchallString, "*"); final CacheStrategy snippetFetchStrategy = CacheStrategy.parse(post.get("verify", sb.getConfig("search.verify", ""))); - final String snippetFetchStrategyName = snippetFetchStrategy == null - ? sb.getConfig("search.verify", CacheStrategy.IFFRESH.toName()) - : snippetFetchStrategy.toName(); + final String snippetFetchStrategyName = snippetFetchStrategy == null + ? sb.getConfig("search.verify", CacheStrategy.IFFRESH.toName()) + : snippetFetchStrategy.toName(); final int startRecord = post.getInt("startRecord", post.getInt("offset", post.getInt("start", 0))); - /* Maximum number of suggestions to display in the first results page */ + /* Maximum number of suggestions to display in the first results page */ final int meanMax = post.getInt("meanCount", 0); - + prop.put("resource-switches", adminAuthenticated && (stealthmode || global)); prop.put("resource-switches_global", adminAuthenticated && global); appendSearchFormValues("resource-switches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax); - + /* The search event has been found : we can render ranking switches */ prop.put("ranking-switches", true); appendSearchFormValues("ranking-switches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax); - + /* Add information about the current navigators generation (number of updates since their initialization) */ prop.put("ranking-switches_nav-generation", theSearch.getNavGeneration()); - + prop.put("ranking-switches_contextRanking", !former.contains(" /date")); prop.put("ranking-switches_contextRanking_formerWithoutDate", former.replace(" /date", "")); prop.put("ranking-switches_dateRanking", former.contains(" /date")); prop.put("ranking-switches_dateRanking_former", former); - + appendSearchFormValues("searchdomswitches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax); - + prop.put("searchdomswitches_searchtext", sb.getConfigBool("search.text", true) ? 1 : 0); prop.put("searchdomswitches_searchaudio", sb.getConfigBool("search.audio", true) ? 1 : 0); prop.put("searchdomswitches_searchvideo", sb.getConfigBool("search.video", true) ? 1 : 0); @@ -151,7 +151,7 @@ public class yacysearchtrailer { prop.put("searchdomswitches_searchvideo_check", (contentdom == ContentDomain.VIDEO) ? "1" : "0"); prop.put("searchdomswitches_searchimage_check", (contentdom == ContentDomain.IMAGE) ? "1" : "0"); prop.put("searchdomswitches_searchapp_check", (contentdom == ContentDomain.APP) ? "1" : "0"); - + appendSearchFormValues("searchdomswitches_strictContentDomSwitch_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax); prop.put("searchdomswitches_strictContentDomSwitch", (contentdom != ContentDomain.TEXT && contentdom != ContentDomain.ALL) ? 1 : 0); prop.put("searchdomswitches_strictContentDomSwitch_strictContentDom", theSearch.getQuery().isStrictContentDom() ? 1 : 0); @@ -189,8 +189,8 @@ public class yacysearchtrailer { count = entry.getValue(); prop.put(fileType, "nav-topics_element_" + i + "_modifier", name); prop.put(fileType, "nav-topics_element_" + i + "_name", name); - prop.putUrlEncoded(fileType, "nav-topics_element_" + i + "_url", QueryParams - .navurl(fileType, 0, theSearch.query, name, false, authenticated).toString()); + prop.putUrlEncoded(fileType, "nav-topics_element_" + i + "_url", QueryParams + .navurl(fileType, 0, theSearch.query, name, false, authenticated).toString()); prop.put("nav-topics_element_" + i + "_count", count); int fontsize = TOPWORDS_MINSIZE + (TOPWORDS_MAXSIZE - TOPWORDS_MINSIZE) * (count - mincount) / (1 + maxcount - mincount); fontsize = Math.max(TOPWORDS_MINSIZE, fontsize - (name.length() - 5)); @@ -203,7 +203,7 @@ public class yacysearchtrailer { i--; prop.put("nav-topics_element_" + i + "_nl", 0); } - + // protocol navigators if (theSearch.protocolNavigator == null || theSearch.protocolNavigator.isEmpty()) { prop.put("nav-protocols", 0); @@ -233,15 +233,15 @@ public class yacysearchtrailer { prop.put("nav-protocols_element_" + i + "_on", 0); prop.put("nav-protocols_element_" + i + "_onclick", 0); prop.put(fileType, "nav-protocols_element_" + i + "_modifier", nav); - url = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString(); + url = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString(); } else { neg++; prop.put("nav-protocols_element_" + i + "_on", 1); prop.put("nav-protocols_element_" + i + "_onclick", 1); prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav); - url = QueryParams - .navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav, authenticated) - .toString(); + url = QueryParams + .navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav, authenticated) + .toString(); } prop.put(fileType, "nav-protocols_element_" + i + "_name", name); prop.put("nav-protocols_element_" + i + "_onclick_url", url); @@ -278,7 +278,7 @@ public class yacysearchtrailer { if (name.length() < 10) continue; count = theSearch.dateNavigator.get(name); if(count == 0) { - continue; + continue; } String shortname = name.substring(0, 10); long d = Instant.parse(name).toEpochMilli(); @@ -301,7 +301,7 @@ public class yacysearchtrailer { pos++; prop.put("nav-dates_element_" + i + "_on", 1); } else { - neg++; + neg++; prop.put("nav-dates_element_" + i + "_on", 0); } prop.put(fileType, "nav-dates_element_" + i + "_name", shortname); @@ -347,7 +347,7 @@ public class yacysearchtrailer { navUrl = QueryParams.navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav, authenticated); } prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_name", name); - prop.putUrlEncoded(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", navUrl); + prop.putUrlEncoded(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", navUrl); prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_id", "vocabulary_" + navname + "_" + i); prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_count", count); prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_nl", 1); @@ -376,35 +376,35 @@ public class yacysearchtrailer { prop.put("navs_" + ni + "_displayname", navi.getDisplayName()); prop.put("navs_" + ni + "_name", naviname); prop.put("navs_" + ni + "_count", navi.size()); - - final int navSort; + + final int navSort; if(navi.getSort() == null) { - navSort = 0; + navSort = 0; } else { - if(navi.getSort().getSortType() == NavigatorSortType.COUNT) { - if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) { - navSort = 0; - } else { - navSort = 1; - } - } else { - if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) { - navSort = 2; - } else { - navSort = 3; - } - } + if(navi.getSort().getSortType() == NavigatorSortType.COUNT) { + if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) { + navSort = 0; + } else { + navSort = 1; + } + } else { + if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) { + navSort = 2; + } else { + navSort = 3; + } + } } - prop.put("navs_" + ni + "_navSort", navSort); + prop.put("navs_" + ni + "_navSort", navSort); - navigatorIterator = navi.navigatorKeys(); + navigatorIterator = navi.navigatorKeys(); int i = 0, pos = 0, neg = 0; String nav, rawNav; while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) { name = navigatorIterator.next(); count = navi.get(name); if (count == 0) { - /* This entry has a zero count, but the next may be positive */ + /* This entry has a zero count, but the next may be positive */ continue; } @@ -420,24 +420,24 @@ public class yacysearchtrailer { pos++; prop.put("navs_" + ni + "_element_" + i + "_on", 1); prop.put(fileType, "navs_" + ni + "_element_" + i + "_modifier", nav); - navUrl = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString(); + navUrl = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString(); } else { neg++; prop.put("navs_" + ni + "_element_" + i + "_on", 0); prop.put(fileType, "navs_" + ni + "_element_" + i + "_modifier", "-" + nav); - navUrl = QueryParams.navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav, - authenticated); + navUrl = QueryParams.navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav, + authenticated); } prop.put(fileType, "navs_" + ni + "_element_" + i + "_name", navi.getElementDisplayName(name)); - prop.putUrlEncoded(fileType, "navs_" + ni + "_element_" + i + "_url", navUrl); + prop.putUrlEncoded(fileType, "navs_" + ni + "_element_" + i + "_url", navUrl); prop.put(fileType, "navs_" + ni + "_element_" + i + "_id", naviname + "_" + i); prop.put("navs_" + ni + "_element_" + i + "_count", count); prop.put("navs_" + ni + "_element_" + i + "_nl", 1); i++; } if(i == 0) { - /* The navigator has only entries with value==0 : this is equivalent to empty navigator */ - continue; + /* The navigator has only entries with value==0 : this is equivalent to empty navigator */ + continue; } prop.put("navs_" + ni + "_element", i); prop.put("navs_" + ni + "_count", i); @@ -491,10 +491,10 @@ public class yacysearchtrailer { * @param startRecord the zero based index of the first record to return * @param meanMax the maximum number of suggestions ("Did you mean") to propose */ - private static void appendSearchFormValues(final String templatePrefix, final serverObjects post, final serverObjects prop, - boolean global, final SearchEvent theSearch, final String former, final String snippetFetchStrategyName, - final int startRecord, final int meanMax) { - prop.putHTML(templatePrefix + "former", former); + private static void appendSearchFormValues(final String templatePrefix, final serverObjects post, final serverObjects prop, + boolean global, final SearchEvent theSearch, final String former, final String snippetFetchStrategyName, + final int startRecord, final int meanMax) { + prop.putHTML(templatePrefix + "former", former); prop.put(templatePrefix + "authSearch", post.containsKey("auth")); prop.put(templatePrefix + "contentdom", post.get("contentdom", "text")); prop.put(templatePrefix + "strictContentDom", String.valueOf(theSearch.getQuery().isStrictContentDom())); @@ -507,7 +507,7 @@ public class yacysearchtrailer { prop.put(templatePrefix + "depth", "0"); prop.put(templatePrefix + "constraint", (theSearch.getQuery().constraint == null) ? "" : theSearch.getQuery().constraint.exportB64()); prop.put(templatePrefix + "meanCount", meanMax); - } + } } //http://localhost:8090/yacysearch.html?query=java+&maximumRecords=10&resource=local&verify=cacheonly&nav=hosts,authors,namespace,topics,filetype,protocol&urlmaskfilter=ftp://.*&prefermaskfilter=&constraint=&contentdom=text&former=java+%2Fftp&startRecord=0