boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

xss protection

Browse Source
pull/338/head
Michael Christen 5 years ago
parent 090c0e56ab
commit 57484eb1cc
2 changed files with 222 additions and 221 deletions
Show Stats Download Patch File Download Diff File

297
htroot/yacysearch.java
Unescape View File

@ -112,24 +112,24 @@ public class yacysearch {
final boolean searchAllowed = sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true) || adminAuthenticated;
boolean extendedSearchRights = adminAuthenticated;
if(adminAuthenticated) {
authenticatedUserName = sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");
authenticatedUserName = sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");
} else {
final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null;
if(user != null) {
final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null;
if(user != null) {
extendedSearchRights = user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT);
authenticatedUserName = user.getUserName();
}
}
}
final boolean localhostAccess = header.accessFromLocalhost();
final String promoteSearchPageGreeting =
(env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? env.getConfig(
"network.unit.description",
"") : env.getConfig(SwitchboardConstants.GREETING, "");
final String client = header.getRemoteAddr(); // the search client who initiated the search
// in case that the crawler is running and the search user is the peer admin, we expect that the user wants to check recently crawled document
// to ensure that recent crawl results are inside the search results, we do a soft commit here. This is also important for live demos!
if (extendedSearchRights && sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount() > 0) {
@ -137,13 +137,14 @@ public class yacysearch {
}
final boolean focus = (post == null) ? true : post.get("focus", "1").equals("1");
// get query
final String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim();
String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim();
originalquerystring = originalquerystring.replace('<', ' ').replace('>', ' '); // light xss protection
String querystring = originalquerystring;
CacheStrategy snippetFetchStrategy = (post == null) ? null : CacheStrategy.parse(post.get("verify", sb.getConfig("search.verify", "")));
final servletProperties prop = new servletProperties();
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);
prop.put("authSearch", authenticatedUserName != null);
prop.put("authSearch", authenticatedUserName != null);
// produce vocabulary navigation sidebars
Collection<Tagging> vocabularies = LibraryProvider.autotagging.getVocabularies();
@ -161,7 +162,7 @@ public class yacysearch {
final boolean rss = "rss.atom".contains(EXT);
final boolean json = EXT.equals("json");
prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting);
// adding some additional properties needed for the rss feed
String peerContext = YaCyDefaultServlet.getContext(header, sb);
prop.put("searchBaseURL", peerContext + "/yacysearch.html");
@ -172,7 +173,7 @@ public class yacysearch {
boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);
boolean stealthmode = p2pmode && !global;
if ( post == null || indexSegment == null || env == null || !searchAllowed ) {
if (indexSegment == null) ConcurrentLog.info("yacysearch", "indexSegment == null");
// we create empty entries for template strings
@ -214,18 +215,18 @@ public class yacysearch {
return prop;
}
if (post.containsKey("auth") && authenticatedUserName == null) {
/*
* Access to authentication protected features is explicitely requested here
* but no authentication is provided : ask now for authentication.
if (post.containsKey("auth") && authenticatedUserName == null) {
/*
* Access to authentication protected features is explicitely requested here
* but no authentication is provided : ask now for authentication.
* Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information
* and as this page is not private, protected features would simply be hidden without asking browser again for authentication.
* (see mantis 766 : http://mantis.tokeek.de/view.php?id=766) *
*/
prop.authenticationRequired();
return prop;
}
*/
prop.authenticationRequired();
return prop;
}
// check for JSONP
if ( post.containsKey("callback") ) {
final String jsonp = post.get("callback") + "([";
@ -245,7 +246,7 @@ public class yacysearch {
// time zone
int timezoneOffset = post.getInt("timezoneOffset", 0);
// collect search attributes
// check an determine items per page (max of [100 or configured default]}
@ -295,18 +296,18 @@ public class yacysearch {
// find search domain
final Classification.ContentDomain contentdom = post == null || !post.containsKey("contentdom") ? ContentDomain.ALL : ContentDomain.contentdomParser(post.get("contentdom", "all"));
// Strict/extended content domain constraint : configured setting may be overriden by request param
final boolean strictContentDom = !Boolean.FALSE.toString().equalsIgnoreCase(post.get("strictContentDom",
sb.getConfig(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM,
String.valueOf(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT))));
/* Maximum number of suggestions to display in the first results page */
final boolean strictContentDom = !Boolean.FALSE.toString().equalsIgnoreCase(post.get("strictContentDom",
sb.getConfig(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM,
String.valueOf(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT))));
/* Maximum number of suggestions to display in the first results page */
final int meanMax = post.getInt("meanCount", 0);
boolean jsResort = global
&& (contentdom == ContentDomain.ALL || contentdom == ContentDomain.TEXT) // For now JavaScript resorting can only be applied for text search
&& sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT);
boolean jsResort = global
&& (contentdom == ContentDomain.ALL || contentdom == ContentDomain.TEXT) // For now JavaScript resorting can only be applied for text search
&& sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT);
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
@ -328,16 +329,16 @@ public class yacysearch {
ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM "
+ client
+ " gets no permission to search");
if (!"html".equals(EXT)) {
/* API request : return the relevant HTTP status */
throw new TemplateProcessingException("You are not allowed to search the web with this peer.",
HttpStatus.SC_FORBIDDEN);
}
if (!"html".equals(EXT)) {
/* API request : return the relevant HTTP status */
throw new TemplateProcessingException("You are not allowed to search the web with this peer.",
HttpStatus.SC_FORBIDDEN);
}
} else if ( !extendedSearchRights && !localhostAccess && !intranetMode ) {
// in case that we do a global search or we want to fetch snippets, we check for DoS cases
final int accInThreeSeconds;
final int accInOneMinute;
final int accInTenMinutes;
final int accInThreeSeconds;
final int accInOneMinute;
final int accInTenMinutes;
synchronized ( trackerHandles ) {
accInThreeSeconds =
trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size();
@ -348,14 +349,14 @@ public class yacysearch {
}
// protections against too strong YaCy network load, reduces remote search
if ( global ) {
if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getDefaultValue())
|| accInOneMinute >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getDefaultValue())
|| accInThreeSeconds >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getDefaultValue())) {
if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getDefaultValue())
|| accInOneMinute >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getDefaultValue())
|| accInThreeSeconds >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getDefaultValue())) {
global = false;
jsResort = false;
ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
@ -369,13 +370,13 @@ public class yacysearch {
+ "/600s, "
+ " requests, disallowed global search");
} else if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_10MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_10MN.getDefaultValue())
|| accInOneMinute >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_1MN.getDefaultValue())
|| accInThreeSeconds >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_3S.getDefaultValue())) {
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_10MN.getDefaultValue())
|| accInOneMinute >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_1MN.getDefaultValue())
|| accInThreeSeconds >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_P2P_JSRESORT_ACCESS_3S.getDefaultValue())) {
jsResort = false;
ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
+ client
@ -391,15 +392,15 @@ public class yacysearch {
}
// protection against too many remote server snippet loads (protects traffic on server)
if ( snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ) {
if (accInTenMinutes >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getDefaultValue())
|| accInOneMinute >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getDefaultValue())
|| accInThreeSeconds >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getDefaultValue())) {
if (accInTenMinutes >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getDefaultValue())
|| accInOneMinute >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getDefaultValue())
|| accInThreeSeconds >= sb.getConfigInt(
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getDefaultValue())) {
snippetFetchStrategy = CacheStrategy.CACHEONLY;
ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
+ client
@ -415,23 +416,23 @@ public class yacysearch {
}
// general load protection
String timePeriodMsg = "";
if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())) {
block = true;
timePeriodMsg = "ten minutes";
prop.put("num-results_blockReason", 2);
} else if (accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())) {
block = true;
timePeriodMsg = "one minute";
prop.put("num-results_blockReason", 3);
} else if (accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) {
block = true;
timePeriodMsg = "three seconds";
prop.put("num-results_blockReason", 4);
}
if(block) {
if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())) {
block = true;
timePeriodMsg = "ten minutes";
prop.put("num-results_blockReason", 2);
} else if (accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())) {
block = true;
timePeriodMsg = "one minute";
prop.put("num-results_blockReason", 3);
} else if (accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(),
SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) {
block = true;
timePeriodMsg = "three seconds";
prop.put("num-results_blockReason", 4);
}
if(block) {
ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
+ client
+ ": "
@ -442,21 +443,21 @@ public class yacysearch {
+ accInTenMinutes
+ "/600s, "
+ " requests, disallowed search");
if (!"html".equals(EXT)) {
/*
* API request : return the relevant HTTP status (429 - Too Many Requests - see
* https://tools.ietf.org/html/rfc6585#section-4)
*/
throw new TemplateProcessingException(
"You have reached the maximum allowed number of accesses to this search service within "
+ timePeriodMsg + ". Please try again later or log in as administrator or as a user with extended search right.",
429);
}
if (!"html".equals(EXT)) {
/*
* API request : return the relevant HTTP status (429 - Too Many Requests - see
* https://tools.ietf.org/html/rfc6585#section-4)
*/
throw new TemplateProcessingException(
"You have reached the maximum allowed number of accesses to this search service within "
+ timePeriodMsg + ". Please try again later or log in as administrator or as a user with extended search right.",
429);
}
}
}
if (block) {
prop.put("num-results", 5);
prop.put("num-results", 5);
} else {
String urlmask = (post == null) ? ".*" : post.get("urlmaskfilter", ".*"); // the expression must be a subset of the java Match syntax described in http://lucene.apache.org/core/4_4_0/core/org/apache/lucene/util/automaton/RegExp.html
String tld = null;
@ -475,7 +476,7 @@ public class yacysearch {
// read collection
modifier.collection = post.get("collection", modifier.collection); // post arguments may overrule parsed collection values
int stp = querystring.indexOf('*');
if (stp >= 0) {
// if the star appears as a single entry, use the catchallstring
@ -554,7 +555,7 @@ public class yacysearch {
if (mt != null) {
metatags.add(mt);
} else {
}
}
}
@ -587,13 +588,13 @@ public class yacysearch {
querystring = querystring.replace("/heuristic", "");
modifier.add("/heuristic");
}
final String tldModifierPrefix = "tld:";
final int tldp = querystring.indexOf(tldModifierPrefix, 0);
if (tldp >= 0) {
int ftb = querystring.indexOf(' ', tldp);
if (ftb == -1) {
ftb = querystring.length();
ftb = querystring.length();
}
tld = querystring.substring(tldp + tldModifierPrefix.length(), ftb);
querystring = querystring.replace(tldModifierPrefix + tld, "");
@ -602,20 +603,20 @@ public class yacysearch {
tld = tld.substring(1);
}
if (tld.length() == 0) {
tld = null;
tld = null;
} else {
try {
/* Convert to the same lower case ASCII Compatible Encoding that is used in normalized URLs */
tld = IDN.toASCII(tld, 0);
} catch(final IllegalArgumentException e){
ConcurrentLog.warn("LOCAL_SEARCH", "Failed to convert tld modifier value " + tld + "to ASCII Compatible Encoding (ACE)", e);
}
try {
/* Convert to the same lower case ASCII Compatible Encoding that is used in normalized URLs */
tld = IDN.toASCII(tld, 0);
} catch(final IllegalArgumentException e){
ConcurrentLog.warn("LOCAL_SEARCH", "Failed to convert tld modifier value " + tld + "to ASCII Compatible Encoding (ACE)", e);
}
/* Domain name in an URL is case insensitive : convert now modifier to lower case for further processing over normalized URLs */
tld = tld.toLowerCase(Locale.ROOT);
}
}
if (urlmask == null || urlmask.isEmpty()) urlmask = ".*"; //if no urlmask was given
// read the language from the language-restrict option 'lr'
@ -644,12 +645,12 @@ public class yacysearch {
// filter out stopwords
final SortedSet<String> filtered = SetTools.joinConstructiveByTest(qg.getIncludeWords(), Switchboard.stopwords); //find matching stopwords
qg.removeIncludeWords(filtered);
// if a minus-button was hit, remove a special reference first
if ( post != null && post.containsKey("deleteref") ) {
try {
if ( !sb.verifyAuthentication(header) ) {
prop.authenticationRequired();
prop.authenticationRequired();
return prop;
}
@ -680,7 +681,7 @@ public class yacysearch {
// if a plus-button was hit, create new voting message
if ( post != null && post.containsKey("recommendref") ) {
if ( !sb.verifyAuthentication(header) ) {
prop.authenticationRequired();
prop.authenticationRequired();
return prop;
}
final String recommendHash = post.get("recommendref", ""); // urlhash
@ -783,10 +784,10 @@ public class yacysearch {
sb.getConfigSet("search.navigation"));
theQuery.setStrictContentDom(strictContentDom);
theQuery.setMaxSuggestions(meanMax);
theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(
theQuery.id(true),
@ -837,9 +838,9 @@ public class yacysearch {
sb.getConfigLong(
SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,
sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)));
if(post.getBoolean("resortCachedResults") && cachedEvent == theSearch) {
theSearch.resortCachedResults();
theSearch.resortCachedResults();
}
if ( startRecord == 0 && extendedSearchRights && !stealthmode ) {
@ -888,9 +889,9 @@ public class yacysearch {
try {
suggestion = meanIt.next().toString();
prop.put("didYouMean_suggestions_" + meanCount + "_word", suggestion);
prop.put("didYouMean_suggestions_" + meanCount + "_url",
QueryParams.navUrlWithNewQueryString(RequestHeader.FileType.HTML, 0, theQuery,
suggestion, authenticatedUserName != null));
prop.put("didYouMean_suggestions_" + meanCount + "_url",
QueryParams.navUrlWithNewQueryString(RequestHeader.FileType.HTML, 0, theQuery,
suggestion, authenticatedUserName != null));
prop.put("didYouMean_suggestions_" + meanCount + "_sep", "|");
meanCount++;
} catch (final ConcurrentModificationException e) {
@ -926,7 +927,7 @@ public class yacysearch {
prop.put("geoinfo_loc", i);
prop.put("geoinfo", "1");
}
// update the search tracker
try {
synchronized ( trackerHandles ) {
@ -957,17 +958,17 @@ public class yacysearch {
prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.remote_rwi_stored.get() + theSearch.remote_solr_stored.get(), true));
prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));
prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));
prop.put("jsResort", jsResort);
prop.put("jsResort", jsResort);
prop.put("num-results_jsResort", jsResort);
/* In p2p mode only and if JavaScript resorting is not enabled, add a link allowing user to resort already drained results,
* eventually including fetched results with higher ranks from the Solr and RWI stacks */
prop.put("resortEnabled", !jsResort && global && !stealthmode && theSearch.resortCacheAllowed.availablePermits() > 0 ? 1 : 0);
prop.put("resortEnabled_url",
QueryParams.navurlBase(RequestHeader.FileType.HTML, theQuery, null, true, authenticatedUserName != null)
.append("&startRecord=").append(startRecord).append("&resortCachedResults=true")
.toString());
/* In p2p mode only and if JavaScript resorting is not enabled, add a link allowing user to resort already drained results,
* eventually including fetched results with higher ranks from the Solr and RWI stacks */
prop.put("resortEnabled", !jsResort && global && !stealthmode && theSearch.resortCacheAllowed.availablePermits() > 0 ? 1 : 0);
prop.put("resortEnabled_url",
QueryParams.navurlBase(RequestHeader.FileType.HTML, theQuery, null, true, authenticatedUserName != null)
.append("&startRecord=").append(startRecord).append("&resortCachedResults=true")
.toString());
// generate the search result lines; the content will be produced by another servlet
for ( int i = 0; i < theQuery.itemsPerPage(); i++ ) {
@ -1005,36 +1006,36 @@ public class yacysearch {
prop.put("depth", "0");
prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");
prop.put("jsResort_localQuery", theSearch.query.isLocal() ? "1" : "0");
final boolean showLogin = sb.getConfigBool(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,
SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN_DEFAULT);
SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN_DEFAULT);
if(showLogin) {
if(authenticatedUserName != null) {
/* Show the name of the authenticated user */
prop.put("showLogin", 1);
prop.put("showLogin_userName", authenticatedUserName);
} else {
/* Show a login link */
prop.put("showLogin", 2);
prop.put("showLogin_loginURL",
QueryParams.navurlBase(RequestHeader.FileType.HTML, theQuery, null, true, true).toString());
}
if(authenticatedUserName != null) {
/* Show the name of the authenticated user */
prop.put("showLogin", 1);
prop.put("showLogin_userName", authenticatedUserName);
} else {
/* Show a login link */
prop.put("showLogin", 2);
prop.put("showLogin_loginURL",
QueryParams.navurlBase(RequestHeader.FileType.HTML, theQuery, null, true, true).toString());
}
} else {
prop.put("showLogin", 0);
prop.put("showLogin", 0);
}
}
prop.put("focus", focus ? 1 : 0); // focus search field
prop.put("searchagain", global ? "1" : "0");
String former = originalquerystring.replaceAll(Segment.catchallString, "*");
String former = originalquerystring.replaceAll(Segment.catchallString, "*"); // hide catchallString in output
prop.putHTML("former", former);
try {
prop.put("formerEncoded", URLEncoder.encode(former, StandardCharsets.UTF_8.name()));
} catch (UnsupportedEncodingException e) {
ConcurrentLog.warn("LOCAL_SEARCH", "Unsupported UTF-8 encoding!");
prop.put("formerEncoded", former);
}
prop.put("formerEncoded", URLEncoder.encode(former, StandardCharsets.UTF_8.name()));
} catch (UnsupportedEncodingException e) {
ConcurrentLog.warn("LOCAL_SEARCH", "Unsupported UTF-8 encoding!");
prop.put("formerEncoded", former);
}
prop.put("count", itemsPerPage);
prop.put("offset", startRecord);
prop.put("resource", global ? "global" : "local");

146
htroot/yacysearchtrailer.java
Unescape View File

@ -64,31 +64,31 @@ public class yacysearchtrailer {
@SuppressWarnings({ })
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
if (post == null) {
throw new TemplateMissingParameterException("The eventID parameter is required");
}
if (post == null) {
throw new TemplateMissingParameterException("The eventID parameter is required");
}
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
final String eventID = post.get("eventID", "");
final boolean adminAuthenticated = sb.verifyAuthentication(header);
final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null;
final boolean authenticated = adminAuthenticated || user != null;
if (post.containsKey("auth") && !authenticated) {
/*
* Authenticated search is explicitely requested here
* but no authentication is provided : ask now for authentication.
final boolean authenticated = adminAuthenticated || user != null;
if (post.containsKey("auth") && !authenticated) {
/*
* Authenticated search is explicitely requested here
* but no authentication is provided : ask now for authentication.
* Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information
* and as this page is not private, protected features would simply be hidden without asking browser again for authentication.
* (see mantis 766 : http://mantis.tokeek.de/view.php?id=766) *
*/
prop.authenticationRequired();
return prop;
}
*/
prop.authenticationRequired();
return prop;
}
// find search event
final SearchEvent theSearch = SearchEventCache.getEvent(eventID);
if (theSearch == null) {
@ -96,14 +96,13 @@ public class yacysearchtrailer {
return prop;
}
final RequestHeader.FileType fileType = header.fileType();
final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER);
final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch;
boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);
boolean stealthmode = p2pmode && !global;
// compose search navigation
ContentDomain contentdom = theSearch.getQuery().contentdom;
prop.put("searchdomswitches",
@ -112,35 +111,36 @@ public class yacysearchtrailer {
|| sb.getConfigBool("search.video", true)
|| sb.getConfigBool("search.image", true)
|| sb.getConfigBool("search.app", true) ? 1 : 0);
final String originalquerystring = post.get("query", post.get("search", "")).trim();
String originalquerystring = post.get("query", post.get("search", "")).trim();
originalquerystring = originalquerystring.replace('<', ' ').replace('>', ' '); // light xss protection
final String former = originalquerystring.replaceAll(Segment.catchallString, "*");
final CacheStrategy snippetFetchStrategy = CacheStrategy.parse(post.get("verify", sb.getConfig("search.verify", "")));
final String snippetFetchStrategyName = snippetFetchStrategy == null
? sb.getConfig("search.verify", CacheStrategy.IFFRESH.toName())
: snippetFetchStrategy.toName();
final String snippetFetchStrategyName = snippetFetchStrategy == null
? sb.getConfig("search.verify", CacheStrategy.IFFRESH.toName())
: snippetFetchStrategy.toName();
final int startRecord = post.getInt("startRecord", post.getInt("offset", post.getInt("start", 0)));
/* Maximum number of suggestions to display in the first results page */
/* Maximum number of suggestions to display in the first results page */
final int meanMax = post.getInt("meanCount", 0);
prop.put("resource-switches", adminAuthenticated && (stealthmode || global));
prop.put("resource-switches_global", adminAuthenticated && global);
appendSearchFormValues("resource-switches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);
/* The search event has been found : we can render ranking switches */
prop.put("ranking-switches", true);
appendSearchFormValues("ranking-switches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);
/* Add information about the current navigators generation (number of updates since their initialization) */
prop.put("ranking-switches_nav-generation", theSearch.getNavGeneration());
prop.put("ranking-switches_contextRanking", !former.contains(" /date"));
prop.put("ranking-switches_contextRanking_formerWithoutDate", former.replace(" /date", ""));
prop.put("ranking-switches_dateRanking", former.contains(" /date"));
prop.put("ranking-switches_dateRanking_former", former);
appendSearchFormValues("searchdomswitches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);
prop.put("searchdomswitches_searchtext", sb.getConfigBool("search.text", true) ? 1 : 0);
prop.put("searchdomswitches_searchaudio", sb.getConfigBool("search.audio", true) ? 1 : 0);
prop.put("searchdomswitches_searchvideo", sb.getConfigBool("search.video", true) ? 1 : 0);
@ -151,7 +151,7 @@ public class yacysearchtrailer {
prop.put("searchdomswitches_searchvideo_check", (contentdom == ContentDomain.VIDEO) ? "1" : "0");
prop.put("searchdomswitches_searchimage_check", (contentdom == ContentDomain.IMAGE) ? "1" : "0");
prop.put("searchdomswitches_searchapp_check", (contentdom == ContentDomain.APP) ? "1" : "0");
appendSearchFormValues("searchdomswitches_strictContentDomSwitch_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);
prop.put("searchdomswitches_strictContentDomSwitch", (contentdom != ContentDomain.TEXT && contentdom != ContentDomain.ALL) ? 1 : 0);
prop.put("searchdomswitches_strictContentDomSwitch_strictContentDom", theSearch.getQuery().isStrictContentDom() ? 1 : 0);
@ -189,8 +189,8 @@ public class yacysearchtrailer {
count = entry.getValue();
prop.put(fileType, "nav-topics_element_" + i + "_modifier", name);
prop.put(fileType, "nav-topics_element_" + i + "_name", name);
prop.putUrlEncoded(fileType, "nav-topics_element_" + i + "_url", QueryParams
.navurl(fileType, 0, theSearch.query, name, false, authenticated).toString());
prop.putUrlEncoded(fileType, "nav-topics_element_" + i + "_url", QueryParams
.navurl(fileType, 0, theSearch.query, name, false, authenticated).toString());
prop.put("nav-topics_element_" + i + "_count", count);
int fontsize = TOPWORDS_MINSIZE + (TOPWORDS_MAXSIZE - TOPWORDS_MINSIZE) * (count - mincount) / (1 + maxcount - mincount);
fontsize = Math.max(TOPWORDS_MINSIZE, fontsize - (name.length() - 5));
@ -203,7 +203,7 @@ public class yacysearchtrailer {
i--;
prop.put("nav-topics_element_" + i + "_nl", 0);
}
// protocol navigators
if (theSearch.protocolNavigator == null || theSearch.protocolNavigator.isEmpty()) {
prop.put("nav-protocols", 0);
@ -233,15 +233,15 @@ public class yacysearchtrailer {
prop.put("nav-protocols_element_" + i + "_on", 0);
prop.put("nav-protocols_element_" + i + "_onclick", 0);
prop.put(fileType, "nav-protocols_element_" + i + "_modifier", nav);
url = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString();
url = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString();
} else {
neg++;
prop.put("nav-protocols_element_" + i + "_on", 1);
prop.put("nav-protocols_element_" + i + "_onclick", 1);
prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav);
url = QueryParams
.navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav, authenticated)
.toString();
url = QueryParams
.navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav, authenticated)
.toString();
}
prop.put(fileType, "nav-protocols_element_" + i + "_name", name);
prop.put("nav-protocols_element_" + i + "_onclick_url", url);
@ -278,7 +278,7 @@ public class yacysearchtrailer {
if (name.length() < 10) continue;
count = theSearch.dateNavigator.get(name);
if(count == 0) {
continue;
continue;
}
String shortname = name.substring(0, 10);
long d = Instant.parse(name).toEpochMilli();
@ -301,7 +301,7 @@ public class yacysearchtrailer {
pos++;
prop.put("nav-dates_element_" + i + "_on", 1);
} else {
neg++;
neg++;
prop.put("nav-dates_element_" + i + "_on", 0);
}
prop.put(fileType, "nav-dates_element_" + i + "_name", shortname);
@ -347,7 +347,7 @@ public class yacysearchtrailer {
navUrl = QueryParams.navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav, authenticated);
}
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_name", name);
prop.putUrlEncoded(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", navUrl);
prop.putUrlEncoded(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", navUrl);
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_id", "vocabulary_" + navname + "_" + i);
prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_count", count);
prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_nl", 1);
@ -376,35 +376,35 @@ public class yacysearchtrailer {
prop.put("navs_" + ni + "_displayname", navi.getDisplayName());
prop.put("navs_" + ni + "_name", naviname);
prop.put("navs_" + ni + "_count", navi.size());
final int navSort;
final int navSort;
if(navi.getSort() == null) {
navSort = 0;
navSort = 0;
} else {
if(navi.getSort().getSortType() == NavigatorSortType.COUNT) {
if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) {
navSort = 0;
} else {
navSort = 1;
}
} else {
if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) {
navSort = 2;
} else {
navSort = 3;
}
}
if(navi.getSort().getSortType() == NavigatorSortType.COUNT) {
if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) {
navSort = 0;
} else {
navSort = 1;
}
} else {
if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) {
navSort = 2;
} else {
navSort = 3;
}
}
}
prop.put("navs_" + ni + "_navSort", navSort);
prop.put("navs_" + ni + "_navSort", navSort);
navigatorIterator = navi.navigatorKeys();
navigatorIterator = navi.navigatorKeys();
int i = 0, pos = 0, neg = 0;
String nav, rawNav;
while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = navi.get(name);
if (count == 0) {
/* This entry has a zero count, but the next may be positive */
/* This entry has a zero count, but the next may be positive */
continue;
}
@ -420,24 +420,24 @@ public class yacysearchtrailer {
pos++;
prop.put("navs_" + ni + "_element_" + i + "_on", 1);
prop.put(fileType, "navs_" + ni + "_element_" + i + "_modifier", nav);
navUrl = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString();
navUrl = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString();
} else {
neg++;
prop.put("navs_" + ni + "_element_" + i + "_on", 0);
prop.put(fileType, "navs_" + ni + "_element_" + i + "_modifier", "-" + nav);
navUrl = QueryParams.navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav,
authenticated);
navUrl = QueryParams.navUrlWithSingleModifierRemoved(fileType, 0, theSearch.query, rawNav,
authenticated);
}
prop.put(fileType, "navs_" + ni + "_element_" + i + "_name", navi.getElementDisplayName(name));
prop.putUrlEncoded(fileType, "navs_" + ni + "_element_" + i + "_url", navUrl);
prop.putUrlEncoded(fileType, "navs_" + ni + "_element_" + i + "_url", navUrl);
prop.put(fileType, "navs_" + ni + "_element_" + i + "_id", naviname + "_" + i);
prop.put("navs_" + ni + "_element_" + i + "_count", count);
prop.put("navs_" + ni + "_element_" + i + "_nl", 1);
i++;
}
if(i == 0) {
/* The navigator has only entries with value==0 : this is equivalent to empty navigator */
continue;
/* The navigator has only entries with value==0 : this is equivalent to empty navigator */
continue;
}
prop.put("navs_" + ni + "_element", i);
prop.put("navs_" + ni + "_count", i);
@ -491,10 +491,10 @@ public class yacysearchtrailer {
* @param startRecord the zero based index of the first record to return
* @param meanMax the maximum number of suggestions ("Did you mean") to propose
*/
private static void appendSearchFormValues(final String templatePrefix, final serverObjects post, final serverObjects prop,
boolean global, final SearchEvent theSearch, final String former, final String snippetFetchStrategyName,
final int startRecord, final int meanMax) {
prop.putHTML(templatePrefix + "former", former);
private static void appendSearchFormValues(final String templatePrefix, final serverObjects post, final serverObjects prop,
boolean global, final SearchEvent theSearch, final String former, final String snippetFetchStrategyName,
final int startRecord, final int meanMax) {
prop.putHTML(templatePrefix + "former", former);
prop.put(templatePrefix + "authSearch", post.containsKey("auth"));
prop.put(templatePrefix + "contentdom", post.get("contentdom", "text"));
prop.put(templatePrefix + "strictContentDom", String.valueOf(theSearch.getQuery().isStrictContentDom()));
@ -507,7 +507,7 @@ public class yacysearchtrailer {
prop.put(templatePrefix + "depth", "0");
prop.put(templatePrefix + "constraint", (theSearch.getQuery().constraint == null) ? "" : theSearch.getQuery().constraint.exportB64());
prop.put(templatePrefix + "meanCount", meanMax);
}
}
}
//http://localhost:8090/yacysearch.html?query=java+&amp;maximumRecords=10&amp;resource=local&amp;verify=cacheonly&amp;nav=hosts,authors,namespace,topics,filetype,protocol&amp;urlmaskfilter=ftp://.*&amp;prefermaskfilter=&amp;constraint=&amp;contentdom=text&amp;former=java+%2Fftp&amp;startRecord=0

Write Preview
Loading…
Cancel
Save