boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Made the dates navigator max elements number user configurable.

Browse Source 
Also used object properties on QueryParams instances, rather than using
mutable class (static) properties.
pull/127/merge
luccioman 7 years ago
parent 0b0980b364
commit ef8aea7f8d
11 changed files with 97 additions and 19 deletions
Show Stats Download Patch File Download Diff File

3
defaults/yacy.init
Unescape View File

@ -885,6 +885,9 @@ search.navigation=location,hosts,authors,namespace,topics,filetype,protocol,lang
# max number of items displayed in search navigators
search.navigation.maxcount=100
# max number of items displayed in the dates navigator
search.navigation.dates.maxcount=640
# search result verification and snippet fetch caching rules
# each search result can be verified byloading the link from the web
# this can be enhanced using a cache. In some cases it may be appropriate

5
htroot/ConfigSearchPage_p.html
Unescape View File

@ -179,13 +179,16 @@
<input type="checkbox" name="search.navigation.date" value="true" #(search.navigation.date)#::checked="checked" #(/search.navigation.date)# /> Date Navigation
<label title="Maximum days number in the histogram. Beware that a large value may trigger high CPU loads both on the server and on the browser with large result sets.">Maximum range (in days)
<input type="number" name="search.navigation.dates.maxcount" value="#[search.navigation.dates.maxcount]#" min="6" max="2147483647"/>
</label>
<link rel="stylesheet" href="env/morris.css">
<script src="js/raphael.min.js"></script>
<script src="js/morris.js"></script>
<script type="text/javascript" src="js/accessibleHistogram.js" charset="UTF-8"></script>
<div id="graph" style="height:200px"></div>
<script>
var solr= $.getJSON("solr/collection1/select?q=*:*&defType=edismax&start=0&rows=0&wt=json&facet=true&facet.field=dates_in_content_dts&facet.sort=index", function(data) {
var solr= $.getJSON("solr/collection1/select?q=*:*&defType=edismax&start=0&rows=0&wt=json&facet=true&facet.field=dates_in_content_dts&facet.sort=index&facet.limit=#[search.navigation.dates.maxcount]#", function(data) {
dates_in_content_dts = data.facet_counts.facet_fields.dates_in_content_dts;
var parsed = [];
for (var i = 0; i < dates_in_content_dts.length; i = i + 2) {

22
htroot/ConfigSearchPage_p.java
Unescape View File

@ -104,11 +104,17 @@ public class ConfigSearchPage_p {
if (nav.endsWith(",")) nav = nav.substring(0, nav.length() - 1);
sb.setConfig("search.navigation", nav);
// maxcount nav entries, default
int navmaxcnt = post.getInt("search.navigation.maxcount", QueryParams.FACETS_STANDARD_MAXCOUNT);
int navmaxcnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);
if (navmaxcnt > 5) {
sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, navmaxcnt);
if (navmaxcnt != QueryParams.FACETS_STANDARD_MAXCOUNT) QueryParams.FACETS_STANDARD_MAXCOUNT = navmaxcnt;
}
// maxcount dates navigator entries
int datesNavMaxCnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT);
if (datesNavMaxCnt > 5) {
sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, datesNavMaxCnt);
}
}
if (post.containsKey("add.nav")) { // button: add navigator plugin to ative list
@ -168,6 +174,12 @@ public class ConfigSearchPage_p {
sb.setConfig("search.result.show.proxy", config.getProperty("search.result.show.proxy","false"));
sb.setConfig("search.result.show.hostbrowser", config.getProperty("search.result.show.hostbrowser","true"));
sb.setConfig("search.result.show.snapshots", config.getProperty("search.result.show.snapshots","true"));
sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
String.valueOf(QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)));
sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
String.valueOf(QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT)));
}
}
@ -238,7 +250,11 @@ public class ConfigSearchPage_p {
}
prop.put("search.navigation.list", i);
prop.put("search.navigation.maxcount", sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT));
prop.put(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, sb.getConfigInt(
SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
prop.put(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, sb.getConfigInt(
SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
prop.put("about.headline", sb.getConfig("about.headline", "About"));
prop.put("about.body", sb.getConfig("about.body", ""));

6
htroot/IndexControlRWIs_p.java
Unescape View File

@ -679,7 +679,11 @@ public class IndexControlRWIs_p {
sb.getRanking(),
"",//userAgent
0.0d, 0.0d, 0.0d,
new String[0]);
new String[0]);
query.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
query.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
final SearchEvent theSearch = SearchEventCache.getEvent(query, sb.peers, sb.tables, null, false, sb.loader, Integer.MAX_VALUE, Long.MAX_VALUE);
if (theSearch.rwiProcess != null && theSearch.rwiProcess.isAlive()) try {theSearch.rwiProcess.join();} catch (final InterruptedException e) {}
if (theSearch.local_rwi_available.get() == 0) {

4
htroot/yacy/search.java
Unescape View File

@ -255,6 +255,10 @@ public final class search {
0.0d,
new String[0]
);
theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
Network.log.info("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.getQueryGoal().getIncludeHashes()) + " - " + theQuery.itemsPerPage() + " links");
final long timer = System.currentTimeMillis();

4
htroot/yacysearch.java
Unescape View File

@ -653,6 +653,10 @@ public class yacysearch {
header.get(HeaderFramework.USER_AGENT, ""),
lat, lon, rad,
sb.getConfigArray("search.navigation", ""));
theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(
theQuery.id(true),

10
htroot/yacysearchtrailer.java
Unescape View File

@ -110,7 +110,7 @@ public class yacysearchtrailer {
navigatorIterator = languageNavigator.keys(false);
int i = 0, pos = 0, neg = 0;
String nav, rawNav;
while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {
while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = languageNavigator.get(name);
if (count == 0) break;
@ -202,7 +202,7 @@ public class yacysearchtrailer {
if (oldProtocolModifier != null && oldProtocolModifier.length() > 0) {theSearch.query.modifier.remove("/" + oldProtocolModifier); theSearch.query.modifier.remove(oldProtocolModifier);}
theSearch.query.modifier.protocol = "";
theSearch.query.getQueryGoal().query_original = oldQuery.replaceAll(" /https", "").replaceAll(" /http", "").replaceAll(" /ftp", "").replaceAll(" /smb", "").replaceAll(" /file", "");
while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {
while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next().trim();
count = theSearch.protocolNavigator.get(name);
if (count == 0) break;
@ -253,7 +253,7 @@ public class yacysearchtrailer {
if (fromconstraint == null) fromconstraint = new Date(System.currentTimeMillis() - AbstractFormatter.normalyearMillis);
Date toconstraint = theSearch.getQuery().modifier.to == null ? null : DateDetection.parseLine(theSearch.getQuery().modifier.to, theSearch.getQuery().timezoneOffset);
if (toconstraint == null) toconstraint = new Date(System.currentTimeMillis() + AbstractFormatter.normalyearMillis);
while (i < QueryParams.FACETS_DATE_MAXCOUNT && navigatorIterator.hasNext()) {
while (i < theSearch.getQuery().getDateFacetMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next().trim();
if (name.length() < 10) continue;
count = theSearch.dateNavigator.get(name);
@ -266,7 +266,7 @@ public class yacysearchtrailer {
if (fromconstraint != null && dd.before(fromconstraint)) continue;
if (toconstraint != null && dd.after(toconstraint)) break;
if (dx > 0) {
while (d - dx > AbstractFormatter.dayMillis) {
while (d - dx > AbstractFormatter.dayMillis && i < theSearch.getQuery().getDateFacetMaxCount()) {
dx += AbstractFormatter.dayMillis;
String sn = new Date(dx).toInstant().toString().substring(0, 10);
prop.put("nav-dates_element_" + i + "_on", 0);
@ -359,7 +359,7 @@ public class yacysearchtrailer {
navigatorIterator = navi.keys(false);
int i = 0, pos = 0, neg = 0;
String nav, rawNav;
while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {
while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = navi.get(name);
if (count == 0) {

4
source/net/yacy/cora/federate/FederateSearchManager.java
Unescape View File

@ -223,6 +223,10 @@ public class FederateSearchManager {
"",//userAgent
0.0d, 0.0d, 0.0d,
new String[0]);
query.setStandardFacetsMaxCount(this.switchboard.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
query.setDateFacetMaxCount(this.switchboard.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
return query(query);
}

4
source/net/yacy/search/Switchboard.java
Unescape View File

@ -217,7 +217,6 @@ import net.yacy.search.index.Fulltext;
import net.yacy.search.index.Segment;
import net.yacy.search.index.Segment.ReferenceReportCache;
import net.yacy.search.query.AccessTracker;
import net.yacy.search.query.QueryParams;
import net.yacy.search.query.SearchEvent;
import net.yacy.search.query.SearchEventCache;
import net.yacy.search.ranking.RankingProfile;
@ -1235,9 +1234,6 @@ public final class Switchboard extends serverSwitch {
this.trail = new LinkedBlockingQueue<String>();
// set configurable ui defaults
QueryParams.FACETS_STANDARD_MAXCOUNT = sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT); // max number of navigator/facet lines
this.log.config("Finished Switchboard Initialization");
}

6
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -553,7 +553,11 @@ public final class SwitchboardConstants {
/** Default setting value controlling whether search results resorting by browser JavaScript is enabled */
public static final boolean SEARCH_JS_RESORT_DEFAULT = false;
public static final String SEARCH_NAVIGATION_MAXCOUNT = "search.navigation.maxcount"; // max lines displayed in standard search navigators/facets
/** Key of the setting controlling the max lines displayed in standard search navigators/facets */
public static final String SEARCH_NAVIGATION_MAXCOUNT = "search.navigation.maxcount";
/** Key of the setting controlling the max lines displayed in the dates navigator */
public static final String SEARCH_NAVIGATION_DATES_MAXCOUNT = "search.navigation.dates.maxcount";
/** Key of the setting controlling whether a noreferrer link type should be added to search result links */
public static final String SEARCH_RESULT_NOREFERRER = "search.result.noreferrer";

48
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -80,8 +80,11 @@ import org.apache.solr.common.params.FacetParams;
public final class QueryParams {
public static int FACETS_STANDARD_MAXCOUNT = 100; // max count of item lines in navigator
public static int FACETS_DATE_MAXCOUNT = 640;
/** The default max count of item lines in navigator */
public static final int FACETS_STANDARD_MAXCOUNT_DEFAULT = 100;
/** The default maximum number of date elements in the date navigator */
public static final int FACETS_DATE_MAXCOUNT_DEFAULT = 640;
public enum Searchdom {
LOCAL, CLUSTER, GLOBAL;
@ -148,6 +151,13 @@ public final class QueryParams {
private SolrQuery cachedQuery;
private CollectionConfiguration solrSchema;
public final int timezoneOffset;
/** The max count of item lines in navigator */
private int standardFacetsMaxCount;
/** The maximum number of date elements in the date navigator */
private int dateFacetMaxCount;
public QueryParams(
final QueryGoal queryGoal,
@ -269,6 +279,8 @@ public final class QueryParams {
this.facetfields.add(CollectionSchema.VOCABULARY_PREFIX + context + CollectionSchema.VOCABULARY_TERMS_SUFFIX);
}
this.cachedQuery = null;
this.standardFacetsMaxCount = FACETS_STANDARD_MAXCOUNT_DEFAULT;
this.dateFacetMaxCount = FACETS_DATE_MAXCOUNT_DEFAULT;
}
private double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid
@ -294,6 +306,34 @@ public final class QueryParams {
public boolean isLocal() {
return this.domType == Searchdom.LOCAL;
}
/**
* @return the max count of item lines in standard navigators
*/
public int getStandardFacetsMaxCount() {
return this.standardFacetsMaxCount;
}
/**
* @param standardFacetsMaxCount the max count of item lines in standard navigators
*/
public void setStandardFacetsMaxCount(final int standardFacetsMaxCount) {
this.standardFacetsMaxCount = standardFacetsMaxCount;
}
/**
* @return the maximum number of date elements in the date navigator
*/
public int getDateFacetMaxCount() {
return this.dateFacetMaxCount;
}
/**
* @param dateFacetMaxCount the maximum number of date elements in the date navigator
*/
public void setDateFacetMaxCount(final int dateFacetMaxCount) {
this.dateFacetMaxCount = dateFacetMaxCount;
}
public static HandleSet hashes2Set(final String query) {
final HandleSet keyhashes = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
@ -483,7 +523,7 @@ public final class QueryParams {
if (getFacets && this.facetfields.size() > 0) {
params.setFacet(true);
params.setFacetMinCount(1);
params.setFacetLimit(FACETS_STANDARD_MAXCOUNT);
params.setFacetLimit(this.standardFacetsMaxCount);
params.setFacetSort(FacetParams.FACET_SORT_COUNT);
params.setParam(FacetParams.FACET_METHOD, FacetParams.FACET_METHOD_enum); // fight the fieldcache
for (String field: this.facetfields) params.addFacetField("{!ex=" + field + "}" + field); // params.addFacetField("{!ex=" + field + "}" + field);
@ -495,7 +535,7 @@ public final class QueryParams {
params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.end", end);
params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.gap", "+1DAY");
params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.sort", "index");
params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.limit", Integer.toString(FACETS_DATE_MAXCOUNT)); // the year constraint should cause that limitation already
params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.limit", Integer.toString(this.dateFacetMaxCount)); // the year constraint should cause that limitation already
}
//for (String k: params.getParameterNames()) {ArrayList<String> al = new ArrayList<>(); for (String s: params.getParams(k)) al.add(s); System.out.println("Parameter: " + k + "=" + al.toString());}
//http://localhost:8090/solr/collection1/select?q=*:*&rows=0&facet=true&facet.field=dates_in_content_dts&f.dates_in_content_dts.facet.limit=730&f.dates_in_content_dts.facet.sort=index

Write Preview
Loading…
Cancel
Save