diff --git a/defaults/yacy.init b/defaults/yacy.init index eacbb7e3d..86412fdd0 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -1036,7 +1036,7 @@ search.ranking.rwi.profile = search.ranking.solr.collection.boostname.tmpa.0=Default Profile search.ranking.solr.collection.boostfields.tmpa.0=url_paths_sxt^3.0,synonyms_sxt^0.5,title^5.0,text_t^1.0,host_s^6.0,h1_txt^5.0,url_file_name_tokens_t^4.0,h2_txt^3.0,keywords^2.0,description_txt^1.5,author^1.0 search.ranking.solr.collection.filterquery.tmpa.0= -search.ranking.solr.collection.boostquery.tmpa.0=crawldepth_i:0^0.8 crawldepth_i:1^0.4 +search.ranking.solr.collection.boostquery.tmpa.0=crawldepth_i:0^0.8\ncrawldepth_i:1^0.4 search.ranking.solr.collection.boostfunction.tmpb.0= search.ranking.solr.collection.boostname.tmpa.1=Date Profile: sort by date in descending order for a '/date' usage search.ranking.solr.collection.boostfields.tmpa.1=url_paths_sxt^0.1,title^0.1,text_t^0.1 @@ -1051,7 +1051,7 @@ search.ranking.solr.collection.boostfunction.tmpb.2= search.ranking.solr.collection.boostname.tmpa.3=_unused3 search.ranking.solr.collection.boostfields.tmpa.3=text_t^1.0 search.ranking.solr.collection.filterquery.tmpa.3= -search.ranking.solr.collection.boostquery.tmpa.3=crawldepth_i:0^0.8 crawldepth_i:1^0.4 +search.ranking.solr.collection.boostquery.tmpa.3=crawldepth_i:0^0.8\ncrawldepth_i:1^0.4 search.ranking.solr.collection.boostfunction.tmpb.3= # the following values are used to identify duplicate content diff --git a/htroot/RankingSolr_p.html b/htroot/RankingSolr_p.html index 169a35d54..0f34e98b9 100644 --- a/htroot/RankingSolr_p.html +++ b/htroot/RankingSolr_p.html @@ -50,7 +50,7 @@
- +
diff --git a/htroot/RankingSolr_p.java b/htroot/RankingSolr_p.java index 069e9d3b8..9905ff2e6 100644 --- a/htroot/RankingSolr_p.java +++ b/htroot/RankingSolr_p.java @@ -97,7 +97,7 @@ public class RankingSolr_p { } } if (post != null && post.containsKey("ResetBQ")) { - String bq = "crawldepth_i:0^0.8 crawldepth_i:1^0.4"; + String bq = "crawldepth_i:0^0.8\ncrawldepth_i:1^0.4"; if (bq != null) { sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTQUERY_ + profileNr, bq); sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostQuery(bq); diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index b0fdc7abf..e93eb6891 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -59,6 +59,7 @@ import net.yacy.search.query.SearchEvent; import net.yacy.search.schema.CollectionSchema; import net.yacy.search.schema.WebgraphSchema; +import org.apache.commons.lang.StringUtils; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; @@ -168,7 +169,7 @@ public class SolrSelectServlet extends HttpServlet { String bq = ranking.getBoostQuery(); String bf = ranking.getBoostFunction(); if (fq.length() > 0) mmsp.getMap().put(CommonParams.FQ, new String[]{fq}); - if (bq.length() > 0) mmsp.getMap().put(DisMaxParams.BQ, new String[]{bq}); + if (bq.length() > 0) mmsp.getMap().put(DisMaxParams.BQ, StringUtils.split(bq,"\t\n\r\f")); // bq split into multiple query params, allowing space in single query if (bf.length() > 0) mmsp.getMap().put("boost", new String[]{bf}); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 } diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index 699629e25..6aabae242 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -54,13 +54,14 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.storage.Files; import net.yacy.cora.util.ConcurrentLog; +import org.apache.commons.lang.StringUtils; + import org.mozilla.intl.chardet.nsDetector; import org.mozilla.intl.chardet.nsPSMDetector; @@ -419,11 +420,10 @@ public final class FileUtils { return mb; } - private final static Pattern backslashbackslash = Pattern.compile("\\\\"); - private final static Pattern unescaped_equal = Pattern.compile("="); - private final static Pattern escaped_equal = Pattern.compile("\\=", Pattern.LITERAL); - private final static Pattern escaped_newline = Pattern.compile("\\n", Pattern.LITERAL); - private final static Pattern escaped_backslash = Pattern.compile(Pattern.quote("\\"), Pattern.LITERAL); + private final static String[] unescaped_strings_in = {"\r\n", "\r", "\n", "=", "\\"}; + private final static String[] escaped_strings_out = {"\\n", "\\n", "\\n", "\\=", "\\\\"}; + private final static String[] escaped_strings_in = {"\\\\", "\\n", "\\="}; + private final static String[] unescaped_strings_out = {"\\", "\n", "="}; public static void saveMap(final File file, final Map props, final String comment) { boolean err = false; @@ -436,16 +436,13 @@ public final class FileUtils { for ( final Map.Entry entry : props.entrySet() ) { key = entry.getKey(); if ( key != null ) { - key = backslashbackslash.matcher(key).replaceAll("\\\\"); - key = escaped_newline.matcher(key).replaceAll("\\n"); - key = unescaped_equal.matcher(key).replaceAll("\\="); + key = StringUtils.replaceEach(key, unescaped_strings_in, escaped_strings_out); } if ( entry.getValue() == null ) { value = ""; } else { value = entry.getValue(); - value = backslashbackslash.matcher(value).replaceAll("\\\\"); - value = escaped_newline.matcher(value).replaceAll("\\n"); + value = StringUtils.replaceEach(value, unescaped_strings_in, escaped_strings_out); } pw.println(key + "=" + value); } @@ -494,11 +491,8 @@ public final class FileUtils { pos = line.indexOf('=', pos + 1); } while ( pos > 0 && line.charAt(pos - 1) == '\\' ); if ( pos > 0 ) try { - String key = escaped_equal.matcher(line.substring(0, pos).trim()).replaceAll("="); - key = escaped_newline.matcher(key).replaceAll("\n"); - key = escaped_backslash.matcher(key).replaceAll("\\"); - String value = escaped_newline.matcher(line.substring(pos + 1).trim()).replaceAll("\n"); - value = value.replace("\\\\", "\\"); // does not work: escaped_backslashbackslash.matcher(value).replaceAll("\\"); + String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out); + String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out); //System.out.println("key = " + key + ", value = " + value); props.put(key, value); } catch (final IndexOutOfBoundsException e) { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 4f90a326e..43ba97188 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -537,6 +537,7 @@ public final class Switchboard extends serverSwitch { bf.equals("scale(cr_host_norm_i,1,20)")) bf = ""; if (bf.equals("recip(rord(last_modified),1,1000,1000))")) bf = "recip(ms(NOW,last_modified),3.16e-11,1,1)"; // that was an outdated date boost that did not work well if (i == 0 && bq.equals("fuzzy_signature_unique_b:true^100000.0")) bq = "crawldepth_i:0^0.8 crawldepth_i:1^0.4"; + if (bq.equals("crawldepth_i:0^0.8 crawldepth_i:1^0.4")) bq = "crawldepth_i:0^0.8\ncrawldepth_i:1^0.4"; // Fix issue with multiple Boost Queries if (boosts.equals("url_paths_sxt^1000.0,synonyms_sxt^1.0,title^10000.0,text_t^2.0,h1_txt^1000.0,h2_txt^100.0,host_organization_s^100000.0")) boosts = "url_paths_sxt^3.0,synonyms_sxt^0.5,title^5.0,text_t^1.0,host_s^6.0,h1_txt^5.0,url_file_name_tokens_t^4.0,h2_txt^2.0"; r.setName(name); r.updateBoosts(boosts); diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 0e0797145..aac1a2866 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -386,7 +386,7 @@ public final class QueryParams { if (!qf.isEmpty()) params.setParam(DisMaxParams.QF, qf); if (this.queryGoal.getIncludeSize() > 1) { // add boost on combined words - if (bq.length() > 0) bq += " "; + if (bq.length() > 0) bq += "\n"; bq += CollectionSchema.text_t.getSolrFieldName() + ":\"" + this.queryGoal.getIncludeString() + "\"^10"; } if (fq.length() > 0) { @@ -396,7 +396,7 @@ public final class QueryParams { newfq.add(fq); params.setFilterQueries(newfq.toArray(new String[newfq.size()])); } - if (bq.length() > 0) params.setParam(DisMaxParams.BQ, bq); + if (bq.length() > 0) params.setParam(DisMaxParams.BQ, bq.split("[\\r\\n]+")); // split on any sequence consisting of CR and/or LF if (bf.length() > 0) params.setParam("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 // prepare result