From f8a3ab2d82c924151929fdfd1fb7fc860a2831ac Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 2 Oct 2012 14:29:45 +0200 Subject: [PATCH] added the usage of synonyms to the GSA search interface --- htroot/gsa/searchresult.java | 12 +++- .../responsewriter/GSAResponseWriter.java | 4 +- .../yacy/search/index/SolrConfiguration.java | 2 +- source/net/yacy/search/query/QueryParams.java | 67 +++++++++++-------- source/net/yacy/server/serverObjects.java | 2 +- 5 files changed, 52 insertions(+), 35 deletions(-) diff --git a/htroot/gsa/searchresult.java b/htroot/gsa/searchresult.java index 7ac439ba3..62313c97b 100644 --- a/htroot/gsa/searchresult.java +++ b/htroot/gsa/searchresult.java @@ -23,6 +23,7 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.ArrayList; +import java.util.Collection; import java.util.Map; import java.util.regex.Pattern; @@ -35,6 +36,7 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import net.yacy.search.query.AccessTracker; +import net.yacy.search.query.QueryParams; import net.yacy.search.query.SnippetProcess; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -105,10 +107,11 @@ public class searchresult { //post.put(, post.remove("client"));//required, example: myfrontend //post.put(, post.remove("output"));//required, example: xml,xml_no_dtd String q = post.get(CommonParams.Q, ""); + post.put("originalQuery", q); post.put(CommonParams.ROWS, post.remove("num")); post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 5000 : 100)); post.put("hl", "true"); - post.put("hl.fl", YaCySchema.h1_txt.name() + ","+ YaCySchema.h1_txt.name() + ","+ YaCySchema.text_t.name()); + post.put("hl.fl", YaCySchema.h1_txt.name() + "," + YaCySchema.h2_txt.name() + "," + YaCySchema.text_t.name()); post.put("hl.alternateField", YaCySchema.description.name()); post.put("hl.simple.pre", ""); post.put("hl.simple.post", ""); @@ -124,6 +127,10 @@ public class searchresult { String access = post.remove("access"); String entqr = post.remove("entqr"); + // get a solr query string + Collection[] cq = QueryParams.cleanQuery(q); + q = QueryParams.solrQueryString(cq[0], cq[1], sb.index.fulltext().getSolrScheme()).toString(); + // add sites operator if (site != null && site.length() > 0) { String[] s0 = site.split(Pattern.quote("|")); @@ -143,7 +150,8 @@ public class searchresult { } post.put(CommonParams.Q, q); } - + post.put(CommonParams.Q, q); + // get the embedded connector EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.fulltext().getLocalSolr(); if (connector == null) return null; diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java index 99b41a6cf..28641d2b7 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java @@ -161,7 +161,7 @@ public class GSAResponseWriter implements QueryResponseWriter { // write header writer.write(XML_START); - String query = request.getParams().get("q"); + String query = request.getParams().get("originalQuery"); String site = (String) context.get("site"); OpensearchResponseWriter.solitaireTag(writer, "TM", Long.toString(System.currentTimeMillis() - start)); OpensearchResponseWriter.solitaireTag(writer, "Q", query); @@ -170,7 +170,7 @@ public class GSAResponseWriter implements QueryResponseWriter { paramTag(writer, "ie", "UTF-8"); paramTag(writer, "oe", "UTF-8"); paramTag(writer, "client", (String) context.get("client")); - paramTag(writer, "q", request.getParams().get("q")); + paramTag(writer, "q", query); paramTag(writer, "site", site); paramTag(writer, "start", Integer.toString(resHead.offset)); paramTag(writer, "num", Integer.toString(resHead.rows)); diff --git a/source/net/yacy/search/index/SolrConfiguration.java b/source/net/yacy/search/index/SolrConfiguration.java index 584b9aa40..7af31ea3a 100644 --- a/source/net/yacy/search/index/SolrConfiguration.java +++ b/source/net/yacy/search/index/SolrConfiguration.java @@ -879,7 +879,7 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable } return a; } - + /** * register an entry as error document * @param digestURI diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 020f316e9..8cb4ed7f2 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -65,6 +65,7 @@ import net.yacy.kelondro.util.Bitfield; import net.yacy.kelondro.util.SetTools; import net.yacy.peers.Seed; import net.yacy.search.index.Segment; +import net.yacy.search.index.SolrConfiguration; import net.yacy.search.ranking.RankingProfile; public final class QueryParams { @@ -469,7 +470,7 @@ public final class QueryParams { final static YaCySchema[] fields = new YaCySchema[]{ YaCySchema.sku,YaCySchema.title,YaCySchema.h1_txt,YaCySchema.h2_txt, - YaCySchema.author,YaCySchema.description,YaCySchema.keywords,YaCySchema.text_t + YaCySchema.author,YaCySchema.description,YaCySchema.keywords,YaCySchema.text_t,YaCySchema.synonyms_sxt }; final static Map boosts = new LinkedHashMap(); @@ -499,34 +500,8 @@ public final class QueryParams { public String solrQueryString() { if (this.solrQueryString != null) return this.solrQueryString; if (this.query_include_words == null || this.query_include_words.size() == 0) return null; - final StringBuilder q = new StringBuilder(80); - - // add text query - int wc = 0; - StringBuilder w = new StringBuilder(80); - for (String s: this.query_include_words) { - if (wc > 0) w.append(" AND "); - w.append(s); - wc++; - } - for (String s: this.query_exclude_words){ - if (wc > 0) w.append(" AND -"); - w.append(s); - wc++; - } - - // combine these queries for all relevant fields - wc = 0; - for (YaCySchema field: fields) { - if (wc > 0) q.append(" OR "); - q.append('(').append(field.name()).append(':').append(w).append(')'); - wc++; - } - q.insert(0, '('); - q.append(')'); - - // add filter to prevent that results come from failed urls - q.append(" AND -").append(YaCySchema.failreason_t.name()).append(":[* TO *]"); + // get text query + final StringBuilder q = solrQueryString(this.query_include_words, this.query_exclude_words, this.indexSegment.fulltext().getSolrScheme()); // add constraints if ( this.sitehash == null ) { @@ -571,6 +546,40 @@ public final class QueryParams { return this.solrQueryString; } + public static StringBuilder solrQueryString(Collection include, Collection exclude, SolrConfiguration configuration) { + final StringBuilder q = new StringBuilder(80); + + // add text query + int wc = 0; + StringBuilder w = new StringBuilder(80); + for (String s: include) { + if (wc > 0) w.append(" AND "); + w.append(s); + wc++; + } + for (String s: exclude){ + if (wc > 0) w.append(" AND -"); + w.append(s); + wc++; + } + + // combine these queries for all relevant fields + wc = 0; + for (YaCySchema field: fields) { + if (configuration != null && !configuration.contains(field.name())) continue; + if (wc > 0) q.append(" OR "); + q.append('(').append(field.name()).append(':').append(w).append(')'); + wc++; + } + q.insert(0, '('); + q.append(')'); + + // add filter to prevent that results come from failed urls + q.append(" AND -").append(YaCySchema.failreason_t.name()).append(":[* TO *]"); + + return q; + } + public String queryStringForUrl() { try { return URLEncoder.encode(this.queryString, "UTF-8"); diff --git a/source/net/yacy/server/serverObjects.java b/source/net/yacy/server/serverObjects.java index e2171b1d5..0f288e4a2 100644 --- a/source/net/yacy/server/serverObjects.java +++ b/source/net/yacy/server/serverObjects.java @@ -472,7 +472,7 @@ public class serverObjects extends HashMap implements Cloneable public SolrParams toSolrParams(YaCySchema[] facets) { // check if all required post fields are there - if (!this.containsKey(CommonParams.DF)) this.put(CommonParams.DF, YaCySchema.text_t.name()); // set default field to all fields + if (!this.containsKey(CommonParams.DF)) this.put(CommonParams.DF, YaCySchema.text_t.name()); // set default field to the text field if (!this.containsKey(CommonParams.START)) this.put(CommonParams.START, "0"); // set default start item if (!this.containsKey(CommonParams.ROWS)) this.put(CommonParams.ROWS, "10"); // set default number of search results