diff --git a/htroot/solr/select.java b/htroot/solr/select.java index 77c99a592..acdbe16c2 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -51,7 +51,7 @@ import net.yacy.server.serverSwitch; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; @@ -152,43 +152,53 @@ public class select { } sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time + MultiMapSolrParams mmsp = post.toSolrParams(/*responseWriter instanceof JsonResponseWriter ? new YaCySchema[]{YaCySchema.host_s, YaCySchema.url_file_ext_s, YaCySchema.url_protocol_s} :*/ null); + // count remote searches if this was part of a p2p search - if (post.containsKey("partitions")) { - final int partitions = post.getInt("partitions", 30); + if (mmsp.getMap().containsKey("partitions")) { + final int partitions = mmsp.getInt("partitions", 30); sb.searchQueriesGlobal += 1.0f / partitions; // increase query counter } // get the ranking profile id - int profileNr = post.getInt("profileNr", 0); + int profileNr = mmsp.getInt("profileNr", 0); // rename post fields according to result style - if (!post.containsKey(CommonParams.Q) && post.containsKey("query")) { - String querystring = post.get("query", ""); - post.remove("query"); + if (!mmsp.getMap().containsKey(CommonParams.Q) && mmsp.getMap().containsKey("query")) { + String querystring = mmsp.get("query", ""); + mmsp.getMap().remove("query"); QueryModifier modifier = new QueryModifier(); querystring = modifier.parse(querystring); - modifier.apply(post); + modifier.apply(mmsp); QueryGoal qg = new QueryGoal(querystring, querystring); StringBuilder solrQ = qg.collectionTextQueryString(sb.index.fulltext().getDefaultConfiguration(), profileNr, false); - post.put(CommonParams.Q, solrQ.toString()); // sru patch + mmsp.getMap().put(CommonParams.Q, new String[]{solrQ.toString()}); // sru patch + } + String q = mmsp.get(CommonParams.Q, ""); + if (!mmsp.getMap().containsKey(CommonParams.START)) { + int startRecord = mmsp.getFieldInt("startRecord", "0"); + mmsp.getMap().remove("startRecord"); + mmsp.getMap().put(CommonParams.START, new String[]{Integer.toString(startRecord)}); // sru patch + } + if (!mmsp.getMap().containsKey(CommonParams.ROWS)) { + int maximumRecords = mmsp.getFieldInt("maximumRecords", "10"); + mmsp.getMap().remove("maximumRecords"); + mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(maximumRecords)}); // sru patch } - String q = post.get(CommonParams.Q, ""); - if (!post.containsKey(CommonParams.START)) post.put(CommonParams.START, post.remove("startRecord", 0)); // sru patch - if (!post.containsKey(CommonParams.ROWS)) post.put(CommonParams.ROWS, post.remove("maximumRecords", 10)); // sru patch - post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100)); + mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(Math.min(mmsp.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100))}); // set ranking according to profile number if ranking attributes are not given in the request - if (!post.containsKey("sort") && !post.containsKey("bq") && !post.containsKey("bf") && !post.containsKey("boost")) { - if (!post.containsKey("defType")) post.put("defType", "edismax"); + if (!mmsp.getMap().containsKey("sort") && !mmsp.getMap().containsKey("bq") && !mmsp.getMap().containsKey("bf") && !mmsp.getMap().containsKey("boost")) { + if (!mmsp.getMap().containsKey("defType")) mmsp.getMap().put("defType", new String[]{"edismax"}); Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr); String bq = ranking.getBoostQuery(); String bf = ranking.getBoostFunction(); - if (bq.length() > 0) post.put("bq", bq); - if (bf.length() > 0) post.put("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 + if (bq.length() > 0) mmsp.getMap().put("bq", new String[]{bq}); + if (bf.length() > 0) mmsp.getMap().put("boost", new String[]{bf}); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 } // get a response writer for the result - String wt = post.get(CommonParams.WT, "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml + String wt = mmsp.get(CommonParams.WT, "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml QueryResponseWriter responseWriter = RESPONSE_WRITER.get(wt); if (responseWriter == null) return null; if (responseWriter instanceof OpensearchResponseWriter) { @@ -201,40 +211,39 @@ public class select { } // if this is a call to YaCys special search formats, enhance the query with field assignments - if ((responseWriter instanceof YJsonResponseWriter || responseWriter instanceof OpensearchResponseWriter) && "true".equals(post.get("hl", "true"))) { + if ((responseWriter instanceof YJsonResponseWriter || responseWriter instanceof OpensearchResponseWriter) && "true".equals(mmsp.get("hl", "true"))) { // add options for snippet generation - if (!post.containsKey("hl.q")) post.put("hl.q", q); - if (!post.containsKey("hl.fl")) post.put("hl.fl", CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName()); - if (!post.containsKey("hl.alternateField")) post.put("hl.alternateField", CollectionSchema.description_txt.getSolrFieldName()); - if (!post.containsKey("hl.simple.pre")) post.put("hl.simple.pre", ""); - if (!post.containsKey("hl.simple.post")) post.put("hl.simple.post", ""); - if (!post.containsKey("hl.fragsize")) post.put("hl.fragsize", Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)); + if (!mmsp.getMap().containsKey("hl.q")) mmsp.getMap().put("hl.q", new String[]{q}); + if (!mmsp.getMap().containsKey("hl.fl")) mmsp.getMap().put("hl.fl", new String[]{CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName()}); + if (!mmsp.getMap().containsKey("hl.alternateField")) mmsp.getMap().put("hl.alternateField", new String[]{CollectionSchema.description_txt.getSolrFieldName()}); + if (!mmsp.getMap().containsKey("hl.simple.pre")) mmsp.getMap().put("hl.simple.pre", new String[]{""}); + if (!mmsp.getMap().containsKey("hl.simple.post")) mmsp.getMap().put("hl.simple.post", new String[]{""}); + if (!mmsp.getMap().containsKey("hl.fragsize")) mmsp.getMap().put("hl.fragsize", new String[]{Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)}); } // get the embedded connector - boolean defaultConnector = post == null || post.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); + boolean defaultConnector = mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); EmbeddedSolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME); if (connector == null) return null; // do the solr request, generate facets if we use a special YaCy format - SolrParams params = post.toSolrParams(/*responseWriter instanceof JsonResponseWriter ? new YaCySchema[]{YaCySchema.host_s, YaCySchema.url_file_ext_s, YaCySchema.url_protocol_s} :*/ null); - SolrQueryRequest req = connector.request(params); + SolrQueryRequest req = connector.request(mmsp); Writer ow = null; - SolrQueryResponse response = null; + SolrQueryResponse rsp = null; try { - response = connector.query(req); - if (response != null) { - Exception e = response.getException(); + rsp = connector.query(req); + if (rsp != null) { + Exception e = rsp.getException(); if (e != null) { ConcurrentLog.logException(e); } else { // write the result directly to the output stream if (responseWriter instanceof BinaryResponseWriter) { - ((BinaryResponseWriter) responseWriter).write(out, req, response); + ((BinaryResponseWriter) responseWriter).write(out, req, rsp); } else { ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset)); - responseWriter.write(ow, req, response); + responseWriter.write(ow, req, rsp); ow.flush(); } } @@ -247,10 +256,10 @@ public class select { SolrRequestInfo.clearRequestInfo(); if (ow != null) try {ow.close();} catch (final IOException e1) {} } - if (response == null) return null; + if (rsp == null) return null; // log result - Object rv = response.getValues().get("response"); + Object rv = rsp.getValues().get("response"); int matches = 0; if (rv != null && rv instanceof ResultContext) { matches = ((ResultContext) rv).docs.matches(); @@ -258,7 +267,7 @@ public class select { matches = (int) ((SolrDocumentList) rv).getNumFound(); } AccessTracker.addToDump(q, Integer.toString(matches)); - ConcurrentLog.info("SOLR Query", "results: " + matches + ", for query:" + post.toString()); + ConcurrentLog.info("SOLR Query", "results: " + matches + ", for query:" + mmsp.toString()); return null; } diff --git a/source/net/yacy/cora/federate/solr/SolrServlet.java b/source/net/yacy/cora/federate/solr/SolrServlet.java index ebdc6db2a..a5651fc86 100644 --- a/source/net/yacy/cora/federate/solr/SolrServlet.java +++ b/source/net/yacy/cora/federate/solr/SolrServlet.java @@ -59,7 +59,10 @@ import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; import org.apache.solr.servlet.cache.Method; import org.apache.solr.util.FastWriter; - +/* + * taken from the Solr 3.6.0 code, which is now deprecated; + * this is now done in Solr 4.x.x with org.apache.solr.servlet.SolrDispatchFilter + */ public class SolrServlet implements Filter { private static final QueryResponseWriter responseWriter = new XMLResponseWriter(); diff --git a/source/net/yacy/search/query/QueryModifier.java b/source/net/yacy/search/query/QueryModifier.java index 9908f8646..362ece108 100644 --- a/source/net/yacy/search/query/QueryModifier.java +++ b/source/net/yacy/search/query/QueryModifier.java @@ -23,6 +23,7 @@ package net.yacy.search.query; import java.util.ArrayList; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.MultiMapSolrParams; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.CommonPattern; @@ -148,10 +149,11 @@ public class QueryModifier { public String toString() { return this.modifier.toString(); } + - public void apply(serverObjects post) { + private StringBuilder apply(String FQ) { - final StringBuilder fq = new StringBuilder(post.get(CommonParams.FQ,"")); + final StringBuilder fq = new StringBuilder(FQ); if (this.sitehost != null && this.sitehost.length() > 0 && fq.indexOf(CollectionSchema.host_s.getSolrFieldName()) < 0) { // consider to search for hosts with 'www'-prefix, if not already part of the host name @@ -179,6 +181,12 @@ public class QueryModifier { fq.append(" AND ").append(CollectionSchema.url_protocol_s.getSolrFieldName()).append(":\"").append(this.protocol).append('\"'); } + return fq; + } + public void apply(serverObjects post) { + + final StringBuilder fq = apply(post.get(CommonParams.FQ,"")); + if (fq.length() > 0) { String fqs = fq.toString(); if (fqs.startsWith(" AND ")) fqs = fqs.substring(5); @@ -186,6 +194,18 @@ public class QueryModifier { post.put(CommonParams.FQ, fqs); } } + + public void apply(MultiMapSolrParams mmsp) { + + final StringBuilder fq = apply(mmsp.get(CommonParams.FQ,"")); + + if (fq.length() > 0) { + String fqs = fq.toString(); + if (fqs.startsWith(" AND ")) fqs = fqs.substring(5); + mmsp.getMap().remove(CommonParams.FQ); + mmsp.getMap().put(CommonParams.FQ, new String[]{fqs}); + } + } /** * parse a GSA site description string and create a filter query string diff --git a/source/net/yacy/server/serverObjects.java b/source/net/yacy/server/serverObjects.java index 3937303f0..c7836e045 100644 --- a/source/net/yacy/server/serverObjects.java +++ b/source/net/yacy/server/serverObjects.java @@ -70,7 +70,6 @@ import net.yacy.search.schema.CollectionSchema; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MultiMapSolrParams; -import org.apache.solr.common.params.SolrParams; public class serverObjects implements Serializable, Cloneable { @@ -544,7 +543,7 @@ public class serverObjects implements Serializable, Cloneable { return param.toString(); } - public SolrParams toSolrParams(CollectionSchema[] facets) { + public MultiMapSolrParams toSolrParams(CollectionSchema[] facets) { // check if all required post fields are there if (!this.containsKey(CommonParams.DF)) this.put(CommonParams.DF, CollectionSchema.text_t.getSolrFieldName()); // set default field to the text field if (!this.containsKey(CommonParams.START)) this.put(CommonParams.START, "0"); // set default start item