boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

solr servlet preparation for join with jetty branch

Browse Source
pull/1/head
Michael Peter Christen 11 years ago
parent 8b97489ff2
commit 25250405f1
4 changed files with 73 additions and 42 deletions
Show Stats Download Patch File Download Diff File

83
htroot/solr/select.java
Unescape View File

@ -51,7 +51,7 @@ import net.yacy.server.serverSwitch;
import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.MultiMapSolrParams;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
@ -152,43 +152,53 @@ public class select {
} }
sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time
MultiMapSolrParams mmsp = post.toSolrParams(/*responseWriter instanceof JsonResponseWriter ? new YaCySchema[]{YaCySchema.host_s, YaCySchema.url_file_ext_s, YaCySchema.url_protocol_s} :*/ null);
// count remote searches if this was part of a p2p search // count remote searches if this was part of a p2p search
if (post.containsKey("partitions")) { if (mmsp.getMap().containsKey("partitions")) {
final int partitions = post.getInt("partitions", 30); final int partitions = mmsp.getInt("partitions", 30);
sb.searchQueriesGlobal += 1.0f / partitions; // increase query counter sb.searchQueriesGlobal += 1.0f / partitions; // increase query counter
} }
// get the ranking profile id // get the ranking profile id
int profileNr = post.getInt("profileNr", 0); int profileNr = mmsp.getInt("profileNr", 0);
// rename post fields according to result style // rename post fields according to result style
if (!post.containsKey(CommonParams.Q) && post.containsKey("query")) { if (!mmsp.getMap().containsKey(CommonParams.Q) && mmsp.getMap().containsKey("query")) {
String querystring = post.get("query", ""); String querystring = mmsp.get("query", "");
post.remove("query"); mmsp.getMap().remove("query");
QueryModifier modifier = new QueryModifier(); QueryModifier modifier = new QueryModifier();
querystring = modifier.parse(querystring); querystring = modifier.parse(querystring);
modifier.apply(post); modifier.apply(mmsp);
QueryGoal qg = new QueryGoal(querystring, querystring); QueryGoal qg = new QueryGoal(querystring, querystring);
StringBuilder solrQ = qg.collectionTextQueryString(sb.index.fulltext().getDefaultConfiguration(), profileNr, false); StringBuilder solrQ = qg.collectionTextQueryString(sb.index.fulltext().getDefaultConfiguration(), profileNr, false);
post.put(CommonParams.Q, solrQ.toString()); // sru patch mmsp.getMap().put(CommonParams.Q, new String[]{solrQ.toString()}); // sru patch
}
String q = mmsp.get(CommonParams.Q, "");
if (!mmsp.getMap().containsKey(CommonParams.START)) {
int startRecord = mmsp.getFieldInt("startRecord", "0");
mmsp.getMap().remove("startRecord");
mmsp.getMap().put(CommonParams.START, new String[]{Integer.toString(startRecord)}); // sru patch
}
if (!mmsp.getMap().containsKey(CommonParams.ROWS)) {
int maximumRecords = mmsp.getFieldInt("maximumRecords", "10");
mmsp.getMap().remove("maximumRecords");
mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(maximumRecords)}); // sru patch
} }
String q = post.get(CommonParams.Q, ""); mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(Math.min(mmsp.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100))});
if (!post.containsKey(CommonParams.START)) post.put(CommonParams.START, post.remove("startRecord", 0)); // sru patch
if (!post.containsKey(CommonParams.ROWS)) post.put(CommonParams.ROWS, post.remove("maximumRecords", 10)); // sru patch
post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100));
// set ranking according to profile number if ranking attributes are not given in the request // set ranking according to profile number if ranking attributes are not given in the request
if (!post.containsKey("sort") && !post.containsKey("bq") && !post.containsKey("bf") && !post.containsKey("boost")) { if (!mmsp.getMap().containsKey("sort") && !mmsp.getMap().containsKey("bq") && !mmsp.getMap().containsKey("bf") && !mmsp.getMap().containsKey("boost")) {
if (!post.containsKey("defType")) post.put("defType", "edismax"); if (!mmsp.getMap().containsKey("defType")) mmsp.getMap().put("defType", new String[]{"edismax"});
Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr); Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr);
String bq = ranking.getBoostQuery(); String bq = ranking.getBoostQuery();
String bf = ranking.getBoostFunction(); String bf = ranking.getBoostFunction();
if (bq.length() > 0) post.put("bq", bq); if (bq.length() > 0) mmsp.getMap().put("bq", new String[]{bq});
if (bf.length() > 0) post.put("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 if (bf.length() > 0) mmsp.getMap().put("boost", new String[]{bf}); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29
} }
// get a response writer for the result // get a response writer for the result
String wt = post.get(CommonParams.WT, "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml String wt = mmsp.get(CommonParams.WT, "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml
QueryResponseWriter responseWriter = RESPONSE_WRITER.get(wt); QueryResponseWriter responseWriter = RESPONSE_WRITER.get(wt);
if (responseWriter == null) return null; if (responseWriter == null) return null;
if (responseWriter instanceof OpensearchResponseWriter) { if (responseWriter instanceof OpensearchResponseWriter) {
@ -201,40 +211,39 @@ public class select {
} }
// if this is a call to YaCys special search formats, enhance the query with field assignments // if this is a call to YaCys special search formats, enhance the query with field assignments
if ((responseWriter instanceof YJsonResponseWriter || responseWriter instanceof OpensearchResponseWriter) && "true".equals(post.get("hl", "true"))) { if ((responseWriter instanceof YJsonResponseWriter || responseWriter instanceof OpensearchResponseWriter) && "true".equals(mmsp.get("hl", "true"))) {
// add options for snippet generation // add options for snippet generation
if (!post.containsKey("hl.q")) post.put("hl.q", q); if (!mmsp.getMap().containsKey("hl.q")) mmsp.getMap().put("hl.q", new String[]{q});
if (!post.containsKey("hl.fl")) post.put("hl.fl", CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName()); if (!mmsp.getMap().containsKey("hl.fl")) mmsp.getMap().put("hl.fl", new String[]{CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName()});
if (!post.containsKey("hl.alternateField")) post.put("hl.alternateField", CollectionSchema.description_txt.getSolrFieldName()); if (!mmsp.getMap().containsKey("hl.alternateField")) mmsp.getMap().put("hl.alternateField", new String[]{CollectionSchema.description_txt.getSolrFieldName()});
if (!post.containsKey("hl.simple.pre")) post.put("hl.simple.pre", "<b>"); if (!mmsp.getMap().containsKey("hl.simple.pre")) mmsp.getMap().put("hl.simple.pre", new String[]{"<b>"});
if (!post.containsKey("hl.simple.post")) post.put("hl.simple.post", "</b>"); if (!mmsp.getMap().containsKey("hl.simple.post")) mmsp.getMap().put("hl.simple.post", new String[]{"</b>"});
if (!post.containsKey("hl.fragsize")) post.put("hl.fragsize", Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)); if (!mmsp.getMap().containsKey("hl.fragsize")) mmsp.getMap().put("hl.fragsize", new String[]{Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)});
} }
// get the embedded connector // get the embedded connector
boolean defaultConnector = post == null || post.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); boolean defaultConnector = mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME);
EmbeddedSolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME); EmbeddedSolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME);
if (connector == null) return null; if (connector == null) return null;
// do the solr request, generate facets if we use a special YaCy format // do the solr request, generate facets if we use a special YaCy format
SolrParams params = post.toSolrParams(/*responseWriter instanceof JsonResponseWriter ? new YaCySchema[]{YaCySchema.host_s, YaCySchema.url_file_ext_s, YaCySchema.url_protocol_s} :*/ null); SolrQueryRequest req = connector.request(mmsp);
SolrQueryRequest req = connector.request(params);
Writer ow = null; Writer ow = null;
SolrQueryResponse response = null; SolrQueryResponse rsp = null;
try { try {
response = connector.query(req); rsp = connector.query(req);
if (response != null) { if (rsp != null) {
Exception e = response.getException(); Exception e = rsp.getException();
if (e != null) { if (e != null) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} else { } else {
// write the result directly to the output stream // write the result directly to the output stream
if (responseWriter instanceof BinaryResponseWriter) { if (responseWriter instanceof BinaryResponseWriter) {
((BinaryResponseWriter) responseWriter).write(out, req, response); ((BinaryResponseWriter) responseWriter).write(out, req, rsp);
} else { } else {
ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset)); ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset));
responseWriter.write(ow, req, response); responseWriter.write(ow, req, rsp);
ow.flush(); ow.flush();
} }
} }
@ -247,10 +256,10 @@ public class select {
SolrRequestInfo.clearRequestInfo(); SolrRequestInfo.clearRequestInfo();
if (ow != null) try {ow.close();} catch (final IOException e1) {} if (ow != null) try {ow.close();} catch (final IOException e1) {}
} }
if (response == null) return null; if (rsp == null) return null;
// log result // log result
Object rv = response.getValues().get("response"); Object rv = rsp.getValues().get("response");
int matches = 0; int matches = 0;
if (rv != null && rv instanceof ResultContext) { if (rv != null && rv instanceof ResultContext) {
matches = ((ResultContext) rv).docs.matches(); matches = ((ResultContext) rv).docs.matches();
@ -258,7 +267,7 @@ public class select {
matches = (int) ((SolrDocumentList) rv).getNumFound(); matches = (int) ((SolrDocumentList) rv).getNumFound();
} }
AccessTracker.addToDump(q, Integer.toString(matches)); AccessTracker.addToDump(q, Integer.toString(matches));
ConcurrentLog.info("SOLR Query", "results: " + matches + ", for query:" + post.toString()); ConcurrentLog.info("SOLR Query", "results: " + matches + ", for query:" + mmsp.toString());
return null; return null;
} }

5
source/net/yacy/cora/federate/solr/SolrServlet.java
Unescape View File

@ -59,7 +59,10 @@ import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
import org.apache.solr.servlet.cache.Method; import org.apache.solr.servlet.cache.Method;
import org.apache.solr.util.FastWriter; import org.apache.solr.util.FastWriter;
/*
* taken from the Solr 3.6.0 code, which is now deprecated;
* this is now done in Solr 4.x.x with org.apache.solr.servlet.SolrDispatchFilter
*/
public class SolrServlet implements Filter { public class SolrServlet implements Filter {
private static final QueryResponseWriter responseWriter = new XMLResponseWriter(); private static final QueryResponseWriter responseWriter = new XMLResponseWriter();

24
source/net/yacy/search/query/QueryModifier.java
Unescape View File

@ -23,6 +23,7 @@ package net.yacy.search.query;
import java.util.ArrayList; import java.util.ArrayList;
import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MultiMapSolrParams;
import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.CommonPattern;
@ -148,10 +149,11 @@ public class QueryModifier {
public String toString() { public String toString() {
return this.modifier.toString(); return this.modifier.toString();
} }
public void apply(serverObjects post) { private StringBuilder apply(String FQ) {
final StringBuilder fq = new StringBuilder(post.get(CommonParams.FQ,"")); final StringBuilder fq = new StringBuilder(FQ);
if (this.sitehost != null && this.sitehost.length() > 0 && fq.indexOf(CollectionSchema.host_s.getSolrFieldName()) < 0) { if (this.sitehost != null && this.sitehost.length() > 0 && fq.indexOf(CollectionSchema.host_s.getSolrFieldName()) < 0) {
// consider to search for hosts with 'www'-prefix, if not already part of the host name // consider to search for hosts with 'www'-prefix, if not already part of the host name
@ -179,6 +181,12 @@ public class QueryModifier {
fq.append(" AND ").append(CollectionSchema.url_protocol_s.getSolrFieldName()).append(":\"").append(this.protocol).append('\"'); fq.append(" AND ").append(CollectionSchema.url_protocol_s.getSolrFieldName()).append(":\"").append(this.protocol).append('\"');
} }
return fq;
}
public void apply(serverObjects post) {
final StringBuilder fq = apply(post.get(CommonParams.FQ,""));
if (fq.length() > 0) { if (fq.length() > 0) {
String fqs = fq.toString(); String fqs = fq.toString();
if (fqs.startsWith(" AND ")) fqs = fqs.substring(5); if (fqs.startsWith(" AND ")) fqs = fqs.substring(5);
@ -186,6 +194,18 @@ public class QueryModifier {
post.put(CommonParams.FQ, fqs); post.put(CommonParams.FQ, fqs);
} }
} }
public void apply(MultiMapSolrParams mmsp) {
final StringBuilder fq = apply(mmsp.get(CommonParams.FQ,""));
if (fq.length() > 0) {
String fqs = fq.toString();
if (fqs.startsWith(" AND ")) fqs = fqs.substring(5);
mmsp.getMap().remove(CommonParams.FQ);
mmsp.getMap().put(CommonParams.FQ, new String[]{fqs});
}
}
/** /**
* parse a GSA site description string and create a filter query string * parse a GSA site description string and create a filter query string

3
source/net/yacy/server/serverObjects.java
Unescape View File

@ -70,7 +70,6 @@ import net.yacy.search.schema.CollectionSchema;
import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.params.MultiMapSolrParams;
import org.apache.solr.common.params.SolrParams;
public class serverObjects implements Serializable, Cloneable { public class serverObjects implements Serializable, Cloneable {
@ -544,7 +543,7 @@ public class serverObjects implements Serializable, Cloneable {
return param.toString(); return param.toString();
} }
public SolrParams toSolrParams(CollectionSchema[] facets) { public MultiMapSolrParams toSolrParams(CollectionSchema[] facets) {
// check if all required post fields are there // check if all required post fields are there
if (!this.containsKey(CommonParams.DF)) this.put(CommonParams.DF, CollectionSchema.text_t.getSolrFieldName()); // set default field to the text field if (!this.containsKey(CommonParams.DF)) this.put(CommonParams.DF, CollectionSchema.text_t.getSolrFieldName()); // set default field to the text field
if (!this.containsKey(CommonParams.START)) this.put(CommonParams.START, "0"); // set default start item if (!this.containsKey(CommonParams.START)) this.put(CommonParams.START, "0"); // set default start item

Write Preview
Loading…
Cancel
Save