boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added a hack to forward solr search results from an external attached

Browse Source 
solr to the YaCy built-in solr search servlet. Its not complete and not
fully correct (there is still a utf8 encoding problem) but it is a way
to get easily requests forwarded through YaCy to an external Solr.
pull/1/head
orbiter 10 years ago
parent 025516f682
commit a922b122a3
4 changed files with 140 additions and 43 deletions
Show Stats Download Patch File Download Diff File

2
source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
Unescape View File

@ -348,7 +348,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
params.setRows(count); params.setRows(count);
params.setStart(offset); params.setStart(offset);
params.setFacet(false); params.setFacet(false);
if (fields.length > 0) params.setFields(fields); if (fields != null && fields.length > 0) params.setFields(fields);
params.setIncludeScore(false); params.setIncludeScore(false);
params.setParam("defType", "edismax"); params.setParam("defType", "edismax");
params.setParam(DisMaxParams.QF, CollectionSchema.text_t.getSolrFieldName() + "^1.0"); params.setParam(DisMaxParams.QF, CollectionSchema.text_t.getSolrFieldName() + "^1.0");

92
source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java
Unescape View File

@ -22,14 +22,19 @@ package net.yacy.cora.federate.solr.responsewriter;
import java.io.IOException; import java.io.IOException;
import java.io.Writer; import java.io.Writer;
import java.util.Collection;
import java.util.Date; import java.util.Date;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Set; import java.util.Set;
import net.yacy.cora.federate.solr.SolrType; import net.yacy.cora.federate.solr.SolrType;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.XML; import org.apache.solr.common.util.XML;
@ -83,6 +88,12 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
writer.write(XML_STOP); writer.write(XML_STOP);
} }
public static void write(final Writer writer, final SolrQueryRequest request, final SolrDocumentList sdl) throws IOException {
writer.write(XML_START);
writeDocs(writer, request, sdl);
writer.write(XML_STOP);
}
private static void writeProps(final Writer writer, final String name, final NamedList<?> val) throws IOException { private static void writeProps(final Writer writer, final String name, final NamedList<?> val) throws IOException {
if (val == null) return; if (val == null) return;
int sz = val.size(); int sz = val.size();
@ -124,20 +135,43 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
int id = iterator.nextDoc(); int id = iterator.nextDoc();
Document doc = searcher.doc(id, DEFAULT_FIELD_LIST); Document doc = searcher.doc(id, DEFAULT_FIELD_LIST);
writeDoc(writer, schema, null, doc, (includeScore ? iterator.score() : 0.0f), includeScore); writeDoc(writer, schema, null, doc.getFields(), (includeScore ? iterator.score() : 0.0f), includeScore);
}
writer.write("</result>");
writer.write(lb);
}
private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs) throws IOException {
boolean includeScore = false;
final int sz = docs.size();
writer.write("<result");
writeAttr(writer, "name", "response");
writeAttr(writer, "numFound", Long.toString(docs.getNumFound()));
writeAttr(writer, "start", Long.toString(docs.getStart()));
if (includeScore) {
writeAttr(writer, "maxScore", Float.toString(docs.getMaxScore()));
}
if (sz == 0) {
writer.write("/>");
return;
}
writer.write('>'); writer.write(lb);
Iterator<SolrDocument> iterator = docs.iterator();
for (int i = 0; i < sz; i++) {
SolrDocument doc = iterator.next();
writeDoc(writer, doc);
} }
writer.write("</result>"); writer.write("</result>");
writer.write(lb); writer.write(lb);
} }
private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name, final Document doc, final float score, final boolean includeScore) throws IOException { private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name, final List<IndexableField> fields, final float score, final boolean includeScore) throws IOException {
startTagOpen(writer, "doc", name); startTagOpen(writer, "doc", name);
if (includeScore) { if (includeScore) {
writeTag(writer, "float", "score", Float.toString(score), false); writeTag(writer, "float", "score", Float.toString(score), false);
} }
List<IndexableField> fields = doc.getFields();
int sz = fields.size(); int sz = fields.size();
int fidx1 = 0, fidx2 = 0; int fidx1 = 0, fidx2 = 0;
while (fidx1 < sz) { while (fidx1 < sz) {
@ -156,16 +190,16 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
if (sf.multiValued()) { if (sf.multiValued()) {
startTagOpen(writer, "arr", fieldName); startTagOpen(writer, "arr", fieldName);
String sv = value.stringValue(); String sv = value.stringValue();
writeField(writer, type, null, sv); //sf.write(this, null, f1); writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1);
writer.write("</arr>"); writer.write("</arr>");
} else { } else {
writeField(writer, type, value.name(), value.stringValue()); //sf.write(this, f1.name(), f1); writeField(writer, type.getTypeName(), value.name(), value.stringValue()); //sf.write(this, f1.name(), f1);
} }
} else { } else {
startTagOpen(writer, "arr", fieldName); startTagOpen(writer, "arr", fieldName);
for (int i = fidx1; i < fidx2; i++) { for (int i = fidx1; i < fidx2; i++) {
String sv = fields.get(i).stringValue(); String sv = fields.get(i).stringValue();
writeField(writer, type, null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i)); writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i));
} }
writer.write("</arr>"); writer.write("</arr>");
writer.write(lb); writer.write(lb);
@ -176,9 +210,29 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
writer.write(lb); writer.write(lb);
} }
private static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException {
startTagOpen(writer, "doc", null);
final Map<String, Object> fields = doc.getFieldValueMap();
for (String key: fields.keySet()) {
if (key == null) continue;
Object value = doc.get(key);
if (value == null) {
} else if (value instanceof Collection<?>) {
startTagOpen(writer, "arr", key);
for (Object o: ((Collection<?>) value)) {
writeField(writer, null, o);
}
writer.write("</arr>"); writer.write(lb);
} else {
writeField(writer, key, value);
}
}
writer.write("</doc>");
writer.write(lb);
}
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static void writeField(final Writer writer, final FieldType type, final String name, final String value) throws IOException { private static void writeField(final Writer writer, final String typeName, final String name, final String value) throws IOException {
String typeName = type.getTypeName();
if (typeName.equals(SolrType.text_general.printName()) || if (typeName.equals(SolrType.text_general.printName()) ||
typeName.equals(SolrType.string.printName()) || typeName.equals(SolrType.string.printName()) ||
typeName.equals(SolrType.text_en_splitting_tight.printName())) { typeName.equals(SolrType.text_en_splitting_tight.printName())) {
@ -198,6 +252,24 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
} }
} }
private static void writeField(final Writer writer, final String name, final Object value) throws IOException {
if (value instanceof String) {
writeTag(writer, "str", name, (String) value, true);
} else if (value instanceof Boolean) {
writeTag(writer, "bool", name, ((Boolean) value).toString(), true);
} else if (value instanceof Integer) {
writeTag(writer, "int", name, ((Integer) value).toString(), true);
} else if (value instanceof Long) {
writeTag(writer, "long", name, ((Long) value).toString(), true);
} else if (value instanceof Date) {
writeTag(writer, "date", name, ((Date) value).toString(), true); // this is declared deprecated in solr 4.2.1 but is still used as done here
} else if (value instanceof Float) {
writeTag(writer, "float", name, ((Float) value).toString(), true);
} else if (value instanceof Double) {
writeTag(writer, "double", name, ((Double) value).toString(), true);
}
}
private static void writeTag(final Writer writer, final String tag, final String nameAttr, final String val, final boolean escape) throws IOException { private static void writeTag(final Writer writer, final String tag, final String nameAttr, final String val, final boolean escape) throws IOException {
int contentLen = val.length(); int contentLen = val.length();
if (contentLen == 0) { if (contentLen == 0) {
@ -224,7 +296,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
private static void startTagOpen(final Writer writer, final String tag, final String nameAttr) throws IOException { private static void startTagOpen(final Writer writer, final String tag, final String nameAttr) throws IOException {
writer.write('<'); writer.write(tag); writer.write('<'); writer.write(tag);
if (nameAttr != null) writeAttr(writer, "name", nameAttr); if (nameAttr != null) writeAttr(writer, "name", nameAttr);
writer.write('>'); writer.write(lb); writer.write('>'); //writer.write(lb);
} }
private static void startTagClose(final Writer writer, final String tag, final String nameAttr) throws IOException { private static void startTagClose(final Writer writer, final String tag, final String nameAttr) throws IOException {
@ -236,4 +308,6 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
private static void writeAttr(final Writer writer, final String nameAttr, final String val) throws IOException { private static void writeAttr(final Writer writer, final String nameAttr, final String val) throws IOException {
writer.write(' '); writer.write(nameAttr); writer.write("=\""); XML.escapeAttributeValue(val, writer); writer.write('"'); writer.write(' '); writer.write(nameAttr); writer.write("=\""); XML.escapeAttributeValue(val, writer); writer.write('"');
} }
} }

81
source/net/yacy/http/servlets/SolrSelectServlet.java
Unescape View File

@ -43,6 +43,7 @@ import net.yacy.cora.federate.solr.Ranking;
import net.yacy.cora.federate.solr.SchemaDeclaration; import net.yacy.cora.federate.solr.SchemaDeclaration;
import net.yacy.cora.federate.solr.SolrType; import net.yacy.cora.federate.solr.SolrType;
import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter; import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter;
import net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter; import net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter;
import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter; import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter;
@ -58,6 +59,7 @@ import net.yacy.search.query.SearchEvent;
import net.yacy.search.schema.CollectionSchema; import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema; import net.yacy.search.schema.WebgraphSchema;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.DisMaxParams; import org.apache.solr.common.params.DisMaxParams;
@ -195,7 +197,10 @@ public class SolrSelectServlet extends HttpServlet {
String requestURI = hrequest.getRequestURI(); String requestURI = hrequest.getRequestURI();
boolean defaultConnector = (requestURI.startsWith("/solr/" + WebgraphSchema.CORE_NAME)) ? false : requestURI.startsWith("/solr/" + CollectionSchema.CORE_NAME) || mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); boolean defaultConnector = (requestURI.startsWith("/solr/" + WebgraphSchema.CORE_NAME)) ? false : requestURI.startsWith("/solr/" + CollectionSchema.CORE_NAME) || mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME);
mmsp.getMap().remove("core"); mmsp.getMap().remove("core");
EmbeddedSolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME); SolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME);
if (connector == null) {
connector = defaultConnector ? sb.index.fulltext().getDefaultConnector() : sb.index.fulltext().getConnectorForRead(WebgraphSchema.CORE_NAME);
}
if (connector == null) throw new ServletException("no core"); if (connector == null) throw new ServletException("no core");
// add default queryfield parameter according to local ranking config (or defaultfield) // add default queryfield parameter according to local ranking config (or defaultfield)
@ -228,44 +233,56 @@ public class SolrSelectServlet extends HttpServlet {
} }
// do the solr request, generate facets if we use a special YaCy format // do the solr request, generate facets if we use a special YaCy format
req = connector.request(mmsp); final SolrQueryResponse rsp;
if (connector instanceof EmbeddedSolrConnector) {
SolrQueryResponse rsp = connector.query(req); req = ((EmbeddedSolrConnector) connector).request(mmsp);
rsp = ((EmbeddedSolrConnector) connector).query(req);
// prepare response
hresponse.setHeader("Cache-Control", "no-cache"); // prepare response
HttpCacheHeaderUtil.checkHttpCachingVeto(rsp, hresponse, reqMethod); hresponse.setHeader("Cache-Control", "no-cache");
HttpCacheHeaderUtil.checkHttpCachingVeto(rsp, hresponse, reqMethod);
// check error
if (rsp.getException() != null) { // check error
AccessTracker.addToDump(querystring, "0", new Date()); if (rsp.getException() != null) {
sendError(hresponse, rsp.getException()); AccessTracker.addToDump(querystring, "0", new Date());
return; sendError(hresponse, rsp.getException());
} return;
}
NamedList<?> values = rsp.getValues(); NamedList<?> values = rsp.getValues();
DocList r = ((ResultContext) values.get("response")).docs; DocList r = ((ResultContext) values.get("response")).docs;
int numFound = r.matches(); int numFound = r.matches();
AccessTracker.addToDump(querystring, Integer.toString(numFound), new Date()); AccessTracker.addToDump(querystring, Integer.toString(numFound), new Date());
// write response header // write response header
final String contentType = responseWriter.getContentType(req, rsp); final String contentType = responseWriter.getContentType(req, rsp);
if (null != contentType) response.setContentType(contentType); if (null != contentType) response.setContentType(contentType);
if (Method.HEAD == reqMethod) { if (Method.HEAD == reqMethod) {
return; return;
} }
// write response body // write response body
if (responseWriter instanceof BinaryResponseWriter) { if (responseWriter instanceof BinaryResponseWriter) {
((BinaryResponseWriter) responseWriter).write(response.getOutputStream(), req, rsp); ((BinaryResponseWriter) responseWriter).write(response.getOutputStream(), req, rsp);
} else {
out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), UTF8.charset));
responseWriter.write(out, req, rsp);
out.flush();
}
} else { } else {
out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), UTF8.charset)); // write a 'faked' response using a call to the backend
responseWriter.write(out, req, rsp); SolrDocumentList sdl = connector.getDocumentListByQuery(
out.flush(); mmsp.getMap().get(CommonParams.Q)[0],
mmsp.getMap().get(CommonParams.SORT) == null ? null : mmsp.getMap().get(CommonParams.SORT)[0],
Integer.parseInt(mmsp.getMap().get(CommonParams.START)[0]),
Integer.parseInt(mmsp.getMap().get(CommonParams.ROWS)[0]),
mmsp.getMap().get(CommonParams.FL));
OutputStreamWriter osw = new OutputStreamWriter(response.getOutputStream());
EnhancedXMLResponseWriter.write(osw, req, sdl);
osw.close();
} }
} catch (final Throwable ex) { } catch (final Throwable ex) {
sendError(hresponse, ex); sendError(hresponse, ex);
} finally { } finally {

6
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -164,6 +164,12 @@ public final class Fulltext {
return this.solrInstances.getEmbeddedConnector(corename); return this.solrInstances.getEmbeddedConnector(corename);
} }
public SolrConnector getConnectorForRead(String corename) {
if (this.solrInstances.isConnectedRemote()) return this.solrInstances.getRemoteConnector(corename);
if (this.solrInstances.isConnectedEmbedded()) return this.solrInstances.getEmbeddedConnector(corename);
return null;
}
public RemoteSolrConnector getDefaultRemoteSolrConnector() { public RemoteSolrConnector getDefaultRemoteSolrConnector() {
try { try {
return this.solrInstances.getDefaultRemoteConnector(true); return this.solrInstances.getDefaultRemoteConnector(true);

Write Preview
Loading…
Cancel
Save