boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Handle Solr fields restrict and alias in YaCy html and exml writers

Browse Source 
Thus allowing for example to read more easily the local Solr index full
metadata in HTML by restricting if desired to some fields of interest.

See Solr documentation about the 'fl' (Field List) parameter at
https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-Thefl_FieldList_Parameter
pull/167/merge
luccioman 7 years ago
parent 3da2739bbd
commit 0d7625ecfb
3 changed files with 163 additions and 65 deletions
Show Stats Download Patch File Download Diff File

126
source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java
Unescape View File

@ -23,6 +23,7 @@ package net.yacy.cora.federate.solr.responsewriter;
import java.io.IOException;
import java.io.Writer;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
@ -49,7 +50,9 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TextField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
public class EnhancedXMLResponseWriter implements QueryResponseWriter {
@ -83,14 +86,14 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
@SuppressWarnings("unchecked")
SimpleOrderedMap<Object> highlighting = (SimpleOrderedMap<Object>) values.get("highlighting");
writeProps(writer, "responseHeader", responseHeader); // this.writeVal("responseHeader", responseHeader);
writeDocs(writer, request, response); // this.writeVal("response", response);
writeDocs(writer, request, response, rsp.getReturnFields()); // this.writeVal("response", response);
writeProps(writer, "highlighting", highlighting);
writer.write(XML_STOP);
}
public static void write(final Writer writer, final SolrQueryRequest request, final SolrDocumentList sdl) throws IOException {
writer.write(XML_START);
writeDocs(writer, request, sdl);
writeDocs(writer, request, sdl, new SolrReturnFields(request));
writer.write(XML_STOP);
}
@ -113,7 +116,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
}
}
private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response) throws IOException {
private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response, final ReturnFields returnFields) throws IOException {
boolean includeScore = false;
final int sz = response.size();
writer.write("<result");
@ -135,13 +138,13 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
for (int i = 0; i < sz; i++) {
int id = iterator.nextDoc();
Document doc = searcher.doc(id);
writeDoc(writer, schema, null, doc.getFields(), (includeScore ? iterator.score() : 0.0f), includeScore);
writeDoc(writer, schema, null, doc.getFields(), (includeScore ? iterator.score() : 0.0f), includeScore, returnFields);
}
writer.write("</result>");
writer.write(lb);
}
private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs) throws IOException {
private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs, final ReturnFields returnFields) throws IOException {
boolean includeScore = false;
final int sz = docs.size();
writer.write("<result");
@ -159,18 +162,25 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
Iterator<SolrDocument> iterator = docs.iterator();
for (int i = 0; i < sz; i++) {
SolrDocument doc = iterator.next();
writeDoc(writer, doc);
writeDoc(writer, doc, returnFields);
}
writer.write("</result>");
writer.write(lb);
}
private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name, final List<IndexableField> fields, final float score, final boolean includeScore) throws IOException {
private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name,
final List<IndexableField> fields, final float score, final boolean includeScore,
final ReturnFields returnFields) throws IOException {
startTagOpen(writer, "doc", name);
if (includeScore) {
writeTag(writer, "float", "score", Float.toString(score), false); // this is the special Solr "score" pseudo-field
}
/* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter
* (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */
final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()
: returnFields.getFieldRenames();
int sz = fields.size();
int fidx1 = 0, fidx2 = 0;
@ -181,30 +191,35 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) {
fidx2++;
}
SchemaField sf = schema == null ? null : schema.getFieldOrNull(fieldName);
if (sf == null) {
sf = new SchemaField(fieldName, new TextField());
}
FieldType type = sf.getType();
if (fidx1 + 1 == fidx2) {
if (sf.multiValued()) {
startTagOpen(writer, "arr", fieldName);
writer.write(lb);
String sv = value.stringValue();
writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1);
writer.write("</arr>");
} else {
writeField(writer, type.getTypeName(), value.name(), value.stringValue()); //sf.write(this, f1.name(), f1);
}
} else {
startTagOpen(writer, "arr", fieldName);
writer.write(lb);
for (int i = fidx1; i < fidx2; i++) {
String sv = fields.get(i).stringValue();
writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i));
}
writer.write("</arr>");
writer.write(lb);
if(returnFields == null || returnFields.wantsField(fieldName)) {
SchemaField sf = schema == null ? null : schema.getFieldOrNull(fieldName);
if (sf == null) {
sf = new SchemaField(fieldName, new TextField());
}
final String renderedFieldName = fieldRenamings.getOrDefault(fieldName, fieldName);
FieldType type = sf.getType();
if (fidx1 + 1 == fidx2) {
if (sf.multiValued()) {
startTagOpen(writer, "arr", renderedFieldName);
writer.write(lb);
String sv = value.stringValue();
writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1);
writer.write("</arr>");
} else {
writeField(writer, type.getTypeName(), renderedFieldName, value.stringValue()); //sf.write(this, f1.name(), f1);
}
} else {
startTagOpen(writer, "arr", renderedFieldName);
writer.write(lb);
for (int i = fidx1; i < fidx2; i++) {
String sv = fields.get(i).stringValue();
writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i));
}
writer.write("</arr>");
writer.write(lb);
}
}
fidx1 = fidx2;
}
@ -233,27 +248,68 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
writer.write(lb);
}
public static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException {
/**
* Append XML representation of the given Solr document to the writer
*
* @param writer
* an open writer. Must not be null.
* @param doc
* the solr document to write. Must not be null.
* @param returnFields
* the eventual fields return configuration, allowing for example to
* restrict the actually returned fields. May be null.
* @throws IOException
* when a write error occurred
*/
private static final void writeDoc(final Writer writer, final SolrDocument doc, final ReturnFields returnFields) throws IOException {
startTagOpen(writer, "doc", null);
/* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter
* (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */
final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()
: returnFields.getFieldRenames();
final Map<String, Object> fields = doc.getFieldValueMap();
for (String key: fields.keySet()) {
if (key == null) continue;
if (key == null) {
continue;
}
if (returnFields != null && !returnFields.wantsField(key)) {
continue;
}
Object value = doc.get(key);
final String renderedFieldName = fieldRenamings.getOrDefault(key, key);
if (value == null) {
} else if (value instanceof Collection<?>) {
startTagOpen(writer, "arr", key);
startTagOpen(writer, "arr", renderedFieldName);
writer.write(lb);
for (Object o: ((Collection<?>) value)) {
writeField(writer, null, o);
}
writer.write("</arr>"); writer.write(lb);
} else {
writeField(writer, key, value);
writeField(writer, renderedFieldName, value);
}
}
writer.write("</doc>");
writer.write(lb);
}
/**
* Append XML representation of the given Solr document to the writer
*
* @param writer
* an open writer. Must not be null.
* @param doc
* the solr document to write. Must not be null.
* @throws IOException
* when a write error occurred
*/
public static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException {
writeDoc(writer, doc, null);
}
private static void writeField(final Writer writer, final String typeName, final String name, final String value) throws IOException {
if (typeName.equals(SolrType.text_general.printName()) ||

99
source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java
Unescape View File

@ -24,17 +24,13 @@ import java.io.IOException;
import java.io.Writer;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import net.yacy.cora.federate.solr.SolrType;
import net.yacy.cora.lod.vocabulary.DublinCore;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.params.SolrParams;
@ -50,8 +46,14 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TextField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
import net.yacy.cora.federate.solr.SolrType;
import net.yacy.cora.lod.vocabulary.DublinCore;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
public class HTMLResponseWriter implements QueryResponseWriter {
public static final Pattern dqp = Pattern.compile("\"");
@ -165,7 +167,7 @@ public class HTMLResponseWriter implements QueryResponseWriter {
int id = iterator.nextDoc();
Document doc = searcher.doc(id);
LinkedHashMap<String, String> tdoc = translateDoc(schema, doc);
LinkedHashMap<String, String> tdoc = translateDoc(schema, doc, rsp.getReturnFields());
String title;
@ -187,19 +189,19 @@ public class HTMLResponseWriter implements QueryResponseWriter {
writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");
writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see this page as XML.</span></div>\n");
writeDoc(writer, tdoc, title, coreName);
writeDoc(writer, tdoc, title, coreName, rsp.getReturnFields());
while (iterator.hasNext()) {
id = iterator.nextDoc();
doc = searcher.doc(id);
tdoc = translateDoc(schema, doc);
tdoc = translateDoc(schema, doc, rsp.getReturnFields());
if(CollectionSchema.CORE_NAME.equals(coreName)) {
title = tdoc.get(CollectionSchema.title.getSolrFieldName());
if (title == null) title = "";
} else {
title = "";
}
writeDoc(writer, tdoc, title, coreName);
writeDoc(writer, tdoc, title, coreName, rsp.getReturnFields());
}
} else {
writer.write("<title>No Document Found</title>\n</head><body>\n");
@ -209,7 +211,18 @@ public class HTMLResponseWriter implements QueryResponseWriter {
writer.write("</body></html>\n");
}
private static final void writeDoc(final Writer writer, final LinkedHashMap<String, String> tdoc, final String title, final String coreName) throws IOException {
/**
* Append an html representation of the document fields to the writer.
* @param writer an open output writer. Must not be null.
* @param tdoc the documents fields, mapped from field names to values. Must not be null.
* @param title the document title. May be empty.
* @param coreName the Solr core name.
* @param returnFields the eventual fields return configuration, allowing for example to
* rename fields with a pseudo in the result. May be null.
* @throws IOException when a write error occurred.
*/
private static final void writeDoc(final Writer writer, final LinkedHashMap<String, String> tdoc,
final String title, final String coreName, final ReturnFields returnFields) throws IOException {
writer.write("<form name=\"yacydoc" + title + "\" method=\"post\" action=\"#\" enctype=\"multipart/form-data\" accept-charset=\"UTF-8\">\n");
writer.write("<fieldset>\n");
@ -225,9 +238,13 @@ public class HTMLResponseWriter implements QueryResponseWriter {
}
writer.write("<dl>\n");
/* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter
* (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */
final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()
: returnFields.getFieldRenames();
for (Map.Entry<String, String> entry: tdoc.entrySet()) {
writer.write("<dt>");
writer.write(entry.getKey());
writer.write(fieldRenamings.getOrDefault(entry.getKey(), entry.getKey()));
writer.write("</dt><dd>");
if (entry.getKey().equals("sku")) {
writer.write("<a href=\"" + entry.getValue() + "\">" + entry.getValue() + "</a>");
@ -241,7 +258,19 @@ public class HTMLResponseWriter implements QueryResponseWriter {
writer.write("</form>\n");
}
public static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc) {
/**
* Translate an indexed document in a map of field names to field values.
*
* @param schema
* the schema of the indexed document. Must not be null.
* @param doc
* the indexed document. Must not be null.
* @param returnFields
* the eventual fields return configuration, allowing for example to
* restrict the actually returned fields. May be null.
* @return a map of field names to field values
*/
private static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc, final ReturnFields returnFields) {
List<IndexableField> fields = doc.getFields();
int sz = fields.size();
int fidx1 = 0, fidx2 = 0;
@ -253,29 +282,43 @@ public class HTMLResponseWriter implements QueryResponseWriter {
while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) {
fidx2++;
}
SchemaField sf = schema.getFieldOrNull(fieldName);
if (sf == null) sf = new SchemaField(fieldName, new TextField());
FieldType type = sf.getType();
if(returnFields == null || returnFields.wantsField(fieldName)) {
SchemaField sf = schema.getFieldOrNull(fieldName);
if (sf == null) {
sf = new SchemaField(fieldName, new TextField());
}
FieldType type = sf.getType();
if (fidx1 + 1 == fidx2) {
if (sf.multiValued()) {
String sv = value.stringValue();
kv.put(fieldName, field2string(type, sv));
} else {
kv.put(fieldName, field2string(type, value.stringValue()));
}
} else {
int c = 0;
for (int i = fidx1; i < fidx2; i++) {
String sv = fields.get(i).stringValue();
kv.put(fieldName + "_" + c++, field2string(type, sv));
}
if (fidx1 + 1 == fidx2) {
if (sf.multiValued()) {
String sv = value.stringValue();
kv.put(fieldName, field2string(type, sv));
} else {
kv.put(fieldName, field2string(type, value.stringValue()));
}
} else {
int c = 0;
for (int i = fidx1; i < fidx2; i++) {
String sv = fields.get(i).stringValue();
kv.put(fieldName + "_" + c++, field2string(type, sv));
}
}
}
fidx1 = fidx2;
}
return kv;
}
/**
* Translate an indexed document in a map of field names to field values.
* @param schema the schema of the indexed document. Must not be null.
* @param doc the indexed document. Must not be null.
* @return a map of field names to field values
*/
public static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc) {
return translateDoc(schema, doc, null);
}
private static String field2string(final FieldType type, final String value) {
String typeName = type.getTypeName();

3
source/net/yacy/http/servlets/SolrSelectServlet.java
Unescape View File

@ -99,8 +99,7 @@ public class SolrSelectServlet extends HttpServlet {
RESPONSE_WRITER.putAll(SolrCore.DEFAULT_RESPONSE_WRITERS);
XSLTResponseWriter xsltWriter = new XSLTResponseWriter();
OpensearchResponseWriter opensearchResponseWriter = new OpensearchResponseWriter();
@SuppressWarnings("rawtypes")
NamedList initArgs = new NamedList();
NamedList<?> initArgs = new NamedList<>();
xsltWriter.init(initArgs);
RESPONSE_WRITER.put("xslt", xsltWriter); // try i.e. http://localhost:8090/solr/select?q=*:*&start=0&rows=10&wt=xslt&tr=json.xsl
RESPONSE_WRITER.put("exml", new EnhancedXMLResponseWriter());

Write Preview
Loading…
Cancel
Save