boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Made Solr OpenSearch RSS writer compatible with external Solr index

Browse Source 
Worked previously only with responses from YaCy embedded Solr, now able
to render the response when YaCy is configured to use an external Solr
index.
pull/218/head
luccioman 7 years ago
parent 2bbf070f57
commit f4267ed247
3 changed files with 377 additions and 123 deletions
Show Stats Download Patch File Download Diff File

489
source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java
Unescape View File

@ -33,20 +33,14 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import net.yacy.cora.document.feed.RSSMessage;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.lod.vocabulary.DublinCore;
import net.yacy.cora.lod.vocabulary.Geo;
import net.yacy.cora.lod.vocabulary.YaCyMetadata;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.crawler.retrieval.Response;
import net.yacy.search.schema.CollectionConfiguration;
import net.yacy.search.schema.CollectionSchema;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.XML;
@ -58,7 +52,21 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
import net.yacy.cora.document.feed.RSSMessage;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.lod.vocabulary.DublinCore;
import net.yacy.cora.lod.vocabulary.Geo;
import net.yacy.cora.lod.vocabulary.YaCyMetadata;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.crawler.retrieval.Response;
import net.yacy.search.schema.CollectionConfiguration;
import net.yacy.search.schema.CollectionSchema;
/**
* Solr response writer producing an OpenSearch representation in RSS 2.0 format.
* @see <a href="https://github.com/dewitt/opensearch/blob/master/opensearch-1-1-draft-6.md#example-of-opensearch-response-elements-in-rss-20">Example of OpenSearch response elements in RSS 2.0</a>
*/
public class OpensearchResponseWriter implements QueryResponseWriter, SolrjResponseWriter {
// define a list of simple YaCySchema -> RSS Token matchings
private static final Map<String, String> field2tag = new HashMap<String, String>();
@ -81,7 +89,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo
private String title;
public static class ResHead {
public int offset, rows, numFound;
public long offset, rows, numFound;
//public int status, QTime;
//public String df, q, wt;
//public float maxScore;
@ -107,33 +115,139 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo
@Override
public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException {
NamedList<?> values = rsp.getValues();
final NamedList<?> values = rsp.getValues();
final Object responseObj = rsp.getResponse();
assert values.get("responseHeader") != null;
assert values.get("response") != null;
SimpleOrderedMap<Object> responseHeader = (SimpleOrderedMap<Object>) rsp.getResponseHeader();
DocList response = ((ResultContext) values.get("response")).getDocList();
final NamedList<Object> responseHeader = rsp.getResponseHeader();
write(writer, request, values, responseHeader, responseObj);
}
@Override
public void write(Writer writer, SolrQueryRequest request, String coreName, QueryResponse rsp) throws IOException {
final NamedList<Object> values = rsp.getResponse();
final NamedList<?> responseHeader = rsp.getResponseHeader();
final SolrDocumentList documents = rsp.getResults();
write(writer, request, values, responseHeader, documents);
}
/**
* Append to the writer the OpenSearch RSS representation of the Solr results.
* @param writer an open output writer. Must not be null.
* @param request the initial Solr request. Must not be null.
* @param values the response values. Must not be null.
* @param rsp the Solr response header.
* @throws IOException when a write error occurred
*/
private void write(final Writer writer, final SolrQueryRequest request, final NamedList<?> values,
final NamedList<?> responseHeader, final Object responseObj) throws IOException {
final ResHead resHead = new ResHead();
final int responseCount;
@SuppressWarnings("unchecked")
SimpleOrderedMap<Object> facetCounts = (SimpleOrderedMap<Object>) values.get("facet_counts");
@SuppressWarnings("unchecked")
SimpleOrderedMap<Object> facetFields = facetCounts == null || facetCounts.size() == 0 ? null : (SimpleOrderedMap<Object>) facetCounts.get("facet_fields");
@SuppressWarnings("unchecked")
SimpleOrderedMap<Object> highlighting = (SimpleOrderedMap<Object>) values.get("highlighting");
Map<String, LinkedHashSet<String>> snippets = highlighting(highlighting);
final Map<String, LinkedHashSet<String>> snippets = highlighting(highlighting);
if(responseObj instanceof ResultContext){
/* Regular response object */
final DocList documents = ((ResultContext)responseObj).getDocList();
resHead.offset = documents.offset(); // equal to 'start' Solr param
resHead.numFound = documents.matches();
responseCount = documents.size();
writeHeader(writer, responseHeader, resHead);
// parse response header
ResHead resHead = new ResHead();
NamedList<?> val0 = (NamedList<?>) responseHeader.get("params");
writeDocs(writer, documents, request, responseCount, snippets);
} else if(responseObj instanceof SolrDocumentList) {
/*
* The response object can be a SolrDocumentList when the response is partial,
* for example when the allowed processing time has been exceeded
*/
final SolrDocumentList documents = ((SolrDocumentList)responseObj);
resHead.offset = documents.getStart(); // equal to 'start' Solr param
resHead.numFound = documents.getNumFound();
responseCount = documents.size();
writeHeader(writer, responseHeader, resHead);
writeDocs(writer, documents, responseCount, snippets);
} else {
throw new IOException("Unable to process Solr response format");
}
openTag(writer, "yacy:navigation");
// the facets can be created with the options &facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt
@SuppressWarnings("unchecked")
NamedList<Integer> domains = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.host_s.getSolrFieldName());
@SuppressWarnings("unchecked")
NamedList<Integer> filetypes = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.url_file_ext_s.getSolrFieldName());
@SuppressWarnings("unchecked")
NamedList<Integer> protocols = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.url_protocol_s.getSolrFieldName());
@SuppressWarnings("unchecked")
NamedList<Integer> authors = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.author_sxt.getSolrFieldName());
@SuppressWarnings("unchecked")
NamedList<Integer> collections = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.collection_sxt.getSolrFieldName());
if (domains != null) {
openTag(writer, "yacy:facet name=\"domains\" displayname=\"Domains\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: domains) facetEntry(writer, "site", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
if (filetypes != null) {
openTag(writer, "yacy:facet name=\"filetypes\" displayname=\"Filetypes\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: filetypes) facetEntry(writer, "filetype", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
if (protocols != null) {
openTag(writer, "yacy:facet name=\"protocols\" displayname=\"Protocols\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: protocols) facetEntry(writer, "protocol", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
if (authors != null) {
openTag(writer, "yacy:facet name=\"authors\" displayname=\"Authors\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: authors) facetEntry(writer, "author", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
if (collections != null) {
openTag(writer, "yacy:facet name=\"collections\" displayname=\"Collections\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: collections) facetEntry(writer, "collection", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
closeTag(writer, "yacy:navigation");
closeTag(writer, "channel");
writer.write("</rss>\n".toCharArray());
}
/**
* Append to the writer the header of the OpenSearch RSS representation.
* @param writer an open output writer. Must not be null.
* @param responseHeader the Solr response header. Must not be null.
* @param resHead the calculated results head. Must not be null.
* @throws IOException when an unexpected error occurred while writing
*/
private void writeHeader(final Writer writer, final NamedList<?> responseHeader, final ResHead resHead)
throws IOException {
// parse response header
final NamedList<?> val0 = (NamedList<?>) responseHeader.get("params");
resHead.rows = Integer.parseInt((String) val0.get("rows"));
resHead.offset = response.offset(); // equal to 'start'
resHead.numFound = response.matches();
//resHead.df = (String) val0.get("df");
//resHead.q = (String) val0.get("q");
//resHead.wt = (String) val0.get("wt");
//resHead.status = (Integer) responseHeader.get("status");
//resHead.QTime = (Integer) responseHeader.get("QTime");
//resHead.maxScore = response.maxScore();
// write header
writer.write((
@ -148,21 +262,170 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo
" xmlns:geo=\"" + Geo.NAMESPACE + "\"\n" +
">\n").toCharArray());
openTag(writer, "channel");
solitaireTag(writer, "opensearch:totalResults", Integer.toString(resHead.numFound));
solitaireTag(writer, "opensearch:startIndex", Integer.toString(resHead.offset));
solitaireTag(writer, "opensearch:itemsPerPage", Integer.toString(resHead.rows));
solitaireTag(writer, "opensearch:totalResults", Long.toString(resHead.numFound));
solitaireTag(writer, "opensearch:startIndex", Long.toString(resHead.offset));
solitaireTag(writer, "opensearch:itemsPerPage", Long.toString(resHead.rows));
solitaireTag(writer, RSSMessage.Token.title.name(), this.title);
writer.write("<atom:link rel=\"search\" href=\"/opensearchdescription.xml\" type=\"application/opensearchdescription+xml\"/>");
solitaireTag(writer, "description", "Search Result");
//solitaireTag(writer, "link", "");
//solitaireTag(writer, "image", "");
}
/**
* Append to the writer the OpenSearch RSS representation of Solr documents.
*
* @param writer an open output writer. Must not be null.
* @param documents the documents to render. Must not be null.
* @param responseCount the number of documents to process
* @param snippets snippets Solr computed text snippets (highlighting).
* @throws IOException when an unexpected error occurred while writing
*/
private void writeDocs(final Writer writer, final SolrDocumentList documents, final int responseCount,
final Map<String, LinkedHashSet<String>> snippets) throws IOException {
// parse body
String urlhash = null;
MultiProtocolURL url = null;
final Iterator<SolrDocument> iterator = documents.iterator();
for (int i = 0; i < responseCount; i++) {
openTag(writer, "item");
SolrDocument doc = iterator.next();
List<String> texts = new ArrayList<String>();
List<String> descriptions = new ArrayList<String>();
String docTitle = "";
List<Object> images_protocol_obj = new ArrayList<>();
List<String> images_stub = new ArrayList<>();
for (final Entry<String, Object> fieldEntry : doc) {
final String fieldName = fieldEntry.getKey();
final Object value = fieldEntry.getValue();
if(value == null) {
continue;
}
// parse body
final int responseCount = response.size();
// apply generic matching rule
String stag = field2tag.get(fieldName);
if (stag != null) {
solitaireTag(writer, stag, value.toString());
continue;
}
// take apart the url
if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) {
url = writeLink(writer, value.toString());
continue;
}
// if the rule is not generic, use the specific here
if (CollectionSchema.id.getSolrFieldName().equals(fieldName)) {
urlhash = value.toString();
solitaireTag(writer, RSSMessage.Token.guid.name(), urlhash, "isPermaLink=\"false\"");
continue;
}
if (CollectionSchema.title.getSolrFieldName().equals(fieldName)) {
if(value instanceof Iterable<?>) {
/* Handle multivalued field */
for(final Object valueItem : (Iterable<?>)value) {
docTitle = valueItem.toString();
texts.add(docTitle);
}
} else {
docTitle = value.toString();
texts.add(docTitle);
}
continue;
}
if (CollectionSchema.last_modified.getSolrFieldName().equals(fieldName) && value instanceof Date) {
solitaireTag(writer, RSSMessage.Token.pubDate.name(), HeaderFramework.formatRFC1123((Date)value));
continue;
}
if (CollectionSchema.description_txt.getSolrFieldName().equals(fieldName)) {
if(value instanceof Iterable<?>) {
/* Handle multivalued field */
for(final Object valueItem : (Iterable<?>)value) {
final String description = valueItem.toString();
descriptions.add(description);
texts.add(description);
solitaireTag(writer, DublinCore.Description.getURIref(), description);
}
} else {
final String description = value.toString();
descriptions.add(description);
texts.add(description);
solitaireTag(writer, DublinCore.Description.getURIref(), description);
}
continue;
}
if (CollectionSchema.text_t.getSolrFieldName().equals(fieldName)) {
texts.add(value.toString());
continue;
}
if (CollectionSchema.size_i.getSolrFieldName().equals(fieldName) && value instanceof Integer) {
int size = ((Integer)value).intValue();
solitaireTag(writer, YaCyMetadata.size.getURIref(), Integer.toString(size));
solitaireTag(writer, YaCyMetadata.sizename.getURIref(), RSSMessage.sizename(size));
continue;
}
if (CollectionSchema.h1_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h2_txt.getSolrFieldName().equals(fieldName) ||
CollectionSchema.h3_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h4_txt.getSolrFieldName().equals(fieldName) ||
CollectionSchema.h5_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h6_txt.getSolrFieldName().equals(fieldName)) {
if(value instanceof Iterable<?>) {
// because these are multi-valued fields, there can be several of each
for(final Object valueItem : (Iterable<?>)value) {
texts.add(valueItem.toString());
}
} else {
texts.add(value.toString());
}
continue;
}
if (CollectionSchema.images_protocol_sxt.getSolrFieldName().equals(fieldName)) {
if(value instanceof Iterable<?>) {
/* Handle multivalued field */
for(final Object valueItem : (Iterable<?>)value) {
images_protocol_obj.add(valueItem.toString());
}
} else {
images_protocol_obj.add(value.toString());
}
continue;
}
if (CollectionSchema.images_urlstub_sxt.getSolrFieldName().equals(fieldName)) {
if(value instanceof Iterable<?>) {
/* Handle multivalued field */
for(final Object valueItem : (Iterable<?>)value) {
images_stub.add(valueItem.toString());
}
} else {
images_stub.add(value.toString());
}
continue;
}
}
final Object keywordsObj = doc.get(CollectionSchema.keywords.getSolrFieldName());
final String keywords = (keywordsObj instanceof String) ? (String)keywordsObj : null;
writeDocEnd(writer, snippets, urlhash, url, keywords, texts, descriptions, docTitle, images_protocol_obj,
images_stub);
}
}
/**
* Append to the writer the OpenSearch RSS representation of Solr documents.
*
* @param writer an open output writer. Must not be null.
* @param documents the documents to render. Must not be null.
* @param responseCount the number of documents to process
* @param snippets Solr computed text snippets (highlighting).
* @throws IOException when an unexpected error occurred while writing
*/
private void writeDocs(final Writer writer, final DocList documents, final SolrQueryRequest request, final int responseCount,
final Map<String, LinkedHashSet<String>> snippets) throws IOException {
// parse body
SolrIndexSearcher searcher = request.getSearcher();
DocIterator iterator = response.iterator();
String urlhash = null;
MultiProtocolURL url = null;
final DocIterator iterator = documents.iterator();
for (int i = 0; i < responseCount; i++) {
openTag(writer, "item");
int id = iterator.nextDoc();
@ -171,7 +434,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo
int fieldc = fields.size();
List<String> texts = new ArrayList<String>();
List<String> descriptions = new ArrayList<String>();
String title = "";
String docTitle = "";
List<Object> images_protocol_obj = new ArrayList<>();
List<String> images_stub = new ArrayList<>();
for (int j = 0; j < fieldc; j++) {
@ -187,14 +450,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo
// take apart the url
if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) {
String u = value.stringValue();
solitaireTag(writer, RSSMessage.Token.link.name(), u);
try {
url = new MultiProtocolURL(u);
solitaireTag(writer, YaCyMetadata.host.getURIref(), url.getHost());
solitaireTag(writer, YaCyMetadata.path.getURIref(), url.getPath());
solitaireTag(writer, YaCyMetadata.file.getURIref(), url.getFileName());
} catch (final MalformedURLException e) {}
url = writeLink(writer, value.stringValue());
continue;
}
@ -205,8 +461,8 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo
continue;
}
if (CollectionSchema.title.getSolrFieldName().equals(fieldName)) {
title = value.stringValue();
texts.add(title);
docTitle = value.stringValue();
texts.add(docTitle);
continue;
}
if (CollectionSchema.last_modified.getSolrFieldName().equals(fieldName)) {
@ -248,82 +504,77 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo
}
}
if (Math.min(images_protocol_obj.size(), images_stub.size()) > 0) {
List<String> images_protocol = CollectionConfiguration.indexedList2protocolList(images_protocol_obj, images_stub.size());
String imageurl = images_protocol.get(0) + "://" + images_stub.get(0);
writer.write("<media:content medium=\"image\" url=\"");
XML.escapeCharData(imageurl, writer); writer.write("\"/>\n");
} else {
if (url != null && Response.docTypeExt(MultiProtocolURL.getFileExtension(url.getFile()).toLowerCase(Locale.ROOT)) == Response.DT_IMAGE) {
writer.write("<media:content medium=\"image\" url=\"");
XML.escapeCharData(url.toNormalform(true), writer); writer.write("\"/>\n");
}
}
final Object keywordsObj = doc.get(CollectionSchema.keywords.getSolrFieldName());
final String keywords = (keywordsObj instanceof String) ? (String)keywordsObj : null;
// compute snippet from texts
solitaireTag(writer, RSSMessage.Token.title.name(), title.length() == 0 ? (texts.size() == 0 ? "" : texts.get(0)) : title);
LinkedHashSet<String> snippet = urlhash == null ? null : snippets.get(urlhash);
String tagname = RSSMessage.Token.description.name();
if (snippet == null || snippet.size() == 0) {
writer.write("<"); writer.write(tagname); writer.write('>');
for (String d: descriptions) {
XML.escapeCharData(d, writer);
}
writer.write("</"); writer.write(tagname); writer.write(">\n");
} else {
removeSubsumedTitle(snippet, title);
solitaireTag(writer, tagname, getLargestSnippet(snippet)); // snippet may be size=0
}
solitaireTag(writer, DublinCore.Subject.getURIref(), doc.get(CollectionSchema.keywords.getSolrFieldName()));
closeTag(writer, "item");
}
openTag(writer, "yacy:navigation");
// the facets can be created with the options &facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt
@SuppressWarnings("unchecked")
NamedList<Integer> domains = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.host_s.getSolrFieldName());
@SuppressWarnings("unchecked")
NamedList<Integer> filetypes = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.url_file_ext_s.getSolrFieldName());
@SuppressWarnings("unchecked")
NamedList<Integer> protocols = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.url_protocol_s.getSolrFieldName());
@SuppressWarnings("unchecked")
NamedList<Integer> authors = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.author_sxt.getSolrFieldName());
@SuppressWarnings("unchecked")
NamedList<Integer> collections = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.collection_sxt.getSolrFieldName());
if (domains != null) {
openTag(writer, "yacy:facet name=\"domains\" displayname=\"Domains\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: domains) facetEntry(writer, "site", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
if (filetypes != null) {
openTag(writer, "yacy:facet name=\"filetypes\" displayname=\"Filetypes\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: filetypes) facetEntry(writer, "filetype", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
if (protocols != null) {
openTag(writer, "yacy:facet name=\"protocols\" displayname=\"Protocols\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: protocols) facetEntry(writer, "protocol", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
if (authors != null) {
openTag(writer, "yacy:facet name=\"authors\" displayname=\"Authors\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: authors) facetEntry(writer, "author", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
}
if (collections != null) {
openTag(writer, "yacy:facet name=\"collections\" displayname=\"Collections\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\"");
for (Map.Entry<String, Integer> entry: collections) facetEntry(writer, "collection", entry.getKey(), Integer.toString(entry.getValue()));
closeTag(writer, "yacy:facet");
writeDocEnd(writer, snippets, urlhash, url, keywords, texts, descriptions, docTitle, images_protocol_obj,
images_stub);
}
closeTag(writer, "yacy:navigation");
}
/**
* Append the Solr document URL as a RSS link to the writer
* @param writer
* @param sku
* @return a MultiProtocolURL instance or null
* @throws IOException
*/
private MultiProtocolURL writeLink(final Writer writer, final String sku)
throws IOException {
solitaireTag(writer, RSSMessage.Token.link.name(), sku);
MultiProtocolURL url;
try {
url = new MultiProtocolURL(sku);
solitaireTag(writer, YaCyMetadata.host.getURIref(), url.getHost());
solitaireTag(writer, YaCyMetadata.path.getURIref(), url.getPath());
solitaireTag(writer, YaCyMetadata.file.getURIref(), url.getFileName());
} catch (final MalformedURLException e) {
url = null;
}
return url;
}
/**
* Append to the writer the end of the RSS OpenSearch representation of the Solr
* document.
*/
private void writeDocEnd(final Writer writer, final Map<String, LinkedHashSet<String>> snippets, final String urlhash,
final MultiProtocolURL url, final String keywords, final List<String> texts, final List<String> descriptions, final String docTitle,
final List<Object> imagesProtocolObjs, final List<String> imagesStubs) throws IOException {
if (Math.min(imagesProtocolObjs.size(), imagesStubs.size()) > 0) {
List<String> imagesProtocols = CollectionConfiguration.indexedList2protocolList(imagesProtocolObjs, imagesStubs.size());
String imageurl = imagesProtocols.get(0) + "://" + imagesStubs.get(0);
writer.write("<media:content medium=\"image\" url=\"");
XML.escapeCharData(imageurl, writer); writer.write("\"/>\n");
} else {
if (url != null && Response.docTypeExt(MultiProtocolURL.getFileExtension(url.getFile()).toLowerCase(Locale.ROOT)) == Response.DT_IMAGE) {
writer.write("<media:content medium=\"image\" url=\"");
XML.escapeCharData(url.toNormalform(true), writer); writer.write("\"/>\n");
}
}
// compute snippet from texts
solitaireTag(writer, RSSMessage.Token.title.name(), docTitle.length() == 0 ? (texts.size() == 0 ? "" : texts.get(0)) : docTitle);
LinkedHashSet<String> snippet = urlhash == null ? null : snippets.get(urlhash);
String tagname = RSSMessage.Token.description.name();
if (snippet == null || snippet.size() == 0) {
writer.write("<"); writer.write(tagname); writer.write('>');
for (String d: descriptions) {
XML.escapeCharData(d, writer);
}
writer.write("</"); writer.write(tagname); writer.write(">\n");
} else {
removeSubsumedTitle(snippet, docTitle);
solitaireTag(writer, tagname, getLargestSnippet(snippet)); // snippet may be size=0
}
if(keywords != null) {
solitaireTag(writer, DublinCore.Subject.getURIref(), keywords);
}
closeTag(writer, "item");
}
closeTag(writer, "channel");
writer.write("</rss>\n".toCharArray());
}
/**
* produce snippets from solr (they call that 'highlighting')

8
source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java
Unescape View File

@ -109,7 +109,7 @@ public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrRes
// parse response header
ResHead resHead = new ResHead();
NamedList<?> val0 = (NamedList<?>) responseHeader.get("params");
resHead.rows = Integer.parseInt((String) val0.get("rows"));
resHead.rows = Long.parseLong((String) val0.get("rows"));
resHead.offset = response.offset(); // equal to 'start'
resHead.numFound = response.matches();
@ -121,9 +121,9 @@ public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrRes
// write header
writer.write(("{\"channels\": [{\n").toCharArray());
solitaireTag(writer, "totalResults", Integer.toString(resHead.numFound));
solitaireTag(writer, "startIndex", Integer.toString(resHead.offset));
solitaireTag(writer, "itemsPerPage", Integer.toString(resHead.rows));
solitaireTag(writer, "totalResults", Long.toString(resHead.numFound));
solitaireTag(writer, "startIndex", Long.toString(resHead.offset));
solitaireTag(writer, "itemsPerPage", Long.toString(resHead.rows));
solitaireTag(writer, "title", this.title);
solitaireTag(writer, "description", "Search Result");
writer.write("\"items\": [\n".toCharArray());

3
source/net/yacy/http/servlets/SolrSelectServlet.java
Unescape View File

@ -225,6 +225,9 @@ public class SolrSelectServlet extends HttpServlet {
CollectionSchema.title.getSolrFieldName() + "," +
CollectionSchema.description_txt.getSolrFieldName() + "," +
CollectionSchema.id.getSolrFieldName() + "," +
CollectionSchema.author.getSolrFieldName() + "," +
CollectionSchema.publisher_t.getSolrFieldName() + "," +
CollectionSchema.keywords.getSolrFieldName() + "," +
CollectionSchema.url_paths_sxt.getSolrFieldName() + "," +
CollectionSchema.last_modified.getSolrFieldName() + "," +
CollectionSchema.size_i.getSolrFieldName() + "," +

Write Preview
Loading…
Cancel
Save