diff --git a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java index 747c19b55..db1d72ef5 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java @@ -33,20 +33,14 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.Map.Entry; import java.util.regex.Pattern; -import net.yacy.cora.document.feed.RSSMessage; -import net.yacy.cora.document.id.MultiProtocolURL; -import net.yacy.cora.lod.vocabulary.DublinCore; -import net.yacy.cora.lod.vocabulary.Geo; -import net.yacy.cora.lod.vocabulary.YaCyMetadata; -import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.crawler.retrieval.Response; -import net.yacy.search.schema.CollectionConfiguration; -import net.yacy.search.schema.CollectionSchema; - import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexableField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.XML; @@ -58,7 +52,21 @@ import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; -public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter { +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.id.MultiProtocolURL; +import net.yacy.cora.lod.vocabulary.DublinCore; +import net.yacy.cora.lod.vocabulary.Geo; +import net.yacy.cora.lod.vocabulary.YaCyMetadata; +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.crawler.retrieval.Response; +import net.yacy.search.schema.CollectionConfiguration; +import net.yacy.search.schema.CollectionSchema; + +/** + * Solr response writer producing an OpenSearch representation in RSS 2.0 format. + * @see Example of OpenSearch response elements in RSS 2.0 + */ +public class OpensearchResponseWriter implements QueryResponseWriter, SolrjResponseWriter { // define a list of simple YaCySchema -> RSS Token matchings private static final Map field2tag = new HashMap(); @@ -81,7 +89,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo private String title; public static class ResHead { - public int offset, rows, numFound; + public long offset, rows, numFound; //public int status, QTime; //public String df, q, wt; //public float maxScore; @@ -107,33 +115,139 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo @Override public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { - NamedList values = rsp.getValues(); + final NamedList values = rsp.getValues(); + + final Object responseObj = rsp.getResponse(); assert values.get("responseHeader") != null; assert values.get("response") != null; - SimpleOrderedMap responseHeader = (SimpleOrderedMap) rsp.getResponseHeader(); - DocList response = ((ResultContext) values.get("response")).getDocList(); + final NamedList responseHeader = rsp.getResponseHeader(); + + write(writer, request, values, responseHeader, responseObj); + } + + @Override + public void write(Writer writer, SolrQueryRequest request, String coreName, QueryResponse rsp) throws IOException { + + final NamedList values = rsp.getResponse(); + + final NamedList responseHeader = rsp.getResponseHeader(); + + final SolrDocumentList documents = rsp.getResults(); + + write(writer, request, values, responseHeader, documents); + } + + /** + * Append to the writer the OpenSearch RSS representation of the Solr results. + * @param writer an open output writer. Must not be null. + * @param request the initial Solr request. Must not be null. + * @param values the response values. Must not be null. + * @param rsp the Solr response header. + * @throws IOException when a write error occurred + */ + private void write(final Writer writer, final SolrQueryRequest request, final NamedList values, + final NamedList responseHeader, final Object responseObj) throws IOException { + final ResHead resHead = new ResHead(); + final int responseCount; + @SuppressWarnings("unchecked") SimpleOrderedMap facetCounts = (SimpleOrderedMap) values.get("facet_counts"); @SuppressWarnings("unchecked") SimpleOrderedMap facetFields = facetCounts == null || facetCounts.size() == 0 ? null : (SimpleOrderedMap) facetCounts.get("facet_fields"); @SuppressWarnings("unchecked") SimpleOrderedMap highlighting = (SimpleOrderedMap) values.get("highlighting"); - Map> snippets = highlighting(highlighting); + final Map> snippets = highlighting(highlighting); + + if(responseObj instanceof ResultContext){ + /* Regular response object */ + final DocList documents = ((ResultContext)responseObj).getDocList(); + + resHead.offset = documents.offset(); // equal to 'start' Solr param + resHead.numFound = documents.matches(); + + responseCount = documents.size(); + + writeHeader(writer, responseHeader, resHead); + + writeDocs(writer, documents, request, responseCount, snippets); + } else if(responseObj instanceof SolrDocumentList) { + /* + * The response object can be a SolrDocumentList when the response is partial, + * for example when the allowed processing time has been exceeded + */ + final SolrDocumentList documents = ((SolrDocumentList)responseObj); + + resHead.offset = documents.getStart(); // equal to 'start' Solr param + resHead.numFound = documents.getNumFound(); + + responseCount = documents.size(); + + writeHeader(writer, responseHeader, resHead); + + writeDocs(writer, documents, responseCount, snippets); + } else { + throw new IOException("Unable to process Solr response format"); + } - // parse response header - ResHead resHead = new ResHead(); - NamedList val0 = (NamedList) responseHeader.get("params"); + openTag(writer, "yacy:navigation"); + + // the facets can be created with the options &facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt + @SuppressWarnings("unchecked") + NamedList domains = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.host_s.getSolrFieldName()); + @SuppressWarnings("unchecked") + NamedList filetypes = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.url_file_ext_s.getSolrFieldName()); + @SuppressWarnings("unchecked") + NamedList protocols = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.url_protocol_s.getSolrFieldName()); + @SuppressWarnings("unchecked") + NamedList authors = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.author_sxt.getSolrFieldName()); + @SuppressWarnings("unchecked") + NamedList collections = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.collection_sxt.getSolrFieldName()); + + if (domains != null) { + openTag(writer, "yacy:facet name=\"domains\" displayname=\"Domains\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: domains) facetEntry(writer, "site", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + if (filetypes != null) { + openTag(writer, "yacy:facet name=\"filetypes\" displayname=\"Filetypes\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: filetypes) facetEntry(writer, "filetype", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + if (protocols != null) { + openTag(writer, "yacy:facet name=\"protocols\" displayname=\"Protocols\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: protocols) facetEntry(writer, "protocol", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + if (authors != null) { + openTag(writer, "yacy:facet name=\"authors\" displayname=\"Authors\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: authors) facetEntry(writer, "author", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + if (collections != null) { + openTag(writer, "yacy:facet name=\"collections\" displayname=\"Collections\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: collections) facetEntry(writer, "collection", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + closeTag(writer, "yacy:navigation"); + + closeTag(writer, "channel"); + writer.write("\n".toCharArray()); + } + + /** + * Append to the writer the header of the OpenSearch RSS representation. + * @param writer an open output writer. Must not be null. + * @param responseHeader the Solr response header. Must not be null. + * @param resHead the calculated results head. Must not be null. + * @throws IOException when an unexpected error occurred while writing + */ + private void writeHeader(final Writer writer, final NamedList responseHeader, final ResHead resHead) + throws IOException { + // parse response header + final NamedList val0 = (NamedList) responseHeader.get("params"); resHead.rows = Integer.parseInt((String) val0.get("rows")); - resHead.offset = response.offset(); // equal to 'start' - resHead.numFound = response.matches(); - //resHead.df = (String) val0.get("df"); - //resHead.q = (String) val0.get("q"); - //resHead.wt = (String) val0.get("wt"); - //resHead.status = (Integer) responseHeader.get("status"); - //resHead.QTime = (Integer) responseHeader.get("QTime"); - //resHead.maxScore = response.maxScore(); // write header writer.write(( @@ -148,21 +262,170 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo " xmlns:geo=\"" + Geo.NAMESPACE + "\"\n" + ">\n").toCharArray()); openTag(writer, "channel"); - solitaireTag(writer, "opensearch:totalResults", Integer.toString(resHead.numFound)); - solitaireTag(writer, "opensearch:startIndex", Integer.toString(resHead.offset)); - solitaireTag(writer, "opensearch:itemsPerPage", Integer.toString(resHead.rows)); + solitaireTag(writer, "opensearch:totalResults", Long.toString(resHead.numFound)); + solitaireTag(writer, "opensearch:startIndex", Long.toString(resHead.offset)); + solitaireTag(writer, "opensearch:itemsPerPage", Long.toString(resHead.rows)); solitaireTag(writer, RSSMessage.Token.title.name(), this.title); writer.write(""); solitaireTag(writer, "description", "Search Result"); - //solitaireTag(writer, "link", ""); - //solitaireTag(writer, "image", ""); + } + + /** + * Append to the writer the OpenSearch RSS representation of Solr documents. + * + * @param writer an open output writer. Must not be null. + * @param documents the documents to render. Must not be null. + * @param responseCount the number of documents to process + * @param snippets snippets Solr computed text snippets (highlighting). + * @throws IOException when an unexpected error occurred while writing + */ + private void writeDocs(final Writer writer, final SolrDocumentList documents, final int responseCount, + final Map> snippets) throws IOException { + // parse body + String urlhash = null; + MultiProtocolURL url = null; + final Iterator iterator = documents.iterator(); + for (int i = 0; i < responseCount; i++) { + openTag(writer, "item"); + SolrDocument doc = iterator.next(); + List texts = new ArrayList(); + List descriptions = new ArrayList(); + String docTitle = ""; + List images_protocol_obj = new ArrayList<>(); + List images_stub = new ArrayList<>(); + for (final Entry fieldEntry : doc) { + final String fieldName = fieldEntry.getKey(); + final Object value = fieldEntry.getValue(); + + if(value == null) { + continue; + } + + // apply generic matching rule + String stag = field2tag.get(fieldName); + if (stag != null) { + solitaireTag(writer, stag, value.toString()); + continue; + } + + // take apart the url + if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) { + url = writeLink(writer, value.toString()); + continue; + } + + // if the rule is not generic, use the specific here + if (CollectionSchema.id.getSolrFieldName().equals(fieldName)) { + urlhash = value.toString(); + solitaireTag(writer, RSSMessage.Token.guid.name(), urlhash, "isPermaLink=\"false\""); + continue; + } + if (CollectionSchema.title.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + /* Handle multivalued field */ + for(final Object valueItem : (Iterable)value) { + docTitle = valueItem.toString(); + texts.add(docTitle); + } + } else { + docTitle = value.toString(); + texts.add(docTitle); + } + continue; + } + if (CollectionSchema.last_modified.getSolrFieldName().equals(fieldName) && value instanceof Date) { + solitaireTag(writer, RSSMessage.Token.pubDate.name(), HeaderFramework.formatRFC1123((Date)value)); + continue; + } + if (CollectionSchema.description_txt.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + /* Handle multivalued field */ + for(final Object valueItem : (Iterable)value) { + final String description = valueItem.toString(); + descriptions.add(description); + texts.add(description); + solitaireTag(writer, DublinCore.Description.getURIref(), description); + } + } else { + final String description = value.toString(); + descriptions.add(description); + texts.add(description); + solitaireTag(writer, DublinCore.Description.getURIref(), description); + } + + continue; + } + if (CollectionSchema.text_t.getSolrFieldName().equals(fieldName)) { + texts.add(value.toString()); + continue; + } + if (CollectionSchema.size_i.getSolrFieldName().equals(fieldName) && value instanceof Integer) { + int size = ((Integer)value).intValue(); + solitaireTag(writer, YaCyMetadata.size.getURIref(), Integer.toString(size)); + solitaireTag(writer, YaCyMetadata.sizename.getURIref(), RSSMessage.sizename(size)); + continue; + } + if (CollectionSchema.h1_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h2_txt.getSolrFieldName().equals(fieldName) || + CollectionSchema.h3_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h4_txt.getSolrFieldName().equals(fieldName) || + CollectionSchema.h5_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h6_txt.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + // because these are multi-valued fields, there can be several of each + for(final Object valueItem : (Iterable)value) { + texts.add(valueItem.toString()); + } + } else { + texts.add(value.toString()); + } + continue; + } + if (CollectionSchema.images_protocol_sxt.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + /* Handle multivalued field */ + for(final Object valueItem : (Iterable)value) { + images_protocol_obj.add(valueItem.toString()); + } + } else { + images_protocol_obj.add(value.toString()); + } + continue; + } + if (CollectionSchema.images_urlstub_sxt.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + /* Handle multivalued field */ + for(final Object valueItem : (Iterable)value) { + images_stub.add(valueItem.toString()); + } + } else { + images_stub.add(value.toString()); + } + continue; + } + } + + final Object keywordsObj = doc.get(CollectionSchema.keywords.getSolrFieldName()); + final String keywords = (keywordsObj instanceof String) ? (String)keywordsObj : null; + + writeDocEnd(writer, snippets, urlhash, url, keywords, texts, descriptions, docTitle, images_protocol_obj, + images_stub); + } + } - // parse body - final int responseCount = response.size(); + /** + * Append to the writer the OpenSearch RSS representation of Solr documents. + * + * @param writer an open output writer. Must not be null. + * @param documents the documents to render. Must not be null. + * @param responseCount the number of documents to process + * @param snippets Solr computed text snippets (highlighting). + * @throws IOException when an unexpected error occurred while writing + */ + private void writeDocs(final Writer writer, final DocList documents, final SolrQueryRequest request, final int responseCount, + final Map> snippets) throws IOException { + // parse body SolrIndexSearcher searcher = request.getSearcher(); - DocIterator iterator = response.iterator(); String urlhash = null; MultiProtocolURL url = null; + final DocIterator iterator = documents.iterator(); for (int i = 0; i < responseCount; i++) { openTag(writer, "item"); int id = iterator.nextDoc(); @@ -171,7 +434,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo int fieldc = fields.size(); List texts = new ArrayList(); List descriptions = new ArrayList(); - String title = ""; + String docTitle = ""; List images_protocol_obj = new ArrayList<>(); List images_stub = new ArrayList<>(); for (int j = 0; j < fieldc; j++) { @@ -187,14 +450,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo // take apart the url if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) { - String u = value.stringValue(); - solitaireTag(writer, RSSMessage.Token.link.name(), u); - try { - url = new MultiProtocolURL(u); - solitaireTag(writer, YaCyMetadata.host.getURIref(), url.getHost()); - solitaireTag(writer, YaCyMetadata.path.getURIref(), url.getPath()); - solitaireTag(writer, YaCyMetadata.file.getURIref(), url.getFileName()); - } catch (final MalformedURLException e) {} + url = writeLink(writer, value.stringValue()); continue; } @@ -205,8 +461,8 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo continue; } if (CollectionSchema.title.getSolrFieldName().equals(fieldName)) { - title = value.stringValue(); - texts.add(title); + docTitle = value.stringValue(); + texts.add(docTitle); continue; } if (CollectionSchema.last_modified.getSolrFieldName().equals(fieldName)) { @@ -248,82 +504,77 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo } } - if (Math.min(images_protocol_obj.size(), images_stub.size()) > 0) { - List images_protocol = CollectionConfiguration.indexedList2protocolList(images_protocol_obj, images_stub.size()); - String imageurl = images_protocol.get(0) + "://" + images_stub.get(0); - writer.write("\n"); - } else { - if (url != null && Response.docTypeExt(MultiProtocolURL.getFileExtension(url.getFile()).toLowerCase(Locale.ROOT)) == Response.DT_IMAGE) { - writer.write("\n"); - } - } - - // compute snippet from texts - solitaireTag(writer, RSSMessage.Token.title.name(), title.length() == 0 ? (texts.size() == 0 ? "" : texts.get(0)) : title); - LinkedHashSet snippet = urlhash == null ? null : snippets.get(urlhash); - String tagname = RSSMessage.Token.description.name(); - if (snippet == null || snippet.size() == 0) { - writer.write("<"); writer.write(tagname); writer.write('>'); - for (String d: descriptions) { - XML.escapeCharData(d, writer); - } - writer.write("\n"); - } else { - removeSubsumedTitle(snippet, title); - solitaireTag(writer, tagname, getLargestSnippet(snippet)); // snippet may be size=0 - } - - solitaireTag(writer, DublinCore.Subject.getURIref(), doc.get(CollectionSchema.keywords.getSolrFieldName())); + final Object keywordsObj = doc.get(CollectionSchema.keywords.getSolrFieldName()); + final String keywords = (keywordsObj instanceof String) ? (String)keywordsObj : null; - closeTag(writer, "item"); + writeDocEnd(writer, snippets, urlhash, url, keywords, texts, descriptions, docTitle, images_protocol_obj, + images_stub); } + } + + /** + * Append the Solr document URL as a RSS link to the writer + * @param writer + * @param sku + * @return a MultiProtocolURL instance or null + * @throws IOException + */ + private MultiProtocolURL writeLink(final Writer writer, final String sku) + throws IOException { + solitaireTag(writer, RSSMessage.Token.link.name(), sku); + MultiProtocolURL url; + try { + url = new MultiProtocolURL(sku); + solitaireTag(writer, YaCyMetadata.host.getURIref(), url.getHost()); + solitaireTag(writer, YaCyMetadata.path.getURIref(), url.getPath()); + solitaireTag(writer, YaCyMetadata.file.getURIref(), url.getFileName()); + } catch (final MalformedURLException e) { + url = null; + } + return url; + } - openTag(writer, "yacy:navigation"); - - // the facets can be created with the options &facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt - @SuppressWarnings("unchecked") - NamedList domains = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.host_s.getSolrFieldName()); - @SuppressWarnings("unchecked") - NamedList filetypes = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.url_file_ext_s.getSolrFieldName()); - @SuppressWarnings("unchecked") - NamedList protocols = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.url_protocol_s.getSolrFieldName()); - @SuppressWarnings("unchecked") - NamedList authors = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.author_sxt.getSolrFieldName()); - @SuppressWarnings("unchecked") - NamedList collections = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.collection_sxt.getSolrFieldName()); - - if (domains != null) { - openTag(writer, "yacy:facet name=\"domains\" displayname=\"Domains\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: domains) facetEntry(writer, "site", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - if (filetypes != null) { - openTag(writer, "yacy:facet name=\"filetypes\" displayname=\"Filetypes\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: filetypes) facetEntry(writer, "filetype", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - if (protocols != null) { - openTag(writer, "yacy:facet name=\"protocols\" displayname=\"Protocols\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: protocols) facetEntry(writer, "protocol", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - if (authors != null) { - openTag(writer, "yacy:facet name=\"authors\" displayname=\"Authors\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: authors) facetEntry(writer, "author", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - if (collections != null) { - openTag(writer, "yacy:facet name=\"collections\" displayname=\"Collections\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: collections) facetEntry(writer, "collection", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - closeTag(writer, "yacy:navigation"); - - closeTag(writer, "channel"); - writer.write("\n".toCharArray()); - } + /** + * Append to the writer the end of the RSS OpenSearch representation of the Solr + * document. + */ + private void writeDocEnd(final Writer writer, final Map> snippets, final String urlhash, + final MultiProtocolURL url, final String keywords, final List texts, final List descriptions, final String docTitle, + final List imagesProtocolObjs, final List imagesStubs) throws IOException { + if (Math.min(imagesProtocolObjs.size(), imagesStubs.size()) > 0) { + List imagesProtocols = CollectionConfiguration.indexedList2protocolList(imagesProtocolObjs, imagesStubs.size()); + String imageurl = imagesProtocols.get(0) + "://" + imagesStubs.get(0); + writer.write("\n"); + } else { + if (url != null && Response.docTypeExt(MultiProtocolURL.getFileExtension(url.getFile()).toLowerCase(Locale.ROOT)) == Response.DT_IMAGE) { + writer.write("\n"); + } + } + + // compute snippet from texts + solitaireTag(writer, RSSMessage.Token.title.name(), docTitle.length() == 0 ? (texts.size() == 0 ? "" : texts.get(0)) : docTitle); + LinkedHashSet snippet = urlhash == null ? null : snippets.get(urlhash); + String tagname = RSSMessage.Token.description.name(); + if (snippet == null || snippet.size() == 0) { + writer.write("<"); writer.write(tagname); writer.write('>'); + for (String d: descriptions) { + XML.escapeCharData(d, writer); + } + writer.write("\n"); + } else { + removeSubsumedTitle(snippet, docTitle); + solitaireTag(writer, tagname, getLargestSnippet(snippet)); // snippet may be size=0 + } + + if(keywords != null) { + solitaireTag(writer, DublinCore.Subject.getURIref(), keywords); + } + + closeTag(writer, "item"); + } + /** * produce snippets from solr (they call that 'highlighting') diff --git a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java index 3380f537b..499b12ed7 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java @@ -109,7 +109,7 @@ public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrRes // parse response header ResHead resHead = new ResHead(); NamedList val0 = (NamedList) responseHeader.get("params"); - resHead.rows = Integer.parseInt((String) val0.get("rows")); + resHead.rows = Long.parseLong((String) val0.get("rows")); resHead.offset = response.offset(); // equal to 'start' resHead.numFound = response.matches(); @@ -121,9 +121,9 @@ public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrRes // write header writer.write(("{\"channels\": [{\n").toCharArray()); - solitaireTag(writer, "totalResults", Integer.toString(resHead.numFound)); - solitaireTag(writer, "startIndex", Integer.toString(resHead.offset)); - solitaireTag(writer, "itemsPerPage", Integer.toString(resHead.rows)); + solitaireTag(writer, "totalResults", Long.toString(resHead.numFound)); + solitaireTag(writer, "startIndex", Long.toString(resHead.offset)); + solitaireTag(writer, "itemsPerPage", Long.toString(resHead.rows)); solitaireTag(writer, "title", this.title); solitaireTag(writer, "description", "Search Result"); writer.write("\"items\": [\n".toCharArray()); diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index d5944f40f..f670a91ba 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -225,6 +225,9 @@ public class SolrSelectServlet extends HttpServlet { CollectionSchema.title.getSolrFieldName() + "," + CollectionSchema.description_txt.getSolrFieldName() + "," + CollectionSchema.id.getSolrFieldName() + "," + + CollectionSchema.author.getSolrFieldName() + "," + + CollectionSchema.publisher_t.getSolrFieldName() + "," + + CollectionSchema.keywords.getSolrFieldName() + "," + CollectionSchema.url_paths_sxt.getSolrFieldName() + "," + CollectionSchema.last_modified.getSolrFieldName() + "," + CollectionSchema.size_i.getSolrFieldName() + "," +