From f4267ed247361916eb937ca7121d1d2f8e101015 Mon Sep 17 00:00:00 2001 From: luccioman Date: Sat, 28 Jul 2018 11:03:31 +0200 Subject: [PATCH] Made Solr OpenSearch RSS writer compatible with external Solr index Worked previously only with responses from YaCy embedded Solr, now able to render the response when YaCy is configured to use an external Solr index. --- .../OpensearchResponseWriter.java | 489 +++++++++++++----- .../responsewriter/YJsonResponseWriter.java | 8 +- .../yacy/http/servlets/SolrSelectServlet.java | 3 + 3 files changed, 377 insertions(+), 123 deletions(-) diff --git a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java index 747c19b55..db1d72ef5 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java @@ -33,20 +33,14 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.Map.Entry; import java.util.regex.Pattern; -import net.yacy.cora.document.feed.RSSMessage; -import net.yacy.cora.document.id.MultiProtocolURL; -import net.yacy.cora.lod.vocabulary.DublinCore; -import net.yacy.cora.lod.vocabulary.Geo; -import net.yacy.cora.lod.vocabulary.YaCyMetadata; -import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.crawler.retrieval.Response; -import net.yacy.search.schema.CollectionConfiguration; -import net.yacy.search.schema.CollectionSchema; - import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexableField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.XML; @@ -58,7 +52,21 @@ import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; -public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter { +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.id.MultiProtocolURL; +import net.yacy.cora.lod.vocabulary.DublinCore; +import net.yacy.cora.lod.vocabulary.Geo; +import net.yacy.cora.lod.vocabulary.YaCyMetadata; +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.crawler.retrieval.Response; +import net.yacy.search.schema.CollectionConfiguration; +import net.yacy.search.schema.CollectionSchema; + +/** + * Solr response writer producing an OpenSearch representation in RSS 2.0 format. + * @see Example of OpenSearch response elements in RSS 2.0 + */ +public class OpensearchResponseWriter implements QueryResponseWriter, SolrjResponseWriter { // define a list of simple YaCySchema -> RSS Token matchings private static final Map field2tag = new HashMap(); @@ -81,7 +89,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo private String title; public static class ResHead { - public int offset, rows, numFound; + public long offset, rows, numFound; //public int status, QTime; //public String df, q, wt; //public float maxScore; @@ -107,33 +115,139 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo @Override public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { - NamedList values = rsp.getValues(); + final NamedList values = rsp.getValues(); + + final Object responseObj = rsp.getResponse(); assert values.get("responseHeader") != null; assert values.get("response") != null; - SimpleOrderedMap responseHeader = (SimpleOrderedMap) rsp.getResponseHeader(); - DocList response = ((ResultContext) values.get("response")).getDocList(); + final NamedList responseHeader = rsp.getResponseHeader(); + + write(writer, request, values, responseHeader, responseObj); + } + + @Override + public void write(Writer writer, SolrQueryRequest request, String coreName, QueryResponse rsp) throws IOException { + + final NamedList values = rsp.getResponse(); + + final NamedList responseHeader = rsp.getResponseHeader(); + + final SolrDocumentList documents = rsp.getResults(); + + write(writer, request, values, responseHeader, documents); + } + + /** + * Append to the writer the OpenSearch RSS representation of the Solr results. + * @param writer an open output writer. Must not be null. + * @param request the initial Solr request. Must not be null. + * @param values the response values. Must not be null. + * @param rsp the Solr response header. + * @throws IOException when a write error occurred + */ + private void write(final Writer writer, final SolrQueryRequest request, final NamedList values, + final NamedList responseHeader, final Object responseObj) throws IOException { + final ResHead resHead = new ResHead(); + final int responseCount; + @SuppressWarnings("unchecked") SimpleOrderedMap facetCounts = (SimpleOrderedMap) values.get("facet_counts"); @SuppressWarnings("unchecked") SimpleOrderedMap facetFields = facetCounts == null || facetCounts.size() == 0 ? null : (SimpleOrderedMap) facetCounts.get("facet_fields"); @SuppressWarnings("unchecked") SimpleOrderedMap highlighting = (SimpleOrderedMap) values.get("highlighting"); - Map> snippets = highlighting(highlighting); + final Map> snippets = highlighting(highlighting); + + if(responseObj instanceof ResultContext){ + /* Regular response object */ + final DocList documents = ((ResultContext)responseObj).getDocList(); + + resHead.offset = documents.offset(); // equal to 'start' Solr param + resHead.numFound = documents.matches(); + + responseCount = documents.size(); + + writeHeader(writer, responseHeader, resHead); + + writeDocs(writer, documents, request, responseCount, snippets); + } else if(responseObj instanceof SolrDocumentList) { + /* + * The response object can be a SolrDocumentList when the response is partial, + * for example when the allowed processing time has been exceeded + */ + final SolrDocumentList documents = ((SolrDocumentList)responseObj); + + resHead.offset = documents.getStart(); // equal to 'start' Solr param + resHead.numFound = documents.getNumFound(); + + responseCount = documents.size(); + + writeHeader(writer, responseHeader, resHead); + + writeDocs(writer, documents, responseCount, snippets); + } else { + throw new IOException("Unable to process Solr response format"); + } - // parse response header - ResHead resHead = new ResHead(); - NamedList val0 = (NamedList) responseHeader.get("params"); + openTag(writer, "yacy:navigation"); + + // the facets can be created with the options &facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt + @SuppressWarnings("unchecked") + NamedList domains = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.host_s.getSolrFieldName()); + @SuppressWarnings("unchecked") + NamedList filetypes = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.url_file_ext_s.getSolrFieldName()); + @SuppressWarnings("unchecked") + NamedList protocols = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.url_protocol_s.getSolrFieldName()); + @SuppressWarnings("unchecked") + NamedList authors = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.author_sxt.getSolrFieldName()); + @SuppressWarnings("unchecked") + NamedList collections = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.collection_sxt.getSolrFieldName()); + + if (domains != null) { + openTag(writer, "yacy:facet name=\"domains\" displayname=\"Domains\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: domains) facetEntry(writer, "site", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + if (filetypes != null) { + openTag(writer, "yacy:facet name=\"filetypes\" displayname=\"Filetypes\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: filetypes) facetEntry(writer, "filetype", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + if (protocols != null) { + openTag(writer, "yacy:facet name=\"protocols\" displayname=\"Protocols\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: protocols) facetEntry(writer, "protocol", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + if (authors != null) { + openTag(writer, "yacy:facet name=\"authors\" displayname=\"Authors\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: authors) facetEntry(writer, "author", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + if (collections != null) { + openTag(writer, "yacy:facet name=\"collections\" displayname=\"Collections\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); + for (Map.Entry entry: collections) facetEntry(writer, "collection", entry.getKey(), Integer.toString(entry.getValue())); + closeTag(writer, "yacy:facet"); + } + closeTag(writer, "yacy:navigation"); + + closeTag(writer, "channel"); + writer.write("\n".toCharArray()); + } + + /** + * Append to the writer the header of the OpenSearch RSS representation. + * @param writer an open output writer. Must not be null. + * @param responseHeader the Solr response header. Must not be null. + * @param resHead the calculated results head. Must not be null. + * @throws IOException when an unexpected error occurred while writing + */ + private void writeHeader(final Writer writer, final NamedList responseHeader, final ResHead resHead) + throws IOException { + // parse response header + final NamedList val0 = (NamedList) responseHeader.get("params"); resHead.rows = Integer.parseInt((String) val0.get("rows")); - resHead.offset = response.offset(); // equal to 'start' - resHead.numFound = response.matches(); - //resHead.df = (String) val0.get("df"); - //resHead.q = (String) val0.get("q"); - //resHead.wt = (String) val0.get("wt"); - //resHead.status = (Integer) responseHeader.get("status"); - //resHead.QTime = (Integer) responseHeader.get("QTime"); - //resHead.maxScore = response.maxScore(); // write header writer.write(( @@ -148,21 +262,170 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo " xmlns:geo=\"" + Geo.NAMESPACE + "\"\n" + ">\n").toCharArray()); openTag(writer, "channel"); - solitaireTag(writer, "opensearch:totalResults", Integer.toString(resHead.numFound)); - solitaireTag(writer, "opensearch:startIndex", Integer.toString(resHead.offset)); - solitaireTag(writer, "opensearch:itemsPerPage", Integer.toString(resHead.rows)); + solitaireTag(writer, "opensearch:totalResults", Long.toString(resHead.numFound)); + solitaireTag(writer, "opensearch:startIndex", Long.toString(resHead.offset)); + solitaireTag(writer, "opensearch:itemsPerPage", Long.toString(resHead.rows)); solitaireTag(writer, RSSMessage.Token.title.name(), this.title); writer.write(""); solitaireTag(writer, "description", "Search Result"); - //solitaireTag(writer, "link", ""); - //solitaireTag(writer, "image", ""); + } + + /** + * Append to the writer the OpenSearch RSS representation of Solr documents. + * + * @param writer an open output writer. Must not be null. + * @param documents the documents to render. Must not be null. + * @param responseCount the number of documents to process + * @param snippets snippets Solr computed text snippets (highlighting). + * @throws IOException when an unexpected error occurred while writing + */ + private void writeDocs(final Writer writer, final SolrDocumentList documents, final int responseCount, + final Map> snippets) throws IOException { + // parse body + String urlhash = null; + MultiProtocolURL url = null; + final Iterator iterator = documents.iterator(); + for (int i = 0; i < responseCount; i++) { + openTag(writer, "item"); + SolrDocument doc = iterator.next(); + List texts = new ArrayList(); + List descriptions = new ArrayList(); + String docTitle = ""; + List images_protocol_obj = new ArrayList<>(); + List images_stub = new ArrayList<>(); + for (final Entry fieldEntry : doc) { + final String fieldName = fieldEntry.getKey(); + final Object value = fieldEntry.getValue(); + + if(value == null) { + continue; + } + + // apply generic matching rule + String stag = field2tag.get(fieldName); + if (stag != null) { + solitaireTag(writer, stag, value.toString()); + continue; + } + + // take apart the url + if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) { + url = writeLink(writer, value.toString()); + continue; + } + + // if the rule is not generic, use the specific here + if (CollectionSchema.id.getSolrFieldName().equals(fieldName)) { + urlhash = value.toString(); + solitaireTag(writer, RSSMessage.Token.guid.name(), urlhash, "isPermaLink=\"false\""); + continue; + } + if (CollectionSchema.title.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + /* Handle multivalued field */ + for(final Object valueItem : (Iterable)value) { + docTitle = valueItem.toString(); + texts.add(docTitle); + } + } else { + docTitle = value.toString(); + texts.add(docTitle); + } + continue; + } + if (CollectionSchema.last_modified.getSolrFieldName().equals(fieldName) && value instanceof Date) { + solitaireTag(writer, RSSMessage.Token.pubDate.name(), HeaderFramework.formatRFC1123((Date)value)); + continue; + } + if (CollectionSchema.description_txt.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + /* Handle multivalued field */ + for(final Object valueItem : (Iterable)value) { + final String description = valueItem.toString(); + descriptions.add(description); + texts.add(description); + solitaireTag(writer, DublinCore.Description.getURIref(), description); + } + } else { + final String description = value.toString(); + descriptions.add(description); + texts.add(description); + solitaireTag(writer, DublinCore.Description.getURIref(), description); + } + + continue; + } + if (CollectionSchema.text_t.getSolrFieldName().equals(fieldName)) { + texts.add(value.toString()); + continue; + } + if (CollectionSchema.size_i.getSolrFieldName().equals(fieldName) && value instanceof Integer) { + int size = ((Integer)value).intValue(); + solitaireTag(writer, YaCyMetadata.size.getURIref(), Integer.toString(size)); + solitaireTag(writer, YaCyMetadata.sizename.getURIref(), RSSMessage.sizename(size)); + continue; + } + if (CollectionSchema.h1_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h2_txt.getSolrFieldName().equals(fieldName) || + CollectionSchema.h3_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h4_txt.getSolrFieldName().equals(fieldName) || + CollectionSchema.h5_txt.getSolrFieldName().equals(fieldName) || CollectionSchema.h6_txt.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + // because these are multi-valued fields, there can be several of each + for(final Object valueItem : (Iterable)value) { + texts.add(valueItem.toString()); + } + } else { + texts.add(value.toString()); + } + continue; + } + if (CollectionSchema.images_protocol_sxt.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + /* Handle multivalued field */ + for(final Object valueItem : (Iterable)value) { + images_protocol_obj.add(valueItem.toString()); + } + } else { + images_protocol_obj.add(value.toString()); + } + continue; + } + if (CollectionSchema.images_urlstub_sxt.getSolrFieldName().equals(fieldName)) { + if(value instanceof Iterable) { + /* Handle multivalued field */ + for(final Object valueItem : (Iterable)value) { + images_stub.add(valueItem.toString()); + } + } else { + images_stub.add(value.toString()); + } + continue; + } + } + + final Object keywordsObj = doc.get(CollectionSchema.keywords.getSolrFieldName()); + final String keywords = (keywordsObj instanceof String) ? (String)keywordsObj : null; + + writeDocEnd(writer, snippets, urlhash, url, keywords, texts, descriptions, docTitle, images_protocol_obj, + images_stub); + } + } - // parse body - final int responseCount = response.size(); + /** + * Append to the writer the OpenSearch RSS representation of Solr documents. + * + * @param writer an open output writer. Must not be null. + * @param documents the documents to render. Must not be null. + * @param responseCount the number of documents to process + * @param snippets Solr computed text snippets (highlighting). + * @throws IOException when an unexpected error occurred while writing + */ + private void writeDocs(final Writer writer, final DocList documents, final SolrQueryRequest request, final int responseCount, + final Map> snippets) throws IOException { + // parse body SolrIndexSearcher searcher = request.getSearcher(); - DocIterator iterator = response.iterator(); String urlhash = null; MultiProtocolURL url = null; + final DocIterator iterator = documents.iterator(); for (int i = 0; i < responseCount; i++) { openTag(writer, "item"); int id = iterator.nextDoc(); @@ -171,7 +434,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo int fieldc = fields.size(); List texts = new ArrayList(); List descriptions = new ArrayList(); - String title = ""; + String docTitle = ""; List images_protocol_obj = new ArrayList<>(); List images_stub = new ArrayList<>(); for (int j = 0; j < fieldc; j++) { @@ -187,14 +450,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo // take apart the url if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) { - String u = value.stringValue(); - solitaireTag(writer, RSSMessage.Token.link.name(), u); - try { - url = new MultiProtocolURL(u); - solitaireTag(writer, YaCyMetadata.host.getURIref(), url.getHost()); - solitaireTag(writer, YaCyMetadata.path.getURIref(), url.getPath()); - solitaireTag(writer, YaCyMetadata.file.getURIref(), url.getFileName()); - } catch (final MalformedURLException e) {} + url = writeLink(writer, value.stringValue()); continue; } @@ -205,8 +461,8 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo continue; } if (CollectionSchema.title.getSolrFieldName().equals(fieldName)) { - title = value.stringValue(); - texts.add(title); + docTitle = value.stringValue(); + texts.add(docTitle); continue; } if (CollectionSchema.last_modified.getSolrFieldName().equals(fieldName)) { @@ -248,82 +504,77 @@ public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSo } } - if (Math.min(images_protocol_obj.size(), images_stub.size()) > 0) { - List images_protocol = CollectionConfiguration.indexedList2protocolList(images_protocol_obj, images_stub.size()); - String imageurl = images_protocol.get(0) + "://" + images_stub.get(0); - writer.write("\n"); - } else { - if (url != null && Response.docTypeExt(MultiProtocolURL.getFileExtension(url.getFile()).toLowerCase(Locale.ROOT)) == Response.DT_IMAGE) { - writer.write("\n"); - } - } - - // compute snippet from texts - solitaireTag(writer, RSSMessage.Token.title.name(), title.length() == 0 ? (texts.size() == 0 ? "" : texts.get(0)) : title); - LinkedHashSet snippet = urlhash == null ? null : snippets.get(urlhash); - String tagname = RSSMessage.Token.description.name(); - if (snippet == null || snippet.size() == 0) { - writer.write("<"); writer.write(tagname); writer.write('>'); - for (String d: descriptions) { - XML.escapeCharData(d, writer); - } - writer.write("\n"); - } else { - removeSubsumedTitle(snippet, title); - solitaireTag(writer, tagname, getLargestSnippet(snippet)); // snippet may be size=0 - } - - solitaireTag(writer, DublinCore.Subject.getURIref(), doc.get(CollectionSchema.keywords.getSolrFieldName())); + final Object keywordsObj = doc.get(CollectionSchema.keywords.getSolrFieldName()); + final String keywords = (keywordsObj instanceof String) ? (String)keywordsObj : null; - closeTag(writer, "item"); + writeDocEnd(writer, snippets, urlhash, url, keywords, texts, descriptions, docTitle, images_protocol_obj, + images_stub); } + } + + /** + * Append the Solr document URL as a RSS link to the writer + * @param writer + * @param sku + * @return a MultiProtocolURL instance or null + * @throws IOException + */ + private MultiProtocolURL writeLink(final Writer writer, final String sku) + throws IOException { + solitaireTag(writer, RSSMessage.Token.link.name(), sku); + MultiProtocolURL url; + try { + url = new MultiProtocolURL(sku); + solitaireTag(writer, YaCyMetadata.host.getURIref(), url.getHost()); + solitaireTag(writer, YaCyMetadata.path.getURIref(), url.getPath()); + solitaireTag(writer, YaCyMetadata.file.getURIref(), url.getFileName()); + } catch (final MalformedURLException e) { + url = null; + } + return url; + } - openTag(writer, "yacy:navigation"); - - // the facets can be created with the options &facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt - @SuppressWarnings("unchecked") - NamedList domains = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.host_s.getSolrFieldName()); - @SuppressWarnings("unchecked") - NamedList filetypes = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.url_file_ext_s.getSolrFieldName()); - @SuppressWarnings("unchecked") - NamedList protocols = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.url_protocol_s.getSolrFieldName()); - @SuppressWarnings("unchecked") - NamedList authors = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.author_sxt.getSolrFieldName()); - @SuppressWarnings("unchecked") - NamedList collections = facetFields == null ? null : (NamedList) facetFields.get(CollectionSchema.collection_sxt.getSolrFieldName()); - - if (domains != null) { - openTag(writer, "yacy:facet name=\"domains\" displayname=\"Domains\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: domains) facetEntry(writer, "site", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - if (filetypes != null) { - openTag(writer, "yacy:facet name=\"filetypes\" displayname=\"Filetypes\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: filetypes) facetEntry(writer, "filetype", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - if (protocols != null) { - openTag(writer, "yacy:facet name=\"protocols\" displayname=\"Protocols\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: protocols) facetEntry(writer, "protocol", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - if (authors != null) { - openTag(writer, "yacy:facet name=\"authors\" displayname=\"Authors\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: authors) facetEntry(writer, "author", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - if (collections != null) { - openTag(writer, "yacy:facet name=\"collections\" displayname=\"Collections\" type=\"String\" min=\"0\" max=\"0\" mean=\"0\""); - for (Map.Entry entry: collections) facetEntry(writer, "collection", entry.getKey(), Integer.toString(entry.getValue())); - closeTag(writer, "yacy:facet"); - } - closeTag(writer, "yacy:navigation"); - - closeTag(writer, "channel"); - writer.write("\n".toCharArray()); - } + /** + * Append to the writer the end of the RSS OpenSearch representation of the Solr + * document. + */ + private void writeDocEnd(final Writer writer, final Map> snippets, final String urlhash, + final MultiProtocolURL url, final String keywords, final List texts, final List descriptions, final String docTitle, + final List imagesProtocolObjs, final List imagesStubs) throws IOException { + if (Math.min(imagesProtocolObjs.size(), imagesStubs.size()) > 0) { + List imagesProtocols = CollectionConfiguration.indexedList2protocolList(imagesProtocolObjs, imagesStubs.size()); + String imageurl = imagesProtocols.get(0) + "://" + imagesStubs.get(0); + writer.write("\n"); + } else { + if (url != null && Response.docTypeExt(MultiProtocolURL.getFileExtension(url.getFile()).toLowerCase(Locale.ROOT)) == Response.DT_IMAGE) { + writer.write("\n"); + } + } + + // compute snippet from texts + solitaireTag(writer, RSSMessage.Token.title.name(), docTitle.length() == 0 ? (texts.size() == 0 ? "" : texts.get(0)) : docTitle); + LinkedHashSet snippet = urlhash == null ? null : snippets.get(urlhash); + String tagname = RSSMessage.Token.description.name(); + if (snippet == null || snippet.size() == 0) { + writer.write("<"); writer.write(tagname); writer.write('>'); + for (String d: descriptions) { + XML.escapeCharData(d, writer); + } + writer.write("\n"); + } else { + removeSubsumedTitle(snippet, docTitle); + solitaireTag(writer, tagname, getLargestSnippet(snippet)); // snippet may be size=0 + } + + if(keywords != null) { + solitaireTag(writer, DublinCore.Subject.getURIref(), keywords); + } + + closeTag(writer, "item"); + } + /** * produce snippets from solr (they call that 'highlighting') diff --git a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java index 3380f537b..499b12ed7 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java @@ -109,7 +109,7 @@ public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrRes // parse response header ResHead resHead = new ResHead(); NamedList val0 = (NamedList) responseHeader.get("params"); - resHead.rows = Integer.parseInt((String) val0.get("rows")); + resHead.rows = Long.parseLong((String) val0.get("rows")); resHead.offset = response.offset(); // equal to 'start' resHead.numFound = response.matches(); @@ -121,9 +121,9 @@ public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrRes // write header writer.write(("{\"channels\": [{\n").toCharArray()); - solitaireTag(writer, "totalResults", Integer.toString(resHead.numFound)); - solitaireTag(writer, "startIndex", Integer.toString(resHead.offset)); - solitaireTag(writer, "itemsPerPage", Integer.toString(resHead.rows)); + solitaireTag(writer, "totalResults", Long.toString(resHead.numFound)); + solitaireTag(writer, "startIndex", Long.toString(resHead.offset)); + solitaireTag(writer, "itemsPerPage", Long.toString(resHead.rows)); solitaireTag(writer, "title", this.title); solitaireTag(writer, "description", "Search Result"); writer.write("\"items\": [\n".toCharArray()); diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index d5944f40f..f670a91ba 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -225,6 +225,9 @@ public class SolrSelectServlet extends HttpServlet { CollectionSchema.title.getSolrFieldName() + "," + CollectionSchema.description_txt.getSolrFieldName() + "," + CollectionSchema.id.getSolrFieldName() + "," + + CollectionSchema.author.getSolrFieldName() + "," + + CollectionSchema.publisher_t.getSolrFieldName() + "," + + CollectionSchema.keywords.getSolrFieldName() + "," + CollectionSchema.url_paths_sxt.getSolrFieldName() + "," + CollectionSchema.last_modified.getSolrFieldName() + "," + CollectionSchema.size_i.getSolrFieldName() + "," +