From a12f693ec9cdfa8828fdfcc2211ebc1125bd753d Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Thu, 9 Aug 2012 18:06:48 +0200 Subject: [PATCH] added two response writer for embedded solr interface: a rss/opensearch writer and an enhanced solr xml writer. The enhanced solr writer has less configuration overhead than the original writer and should by slightly faster. The rss/opensearch writer is at this time slightly incomplete compared with the already existing rss search result form YaCy and also snippets are missing at this time. To test the new interface, open for example: http://localhost:8090/solr/select?wt=rss&q=olympia The wt-code for the new result writers are= wt=rss for opensearch wt=exml for the enhanced solr xml writer. Additionally, the SRU search parameters had been added to the solr interface which can now also be used for a normal solr/xml search. --- htroot/solr/select.java | 53 ++- .../solr/EnhancedXMLResponseWriter.java | 221 +++++++++++++ .../federated/solr/MultipleSolrConnector.java | 22 +- .../solr/OpensearchResponseWriter.java | 301 ++++++++++++++++++ .../federated/solr/RetrySolrConnector.java | 6 +- .../kelondro/data/meta/URIMetadataRow.java | 4 +- .../search/solr/EmbeddedSolrConnector.java | 1 + 7 files changed, 592 insertions(+), 16 deletions(-) create mode 100644 source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java create mode 100644 source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java diff --git a/htroot/solr/select.java b/htroot/solr/select.java index 265e068ac..7c6dffa1d 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -2,34 +2,42 @@ import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.util.HashMap; +import java.util.Map; import javax.servlet.ServletException; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.services.federated.solr.EnhancedXMLResponseWriter; +import net.yacy.cora.services.federated.solr.OpensearchResponseWriter; import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.solr.EmbeddedSolrConnector; import net.yacy.search.solr.SolrServlet; +import org.apache.solr.common.SolrException; import org.apache.solr.common.util.FastWriter; +import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.JSONResponseWriter; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.response.XMLResponseWriter; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +// try +// http://localhost:8090/solr/select?q=*:*&start=0&rows=10&indent=on + public class select { private static SolrServlet solrServlet = new SolrServlet(); - private static final QueryResponseWriter xmlResponseWriter = new XMLResponseWriter(); - private static final QueryResponseWriter jsonResponseWriter = new JSONResponseWriter(); - + private final static Map RESPONSE_WRITER = new HashMap(); static { try {solrServlet.init(null);} catch (ServletException e) {} + RESPONSE_WRITER.putAll(SolrCore.DEFAULT_RESPONSE_WRITERS); + RESPONSE_WRITER.put("exml", new EnhancedXMLResponseWriter()); } /** @@ -55,18 +63,45 @@ public class select { // get the embedded connector EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr(); if (connector == null) return null; + + // check post if (post == null) return null; + + // rename post fields according to result style + if (!post.containsKey("q")) post.put("q", post.remove("query")); // sru patch + if (!post.containsKey("start")) post.put("start", post.remove("startRecord")); // sru patch + if (!post.containsKey("rows")) post.put("rows", post.remove("maximumRecords")); // sru patch + + // check if all required post fields are there if (!post.containsKey("df")) post.put("df", "text_t"); // set default field to all fields - QueryResponseWriter responseWriter = xmlResponseWriter; - if (post.get("wt", "").equals("json")) responseWriter = jsonResponseWriter; + if (!post.containsKey("start")) post.put("start", "0"); // set default start item + if (!post.containsKey("rows")) post.put("rows", "10"); // set default number of search results + + // do the solr request SolrQueryRequest req = connector.request(post.toSolrParams()); - SolrQueryResponse response = connector.query(req); - Exception e = response.getException(); + SolrQueryResponse response = null; + Exception e = null; + try {response = connector.query(req);} catch (SolrException ee) {e = ee;} + if (response != null) e = response.getException(); if (e != null) { Log.logException(e); return null; } + // get a response writer for the result + String wt = post.get("wt", "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml + QueryResponseWriter responseWriter = RESPONSE_WRITER.get(wt); + if (responseWriter == null) { + if (RESPONSE_WRITER.get("rss") == null) { + final String promoteSearchPageGreeting = + (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? env.getConfig( + "network.unit.description", + "") : env.getConfig(SwitchboardConstants.GREETING, ""); + RESPONSE_WRITER.put("rss", new OpensearchResponseWriter(promoteSearchPageGreeting)); + responseWriter = RESPONSE_WRITER.get(wt); //try http://localhost:8090/solr/select?wt=rss&q=olympia + } + } + // write the result directly to the output stream Writer ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset)); try { diff --git a/source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java new file mode 100644 index 000000000..a287445d7 --- /dev/null +++ b/source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java @@ -0,0 +1,221 @@ +/** + * FastXMLResponseWriter + * Copyright 2012 by Michael Peter Christen + * First released 06.08.2012 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.services.federated.solr; + +import java.io.IOException; +import java.io.Writer; +import java.util.Date; +import java.util.List; +import java.util.Set; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.XML; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.DateField; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TextField; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; +import org.apache.solr.search.DocSlice; +import org.apache.solr.search.SolrIndexSearcher; + +public class EnhancedXMLResponseWriter implements QueryResponseWriter { + + private static final char lb = '\n'; + private static final char[] XML_START = "\n\n".toCharArray(); + private static final char[] XML_STOP = "\n\n".toCharArray(); + private static final Set DEFAULT_FIELD_LIST = null; + + public EnhancedXMLResponseWriter() { + super(); + } + + @Override + public String getContentType(final SolrQueryRequest request, final SolrQueryResponse response) { + return CONTENT_TYPE_XML_UTF8; + } + + @Override + public void init(@SuppressWarnings("rawtypes") NamedList n) { + } + + @Override + public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { + writer.write(XML_START); + + assert rsp.getValues().get("responseHeader") != null; + assert rsp.getValues().get("response") != null; + + @SuppressWarnings("unchecked") + SimpleOrderedMap responseHeader = (SimpleOrderedMap) rsp.getResponseHeader(); + DocSlice response = (DocSlice) rsp.getValues().get("response"); + writeProps(writer, "responseHeader", responseHeader); // this.writeVal("responseHeader", responseHeader); + writeDocs(writer, request, response); // this.writeVal("response", response); + + writer.write(XML_STOP); + } + + private static void writeProps(final Writer writer, final String name, final NamedList val) throws IOException { + int sz = val.size(); + if (sz <= 0) startTagClose(writer, "lst", name); else startTagOpen(writer, "lst", name); + Object v; + for (int i = 0; i < sz; i++) { + String n = val.getName(i); + v = val.getVal(i); + if (v instanceof Integer) writeTag(writer, "int", n, ((Integer) v).toString(), false); + else if (v instanceof String) writeTag(writer, "str", n, (String) v, true); + else if (v instanceof NamedList) writeProps(writer, n, (NamedList) v); + } + if (sz > 0) { + writer.write(""); + writer.write(lb); + } + } + + private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response) throws IOException { + boolean includeScore = false; + final int sz = response.size(); + writer.write(""); + return; + } + writer.write('>'); writer.write(lb); + SolrIndexSearcher searcher = request.getSearcher(); + DocIterator iterator = response.iterator(); + includeScore = includeScore && response.hasScores(); + IndexSchema schema = request.getSchema(); + for (int i = 0; i < sz; i++) { + int id = iterator.nextDoc(); + Document doc = searcher.doc(id, DEFAULT_FIELD_LIST); + writeDoc(writer, schema, null, doc, (includeScore ? iterator.score() : 0.0f), includeScore); + } + writer.write(""); + writer.write(lb); + } + + private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name, final Document doc, final float score, final boolean includeScore) throws IOException { + startTagOpen(writer, "doc", name); + + if (includeScore) { + writeTag(writer, "float", "score", Float.toString(score), false); + } + + List fields = doc.getFields(); + int sz = fields.size(); + int fidx1 = 0, fidx2 = 0; + while (fidx1 < sz) { + Fieldable f1 = fields.get(fidx1); + String fieldName = f1.name(); + fidx2 = fidx1 + 1; + while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) { + fidx2++; + } + SchemaField sf = schema.getFieldOrNull(fieldName); + if (sf == null) { + sf = new SchemaField(fieldName, new TextField()); + } + FieldType type = sf.getType(); + if (fidx1 + 1 == fidx2) { + if (sf.multiValued()) { + startTagOpen(writer, "arr", fieldName); + writeField(writer, type, null, f1.stringValue()); //sf.write(this, null, f1); + writer.write(""); + } else { + writeField(writer, type, f1.name(), f1.stringValue()); //sf.write(this, f1.name(), f1); + } + } else { + startTagOpen(writer, "arr", fieldName); + for (int i = fidx1; i < fidx2; i++) { + writeField(writer, type, null, fields.get(i).stringValue()); //sf.write(this, null, (Fieldable)this.tlst.get(i)); + } + writer.write(""); + writer.write(lb); + } + fidx1 = fidx2; + } + writer.write(""); + writer.write(lb); + } + + private static void writeField(final Writer writer, final FieldType type, final String name, final String value) throws IOException { + String typeName = type.getTypeName(); + if (typeName.equals("text_general") || typeName.equals("string") || typeName.equals("text_en_splitting_tight")) { + writeTag(writer, "str", name, value, true); + } else if (typeName.equals("boolean")) { + writeTag(writer, "bool", name, "F".equals(value) ? "false" : "true", true); + } else if (typeName.equals("int")) { + writeTag(writer, "int", name, value, true); + } else if (typeName.equals("long")) { + writeTag(writer, "long", name, value, true); + } else if (typeName.equals("date")) { + writeTag(writer, "date", name, DateField.formatExternal(new Date(Long.parseLong(value))), true); + } else if (typeName.equals("float")) { + writeTag(writer, "float", name, value, true); + } else if (typeName.equals("double")) { + writeTag(writer, "double", name, value, true); + } + } + + private static void writeTag(final Writer writer, final String tag, final String nameAttr, final String val, final boolean escape) throws IOException { + int contentLen = val.length(); + if (contentLen == 0) { + startTagClose(writer, tag, nameAttr); + return; + } + startTagOpen(writer, tag, nameAttr); + if (escape) { + XML.escapeCharData(val, writer); + } else { + writer.write(val, 0, contentLen); + } + writer.write("'); writer.write(lb); + } + + private static void startTagOpen(final Writer writer, final String tag, final String nameAttr) throws IOException { + writer.write('<'); writer.write(tag); + if (nameAttr != null) writeAttr(writer, "name", nameAttr); + writer.write('>'); writer.write(lb); + } + + private static void startTagClose(final Writer writer, final String tag, final String nameAttr) throws IOException { + writer.write('<'); writer.write(tag); + if (nameAttr != null) writeAttr(writer, "name", nameAttr); + writer.write("/>"); writer.write(lb); + } + + private static void writeAttr(final Writer writer, final String nameAttr, final String val) throws IOException { + writer.write(' '); writer.write(nameAttr); writer.write("=\""); XML.escapeAttributeValue(val, writer); writer.write('"'); + } +} diff --git a/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java b/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java index 766500e7c..dee5cc4b0 100644 --- a/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java @@ -1,3 +1,23 @@ +/** + * MultipleSolrConnector + * Copyright 2011 by Michael Peter Christen + * First released 08.11.2011 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + package net.yacy.cora.services.federated.solr; import java.io.IOException; @@ -116,7 +136,7 @@ public class MultipleSolrConnector implements SolrConnector { public SolrDocument get(String id) throws IOException { return this.solr.get(id); } - + @Override public void add(final SolrDoc solrdoc) throws IOException, SolrException { try { diff --git a/source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java b/source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java new file mode 100644 index 000000000..ce203c930 --- /dev/null +++ b/source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java @@ -0,0 +1,301 @@ +/** + * OpensearchResponseWriter + * Copyright 2012 by Michael Peter Christen + * First released 06.08.2012 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.services.federated.solr; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocSlice; +import org.apache.solr.search.SolrIndexSearcher; + +public class OpensearchResponseWriter implements QueryResponseWriter { + + private static final char lb = '\n'; + private static final char[] XML_START = ( + "\n" + + "\n" + + "\n").toCharArray(); + private static final char[] XML_STOP = "\n".toCharArray(); + private static final Set DEFAULT_FIELD_LIST = null; + + private final String title; + + private static class ResHead { + public int offset, rows, numFound; + //public int status, QTime; + //public String df, q, wt; + //public float maxScore; + } + + public OpensearchResponseWriter(String searchPageTitle) { + super(); + this.title = searchPageTitle; + } + + @Override + public String getContentType(final SolrQueryRequest request, final SolrQueryResponse response) { + return CONTENT_TYPE_XML_UTF8; + } + + @Override + public void init(@SuppressWarnings("rawtypes") NamedList n) { + } + + @Override + public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { + writer.write(XML_START); + + assert rsp.getValues().get("responseHeader") != null; + assert rsp.getValues().get("response") != null; + + @SuppressWarnings("unchecked") + SimpleOrderedMap responseHeader = (SimpleOrderedMap) rsp.getResponseHeader(); + DocSlice response = (DocSlice) rsp.getValues().get("response"); + + // parse response header + ResHead resHead = new ResHead(); + NamedList val0 = (NamedList) responseHeader.get("params"); + resHead.rows = Integer.parseInt((String) val0.get("rows")); + resHead.offset = response.offset(); // equal to 'start' + resHead.numFound = response.matches(); + //resHead.df = (String) val0.get("df"); + //resHead.q = (String) val0.get("q"); + //resHead.wt = (String) val0.get("wt"); + //resHead.status = (Integer) responseHeader.get("status"); + //resHead.QTime = (Integer) responseHeader.get("QTime"); + //resHead.maxScore = response.maxScore(); + + // write header + startTagOpen(writer, "channel"); + solitaireTag(writer, "opensearch:totalResults", Integer.toString(resHead.numFound)); + solitaireTag(writer, "opensearch:startIndex", Integer.toString(resHead.offset)); + solitaireTag(writer, "opensearch:itemsPerPage", Integer.toString(resHead.rows)); + solitaireTag(writer, "title", this.title); + //solitaireTag(writer, "description", ""); + //solitaireTag(writer, "link", ""); + //solitaireTag(writer, "image", ""); + + // parse body + final int responseCount = response.size(); + SolrIndexSearcher searcher = request.getSearcher(); + DocIterator iterator = response.iterator(); + for (int i = 0; i < responseCount; i++) { + startTagOpen(writer, "item"); + int id = iterator.nextDoc(); + Document doc = searcher.doc(id, DEFAULT_FIELD_LIST); + List fields = doc.getFields(); + int fieldc = fields.size(); + List texts = new ArrayList(); + String description = ""; + for (int j = 0; j < fieldc; j++) { + Fieldable f1 = fields.get(j); + String fieldName = f1.name(); + if ("id".equals(fieldName)) {writer.write(""); writer.write(f1.stringValue()); writer.write(""); writer.write(lb); continue;} + if ("sku".equals(fieldName)) {solitaireTag(writer, "link", f1.stringValue()); continue;} + if ("title".equals(fieldName)) {solitaireTag(writer, "title", f1.stringValue()); texts.add(f1.stringValue()); continue;} + if ("last_modified".equals(fieldName)) {solitaireTag(writer, "pubDate", f1.stringValue()); continue;} + if ("".equals(fieldName)) {solitaireTag(writer, "dc:publisher", f1.stringValue()); continue;} + if ("".equals(fieldName)) {solitaireTag(writer, "dc:creator", f1.stringValue()); continue;} + if ("description".equals(fieldName)) {description = f1.stringValue(); solitaireTag(writer, "dc:subject", description); texts.add(description); continue;} + if ("text_t".equals(fieldName)) {texts.add(f1.stringValue()); continue;} + if ("h1_txt".equals(fieldName) || "h2_txt".equals(fieldName) || "h3_txt".equals(fieldName) || + "h4_txt".equals(fieldName) || "h5_txt".equals(fieldName) || "h6_txt".equals(fieldName)) {texts.add(f1.stringValue()); continue;} + } + // compute snippet from texts + solitaireTag(writer, "description", description); + closeTag(writer, "item"); + } + + closeTag(writer, "channel"); + writer.write(XML_STOP); + } + + public static void startTagOpen(final Writer writer, final String tag) throws IOException { + writer.write('<'); writer.write(tag); writer.write('>'); writer.write(lb); + } + + public static void closeTag(final Writer writer, final String tag) throws IOException { + writer.write("'); writer.write(lb); + } + + public static void solitaireTag(final Writer writer, final String tagname, String value) throws IOException { + writer.write("<"); writer.write(tagname); writer.write('>'); + writer.write(value); + writer.write("'); writer.write(lb); + } + +} + +/* + + + #[promoteSearchPageGreeting]# + Search for #[rss_query]# + #[searchBaseURL]#?query=#[rss_queryenc]#&resource=#[resource]#&contentdom=#[contentdom]#&verify=#[verify]# + + #[rssYacyImageURL]# + Search for #[rss_query]# + #[searchBaseURL]#?query=#[rss_queryenc]#&resource=#[resource]#&contentdom=#[contentdom]#&verify=#[verify]# + + #[num-results_totalcount]# + #[num-results_offset]# + #[num-results_itemsPerPage]# + + + + + + +#[title-xml]# +#[link]# +#[description-xml]# +#[date822]# + + + +#[size]# +#[sizename]# +#[host]# +#[path]# +#[file]# +#[urlhash]# +#(loc)#::#[lat]##[lon]##(/loc)# +:: +#(item)#:: +#[name]# +#[source-xml]# + + +#[urlhash]# +#[sourcedom]# + + + + + + + + + + +#(nav-domains)#:: + +#{element}# + +#{/element}# + +#(/nav-domains)# + +#(nav-namespace)#:: + +#{element}# + +#{/element}# + +#(/nav-namespace)# + +#(nav-authors)#:: + +#{element}# + +#{/element}# + +#(/nav-authors)# + +#(nav-filetype)#:: + +#{element}# + +#{/element}# + +#(/nav-filetype)# + +#(nav-protocol)#:: + +#{element}# + +#{/element}# + +#(/nav-protocol)# + +#(nav-topics)#:: + +#{element}# + +#{/element}# + +#(/nav-topics)# + +#{nav-vocabulary}# + +#{element}# + +#{/element}# + +#{/nav-vocabulary}# + + + +#[num-results_totalcount]# + + */ \ No newline at end of file diff --git a/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java b/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java index f9acb44d8..27418e14b 100644 --- a/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java @@ -3,10 +3,6 @@ * Copyright 2011 by Michael Peter Christen * First released 08.11.2011 at http://yacy.net * - * $LastChangedDate: 2011-04-14 22:05:04 +0200 (Do, 14 Apr 2011) $ - * $LastChangedRevision: 7654 $ - * $LastChangedBy: orbiter $ - * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either @@ -135,7 +131,7 @@ public class RetrySolrConnector implements SolrConnector { if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); return null; } - + @Override public void add(final SolrDoc solrdoc) throws IOException, SolrException { final long t = System.currentTimeMillis() + this.retryMaxTime; diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java index 572af984c..48fbaea3c 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java @@ -387,6 +387,7 @@ public class URIMetadataRow implements URIMetadata { } private String hostHash = null; + @Override public String hosthash() { if (this.hostHash != null) return this.hostHash; this.hostHash = ASCII.String(this.entry.getPrimaryKeyBytes(), 6, 6); @@ -470,6 +471,7 @@ public class URIMetadataRow implements URIMetadata { return decodeDate(col_fresh); } + @Override public byte[] referrerHash() { // return the creator's hash or null if there is none // FIXME: There seem to be some malformed entries in the databasees like "null\0\0\0\0\0\0\0\0" @@ -497,7 +499,7 @@ public class URIMetadataRow implements URIMetadata { @Override public byte[] language() { byte[] b = this.entry.getColBytes(col_lang, true); - if (b == null || b[0] == (byte)'[') { + if ((b == null || b[0] == (byte)'[') && this.metadata().url != null) { String tld = this.metadata().url.getTLD(); if (tld.length() < 2 || tld.length() > 2) return ASCII.getBytes("en"); return ASCII.getBytes(tld); diff --git a/source/net/yacy/search/solr/EmbeddedSolrConnector.java b/source/net/yacy/search/solr/EmbeddedSolrConnector.java index d17e4c9c4..a41049c4e 100644 --- a/source/net/yacy/search/solr/EmbeddedSolrConnector.java +++ b/source/net/yacy/search/solr/EmbeddedSolrConnector.java @@ -134,6 +134,7 @@ public class EmbeddedSolrConnector extends AbstractSolrConnector implements Solr SolrQueryResponse rsp = new SolrQueryResponse(); NamedList responseHeader = new SimpleOrderedMap(); + responseHeader.add("params", req.getOriginalParams().toNamedList()); rsp.add("responseHeader", responseHeader); SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));