From b2b480fff2f95722590ca8da864628003b119609 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 10 Aug 2012 09:48:15 +0200 Subject: [PATCH] more abstraction of the YaCySchema -> Opensearch matching process --- htroot/solr/select.java | 8 +-- .../solr/EnhancedXMLResponseWriter.java | 8 +-- .../solr/OpensearchResponseWriter.java | 67 +++++++++++-------- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/htroot/solr/select.java b/htroot/solr/select.java index dec9eab10..87228cf49 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -63,10 +63,6 @@ public class select { final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header); if (!searchAllowed) return null; - // get the embedded connector - EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr(); - if (connector == null) return null; - // check post if (post == null) return null; @@ -80,6 +76,10 @@ public class select { if (!post.containsKey(CommonParams.START)) post.put(CommonParams.START, "0"); // set default start item if (!post.containsKey(CommonParams.ROWS)) post.put(CommonParams.ROWS, "10"); // set default number of search results + // get the embedded connector + EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr(); + if (connector == null) return null; + // do the solr request SolrQueryRequest req = connector.request(post.toSolrParams()); SolrQueryResponse response = null; diff --git a/source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java index 6cbc65ad9..bec20cc44 100644 --- a/source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java +++ b/source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java @@ -136,8 +136,8 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { int sz = fields.size(); int fidx1 = 0, fidx2 = 0; while (fidx1 < sz) { - Fieldable f1 = fields.get(fidx1); - String fieldName = f1.name(); + Fieldable value = fields.get(fidx1); + String fieldName = value.name(); fidx2 = fidx1 + 1; while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) { fidx2++; @@ -150,10 +150,10 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { if (fidx1 + 1 == fidx2) { if (sf.multiValued()) { startTagOpen(writer, "arr", fieldName); - writeField(writer, type, null, f1.stringValue()); //sf.write(this, null, f1); + writeField(writer, type, null, value.stringValue()); //sf.write(this, null, f1); writer.write(""); } else { - writeField(writer, type, f1.name(), f1.stringValue()); //sf.write(this, f1.name(), f1); + writeField(writer, type, value.name(), value.stringValue()); //sf.write(this, f1.name(), f1); } } else { startTagOpen(writer, "arr", fieldName); diff --git a/source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java b/source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java index 1bffef4b0..a82bc3d65 100644 --- a/source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java +++ b/source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java @@ -23,7 +23,10 @@ package net.yacy.cora.services.federated.solr; import java.io.IOException; import java.io.Writer; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import net.yacy.cora.document.RSSMessage; @@ -56,7 +59,24 @@ public class OpensearchResponseWriter implements QueryResponseWriter { " xmlns:geo=\"http://www.w3.org/2003/01/geo/wgs84_pos#\"\n" + ">\n").toCharArray(); private static final char[] XML_STOP = "\n".toCharArray(); - private static final Set DEFAULT_FIELD_LIST = null; + + // define a list of simple YaCySchema -> RSS Token matchings + private static final Map field2tag = new HashMap(); + + // pre-select a set of YaCy schema fields for the solr searcher which should cause a better caching + private static final YaCySchema[] extrafields = new YaCySchema[]{ + YaCySchema.id, YaCySchema.title, YaCySchema.description, YaCySchema.text_t, + YaCySchema.h1_txt, YaCySchema.h2_txt, YaCySchema.h3_txt, YaCySchema.h4_txt, YaCySchema.h5_txt, YaCySchema.h6_txt, + }; + private static final Set SOLR_FIELDS = new HashSet(); + static { + field2tag.put(YaCySchema.sku.name(), RSSMessage.Token.link.name()); + field2tag.put(YaCySchema.last_modified.name(), RSSMessage.Token.pubDate.name()); + field2tag.put(YaCySchema.publisher_t.name(), DublinCore.Publisher.getURIref()); + field2tag.put(YaCySchema.author.name(), DublinCore.Creator.getURIref()); + SOLR_FIELDS.addAll(field2tag.keySet()); + for (YaCySchema field: extrafields) SOLR_FIELDS.add(field.name()); + } private String title; @@ -86,8 +106,6 @@ public class OpensearchResponseWriter implements QueryResponseWriter { @Override public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { - writer.write(XML_START); - assert rsp.getValues().get("responseHeader") != null; assert rsp.getValues().get("response") != null; @@ -109,6 +127,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter { //resHead.maxScore = response.maxScore(); // write header + writer.write(XML_START); openTag(writer, "channel"); solitaireTag(writer, "opensearch:totalResults", Integer.toString(resHead.numFound)); solitaireTag(writer, "opensearch:startIndex", Integer.toString(resHead.offset)); @@ -125,53 +144,47 @@ public class OpensearchResponseWriter implements QueryResponseWriter { for (int i = 0; i < responseCount; i++) { openTag(writer, "item"); int id = iterator.nextDoc(); - Document doc = searcher.doc(id, DEFAULT_FIELD_LIST); + Document doc = searcher.doc(id, SOLR_FIELDS); List fields = doc.getFields(); int fieldc = fields.size(); List texts = new ArrayList(); String description = ""; for (int j = 0; j < fieldc; j++) { - Fieldable f1 = fields.get(j); - String fieldName = f1.name(); - if (YaCySchema.id.name().equals(fieldName)) { - solitaireTag(writer, RSSMessage.Token.guid.name(), f1.stringValue(), "isPermaLink=\"false\""); + Fieldable value = fields.get(j); + String fieldName = value.name(); + + // apply generic matching rule + String stag = field2tag.get(fieldName); + if (stag != null) { + solitaireTag(writer, stag, value.stringValue()); continue; } - if (YaCySchema.sku.name().equals(fieldName)) { - solitaireTag(writer, RSSMessage.Token.link.name(), f1.stringValue()); + + // if the rule is not generic, use the specific here + if (YaCySchema.id.name().equals(fieldName)) { + solitaireTag(writer, RSSMessage.Token.guid.name(), value.stringValue(), "isPermaLink=\"false\""); continue; } if (YaCySchema.title.name().equals(fieldName)) { - solitaireTag(writer, RSSMessage.Token.title.name(), f1.stringValue()); - texts.add(f1.stringValue()); - continue; - } - if (YaCySchema.last_modified.name().equals(fieldName)) { - solitaireTag(writer, RSSMessage.Token.pubDate.name(), f1.stringValue()); - continue; - } - if (YaCySchema.publisher_t.name().equals(fieldName)) { - solitaireTag(writer, DublinCore.Publisher.getURIref(), f1.stringValue()); - continue; - } - if (YaCySchema.author.name().equals(fieldName)) { - solitaireTag(writer, DublinCore.Creator.getURIref(), f1.stringValue()); + solitaireTag(writer, RSSMessage.Token.title.name(), value.stringValue()); + texts.add(value.stringValue()); continue; } if (YaCySchema.description.name().equals(fieldName)) { - description = f1.stringValue(); + description = value.stringValue(); solitaireTag(writer, DublinCore.Description.getURIref(), description); texts.add(description); continue; } if (YaCySchema.text_t.name().equals(fieldName)) { - texts.add(f1.stringValue()); + texts.add(value.stringValue()); continue; } if (YaCySchema.h1_txt.name().equals(fieldName) || YaCySchema.h2_txt.name().equals(fieldName) || YaCySchema.h3_txt.name().equals(fieldName) || YaCySchema.h4_txt.name().equals(fieldName) || YaCySchema.h5_txt.name().equals(fieldName) || YaCySchema.h6_txt.name().equals(fieldName)) { - texts.add(f1.stringValue()); + // because these are multi-valued fields, there can be several of each + texts.add(value.stringValue()); continue; } }