for solr. Try this: http://localhost:8090/gsa/searchresult?q=pdf&site=col1&num=10pull/1/head
Michael Peter Christen
13 years ago
parent
aab0b680c3
commit
d988ba50cf
@ -0,0 +1,120 @@
|
|||||||
|
/**
|
||||||
|
* search
|
||||||
|
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||||
|
* First released 14.08.2012 at http://yacy.net
|
||||||
|
*
|
||||||
|
* This library is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU Lesser General Public
|
||||||
|
* License as published by the Free Software Foundation; either
|
||||||
|
* version 2.1 of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This library is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* Lesser General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Lesser General Public License
|
||||||
|
* along with this program in the file lgpl21.txt
|
||||||
|
* If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.Writer;
|
||||||
|
|
||||||
|
import net.yacy.cora.document.UTF8;
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.cora.services.federated.solr.GSAResponseWriter;
|
||||||
|
import net.yacy.kelondro.logging.Log;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.search.solr.EmbeddedSolrConnector;
|
||||||
|
|
||||||
|
import org.apache.solr.common.SolrException;
|
||||||
|
import org.apache.solr.common.params.CommonParams;
|
||||||
|
import org.apache.solr.common.util.FastWriter;
|
||||||
|
import org.apache.solr.request.SolrQueryRequest;
|
||||||
|
import org.apache.solr.response.SolrQueryResponse;
|
||||||
|
|
||||||
|
import de.anomic.server.serverObjects;
|
||||||
|
import de.anomic.server.serverSwitch;
|
||||||
|
|
||||||
|
// try
|
||||||
|
// http://localhost:8090/gsa/search?q=chicken+teriyaki&output=xml&client=test&site=test&sort=date:D:S:d1
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is a gsa result formatter for solr search results.
|
||||||
|
* The result format is implemented according to
|
||||||
|
* https://developers.google.com/search-appliance/documentation/68/xml_reference#results_xml
|
||||||
|
*/
|
||||||
|
public class searchresult {
|
||||||
|
|
||||||
|
private final static GSAResponseWriter responseWriter = new GSAResponseWriter();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get the right mime type for this streamed result page
|
||||||
|
* @param header
|
||||||
|
* @param post
|
||||||
|
* @param env
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static String mime(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||||
|
return "text/xml";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param header
|
||||||
|
* @param post
|
||||||
|
* @param env
|
||||||
|
* @param out
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env, final OutputStream out) {
|
||||||
|
|
||||||
|
// this uses the methods in the jetty servlet environment and can be removed if jetty in implemented
|
||||||
|
Switchboard sb = (Switchboard) env;
|
||||||
|
|
||||||
|
// check if user is allowed to search (can be switched in /ConfigPortal.html)
|
||||||
|
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header);
|
||||||
|
if (!searchAllowed) return null;
|
||||||
|
|
||||||
|
// check post
|
||||||
|
if (post == null) return null;
|
||||||
|
|
||||||
|
// rename post fields according to result style
|
||||||
|
//post.put(CommonParams.Q, post.remove("q")); // same as solr
|
||||||
|
//post.put(CommonParams.START, post.remove("start")); // same as solr
|
||||||
|
//post.put(, post.remove("site"));//required, example: col1|col2
|
||||||
|
//post.put(, post.remove("client"));//required, example: myfrontend
|
||||||
|
//post.put(, post.remove("output"));//required, example: xml,xml_no_dtd
|
||||||
|
post.put(CommonParams.ROWS, post.remove("num"));
|
||||||
|
|
||||||
|
// get the embedded connector
|
||||||
|
EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr();
|
||||||
|
if (connector == null) return null;
|
||||||
|
|
||||||
|
// do the solr request
|
||||||
|
SolrQueryRequest req = connector.request(post.toSolrParams());
|
||||||
|
SolrQueryResponse response = null;
|
||||||
|
Exception e = null;
|
||||||
|
try {response = connector.query(req);} catch (SolrException ee) {e = ee;}
|
||||||
|
if (response != null) e = response.getException();
|
||||||
|
if (e != null) {
|
||||||
|
Log.logException(e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// write the result directly to the output stream
|
||||||
|
Writer ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset));
|
||||||
|
try {
|
||||||
|
responseWriter.write(ow, req, response);
|
||||||
|
ow.flush();
|
||||||
|
} catch (IOException e1) {
|
||||||
|
} finally {
|
||||||
|
req.close();
|
||||||
|
try {ow.close();} catch (IOException e1) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,258 @@
|
|||||||
|
/**
|
||||||
|
* GSAResponseWriter
|
||||||
|
* Copyright 2012 by Michael Peter Christen
|
||||||
|
* First released 14.08.2012 at http://yacy.net
|
||||||
|
*
|
||||||
|
* This library is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU Lesser General Public
|
||||||
|
* License as published by the Free Software Foundation; either
|
||||||
|
* version 2.1 of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This library is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* Lesser General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Lesser General Public License
|
||||||
|
* along with this program in the file lgpl21.txt
|
||||||
|
* If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package net.yacy.cora.services.federated.solr;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.Writer;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import net.yacy.cora.document.RSSMessage;
|
||||||
|
import net.yacy.cora.lod.vocabulary.DublinCore;
|
||||||
|
import net.yacy.search.index.YaCySchema;
|
||||||
|
|
||||||
|
import org.apache.lucene.document.Document;
|
||||||
|
import org.apache.lucene.document.Fieldable;
|
||||||
|
import org.apache.solr.common.util.NamedList;
|
||||||
|
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||||
|
import org.apache.solr.request.SolrQueryRequest;
|
||||||
|
import org.apache.solr.response.QueryResponseWriter;
|
||||||
|
import org.apache.solr.response.SolrQueryResponse;
|
||||||
|
import org.apache.solr.search.DocIterator;
|
||||||
|
import org.apache.solr.search.DocSlice;
|
||||||
|
import org.apache.solr.search.SolrIndexSearcher;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* implementation of a GSA search result.
|
||||||
|
* example: GET /gsa/searchresult?q=chicken+teriyaki&output=xml&client=test&site=test&sort=date:D:S:d1
|
||||||
|
* for a xml reference, see https://developers.google.com/search-appliance/documentation/68/xml_reference
|
||||||
|
*/
|
||||||
|
public class GSAResponseWriter implements QueryResponseWriter {
|
||||||
|
|
||||||
|
private static final char lb = '\n';
|
||||||
|
private enum GSAToken {
|
||||||
|
CACHE_LAST_MODIFIED, // Date that the document was crawled, as specified in the Date HTTP header when the document was crawled for this index.
|
||||||
|
CRAWLDATE, // An optional element that shows the date when the page was crawled. It is shown only for pages that have been crawled within the past two days.
|
||||||
|
U, // The URL of the search result.
|
||||||
|
UE, // The URL-encoded version of the URL that is in the U parameter.
|
||||||
|
T, // The title of the search result.
|
||||||
|
RK, // Provides a ranking number used internally by the search appliance.
|
||||||
|
ENT_SOURCE, // Identifies the application ID (serial number) of the search appliance that contributes to a result. Example: <ENT_SOURCE>S5-KUB000F0ADETLA</ENT_SOURCE>
|
||||||
|
FS, // Additional details about the search result.
|
||||||
|
S, // The snippet for the search result. Query terms appear in bold in the results. Line breaks are included for proper text wrapping.
|
||||||
|
LANG, // Indicates the language of the search result. The LANG element contains a two-letter language code.
|
||||||
|
HAS; // Encapsulates special features that are included for this search result.
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static final char[] XML_START = (
|
||||||
|
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<GSP VER=\"3.2\">\n").toCharArray();
|
||||||
|
private static final char[] XML_STOP = "</GSP>\n".toCharArray();
|
||||||
|
|
||||||
|
// define a list of simple YaCySchema -> RSS Token matchings
|
||||||
|
private static final Map<String, String> field2tag = new HashMap<String, String>();
|
||||||
|
|
||||||
|
// pre-select a set of YaCy schema fields for the solr searcher which should cause a better caching
|
||||||
|
private static final YaCySchema[] extrafields = new YaCySchema[]{
|
||||||
|
YaCySchema.id, YaCySchema.title, YaCySchema.description, YaCySchema.text_t,
|
||||||
|
YaCySchema.h1_txt, YaCySchema.h2_txt, YaCySchema.h3_txt, YaCySchema.h4_txt, YaCySchema.h5_txt, YaCySchema.h6_txt,
|
||||||
|
};
|
||||||
|
private static final Set<String> SOLR_FIELDS = new HashSet<String>();
|
||||||
|
static {
|
||||||
|
field2tag.put(YaCySchema.last_modified.name(), GSAToken.CACHE_LAST_MODIFIED.name());
|
||||||
|
field2tag.put(YaCySchema.load_date_dt.name(), GSAToken.CRAWLDATE.name());
|
||||||
|
field2tag.put(YaCySchema.language_txt.name(), GSAToken.LANG.name());
|
||||||
|
SOLR_FIELDS.addAll(field2tag.keySet());
|
||||||
|
for (YaCySchema field: extrafields) SOLR_FIELDS.add(field.name());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class ResHead {
|
||||||
|
public int offset, rows, numFound;
|
||||||
|
//public int status, QTime;
|
||||||
|
//public String df, q, wt;
|
||||||
|
//public float maxScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public GSAResponseWriter() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getContentType(final SolrQueryRequest request, final SolrQueryResponse response) {
|
||||||
|
return CONTENT_TYPE_XML_UTF8;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(@SuppressWarnings("rawtypes") NamedList n) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException {
|
||||||
|
assert rsp.getValues().get("responseHeader") != null;
|
||||||
|
assert rsp.getValues().get("response") != null;
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
SimpleOrderedMap<Object> responseHeader = (SimpleOrderedMap<Object>) rsp.getResponseHeader();
|
||||||
|
DocSlice response = (DocSlice) rsp.getValues().get("response");
|
||||||
|
|
||||||
|
// parse response header
|
||||||
|
ResHead resHead = new ResHead();
|
||||||
|
NamedList<?> val0 = (NamedList<?>) responseHeader.get("params");
|
||||||
|
resHead.rows = Integer.parseInt((String) val0.get("rows"));
|
||||||
|
resHead.offset = response.offset(); // equal to 'start'
|
||||||
|
resHead.numFound = response.matches();
|
||||||
|
//resHead.df = (String) val0.get("df");
|
||||||
|
//resHead.q = (String) val0.get("q");
|
||||||
|
//resHead.wt = (String) val0.get("wt");
|
||||||
|
//resHead.status = (Integer) responseHeader.get("status");
|
||||||
|
//resHead.QTime = (Integer) responseHeader.get("QTime");
|
||||||
|
//resHead.maxScore = response.maxScore();
|
||||||
|
|
||||||
|
// write header
|
||||||
|
writer.write(XML_START);
|
||||||
|
paramTag(writer, "start", Integer.toString(resHead.offset));
|
||||||
|
paramTag(writer, "num", Integer.toString(resHead.rows));
|
||||||
|
|
||||||
|
// parse body
|
||||||
|
final int responseCount = response.size();
|
||||||
|
SolrIndexSearcher searcher = request.getSearcher();
|
||||||
|
DocIterator iterator = response.iterator();
|
||||||
|
for (int i = 0; i < responseCount; i++) {
|
||||||
|
OpensearchResponseWriter.openTag(writer, "R");
|
||||||
|
int id = iterator.nextDoc();
|
||||||
|
Document doc = searcher.doc(id, SOLR_FIELDS);
|
||||||
|
List<Fieldable> fields = doc.getFields();
|
||||||
|
int fieldc = fields.size();
|
||||||
|
List<String> texts = new ArrayList<String>();
|
||||||
|
String description = "";
|
||||||
|
for (int j = 0; j < fieldc; j++) {
|
||||||
|
Fieldable value = fields.get(j);
|
||||||
|
String fieldName = value.name();
|
||||||
|
|
||||||
|
// apply generic matching rule
|
||||||
|
String stag = field2tag.get(fieldName);
|
||||||
|
if (stag != null) {
|
||||||
|
OpensearchResponseWriter.solitaireTag(writer, stag, value.stringValue());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
<RK></RK>
|
||||||
|
<FS NAME="date" VALUE=""/>
|
||||||
|
<S></S>
|
||||||
|
<HAS><L/><C SZ="7k" CID="XN-uikfmLv0J" ENC="UTF-8"/></HAS>
|
||||||
|
*/
|
||||||
|
|
||||||
|
// if the rule is not generic, use the specific here
|
||||||
|
if (YaCySchema.sku.name().equals(fieldName)) {
|
||||||
|
String U = value.stringValue();
|
||||||
|
OpensearchResponseWriter.solitaireTag(writer, GSAToken.U.name(), U);
|
||||||
|
OpensearchResponseWriter.solitaireTag(writer, GSAToken.UE.name(), U);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (YaCySchema.title.name().equals(fieldName)) {
|
||||||
|
OpensearchResponseWriter.solitaireTag(writer, GSAToken.T.name(), value.stringValue());
|
||||||
|
texts.add(value.stringValue());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (YaCySchema.description.name().equals(fieldName)) {
|
||||||
|
description = value.stringValue();
|
||||||
|
OpensearchResponseWriter.solitaireTag(writer, DublinCore.Description.getURIref(), description);
|
||||||
|
texts.add(description);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (YaCySchema.text_t.name().equals(fieldName)) {
|
||||||
|
texts.add(value.stringValue());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (YaCySchema.h1_txt.name().equals(fieldName) || YaCySchema.h2_txt.name().equals(fieldName) ||
|
||||||
|
YaCySchema.h3_txt.name().equals(fieldName) || YaCySchema.h4_txt.name().equals(fieldName) ||
|
||||||
|
YaCySchema.h5_txt.name().equals(fieldName) || YaCySchema.h6_txt.name().equals(fieldName)) {
|
||||||
|
// because these are multi-valued fields, there can be several of each
|
||||||
|
texts.add(value.stringValue());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// compute snippet from texts
|
||||||
|
OpensearchResponseWriter.solitaireTag(writer, RSSMessage.Token.description.name(), description);
|
||||||
|
OpensearchResponseWriter.solitaireTag(writer, GSAToken.ENT_SOURCE.name(), "YaCy");
|
||||||
|
OpensearchResponseWriter.closeTag(writer, "R");
|
||||||
|
}
|
||||||
|
|
||||||
|
writer.write(XML_STOP);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static void paramTag(final Writer writer, final String tagname, String value) throws IOException {
|
||||||
|
if (value == null || value.length() == 0) return;
|
||||||
|
writer.write("<PARAM name=\"");
|
||||||
|
writer.write(tagname);
|
||||||
|
writer.write("\" value=\"");
|
||||||
|
writer.write(value);
|
||||||
|
writer.write("\" original_value=\"");
|
||||||
|
writer.write(value);
|
||||||
|
writer.write("\"/>"); writer.write(lb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
|
<GSP VER="3.2">
|
||||||
|
<TM>0.053898</TM>
|
||||||
|
<Q>pdf</Q>
|
||||||
|
<PARAM name="sort" value="date:D:L:d1" original_value="date:D:L:d1"/>
|
||||||
|
<PARAM name="output" value="xml_no_dtd" original_value="xml_no_dtd"/>
|
||||||
|
<PARAM name="ie" value="UTF-8" original_value="UTF-8"/>
|
||||||
|
<PARAM name="oe" value="UTF-8" original_value="UTF-8"/>
|
||||||
|
<PARAM name="client" value="" original_value=""/>
|
||||||
|
<PARAM name="q" value="pdf" original_value="pdf"/>
|
||||||
|
<PARAM name="site" value="" original_value=""/>
|
||||||
|
<PARAM name="start" value="0" original_value="0"/>
|
||||||
|
<PARAM name="num" value="10" original_value="10"/>
|
||||||
|
<PARAM name="ip" value="" original_value=""/>
|
||||||
|
<PARAM name="access" value="p" original_value="p"/>
|
||||||
|
<PARAM name="entqr" value="3" original_value="3"/>
|
||||||
|
<PARAM name="entqrm" value="0" original_value="0"/>
|
||||||
|
<RES SN="1" EN="10">
|
||||||
|
<M>296</M>
|
||||||
|
<NB>
|
||||||
|
<NU></NU>
|
||||||
|
</NB>
|
||||||
|
|
||||||
|
<R N="1">
|
||||||
|
<U></U>
|
||||||
|
<UE></UE>
|
||||||
|
<T></T>
|
||||||
|
<RK></RK>
|
||||||
|
<ENT_SOURCE></ENT_SOURCE>
|
||||||
|
<FS NAME="date" VALUE=""/>
|
||||||
|
<S></S>
|
||||||
|
<LANG>de</LANG>
|
||||||
|
<HAS><L/><C SZ="7k" CID="XN-uikfmLv0J" ENC="UTF-8"/></HAS>
|
||||||
|
</R>
|
||||||
|
<R N="2"></R>
|
||||||
|
</RES>
|
||||||
|
</GSP>
|
||||||
|
*/
|
||||||
Loading…
Reference in new issue