diff --git a/htroot/gsa/searchresult.java b/htroot/gsa/searchresult.java deleted file mode 100644 index c3740e6da..000000000 --- a/htroot/gsa/searchresult.java +++ /dev/null @@ -1,240 +0,0 @@ -/** - * search - * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.08.2012 at http://yacy.net - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.util.Date; - -import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.federate.solr.Ranking; -import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; -import net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter; -import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.search.Switchboard; -import net.yacy.search.query.AccessTracker; -import net.yacy.search.query.QueryGoal; -import net.yacy.search.query.QueryModifier; -import net.yacy.search.query.SearchEvent; -import net.yacy.search.schema.CollectionSchema; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.response.ResultContext; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.FastWriter; - - -// try -// http://localhost:8090/gsa/searchresult?q=chicken+teriyaki&output=xml&client=test&site=test&sort=date:D:S:d1 - -/** - * This is a gsa result formatter for solr search results. - * The result format is implemented according to - * https://developers.google.com/search-appliance/documentation/68/xml_reference#results_xml - */ -public class searchresult { - - private final static GSAResponseWriter responseWriter = new GSAResponseWriter(); - - /** - * get the right mime type for this streamed result page - * @param header - * @param post - * @param env - * @return - */ - public static String mime(final RequestHeader header, final serverObjects post, final serverSwitch env) { - return "text/xml"; - } - - - /** - * @param header - * @param post - * @param env - * @param out - * @return - */ - public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env, final OutputStream out) { - - // this uses the methods in the jetty servlet environment and can be removed if jetty in implemented - Switchboard sb = (Switchboard) env; - - // remember the peer contact for peer statistics - final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""); // read an artificial header addendum - final String userAgent = header.get(HeaderFramework.USER_AGENT, ""); - sb.peers.peerActions.setUserAgent(clientip, userAgent); - - // check if user is allowed to search (can be switched in /ConfigPortal.html) - boolean authenticated = sb.adminAuthenticated(header) >= 2; - final boolean searchAllowed = authenticated || sb.getConfigBool("publicSearchpage", true); - if (!searchAllowed) return null; - - // check post - if (post == null) {post = new serverObjects(); post.put("q", ""); post.put("num", "0");} - ConcurrentLog.info("GSA Query", post.toString()); - sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time - - // rename post fields according to result style - //post.put(CommonParams.Q, post.remove("q")); // same as solr - //post.put(CommonParams.START, post.remove("start")); // same as solr - //post.put(, post.remove("client"));//required, example: myfrontend - //post.put(, post.remove("output"));//required, example: xml,xml_no_dtd - String originalQuery = post.get(CommonParams.Q, ""); - post.put("originalQuery", originalQuery); - - // get a solr query string - QueryGoal qg = new QueryGoal(originalQuery, originalQuery); - StringBuilder solrQ = qg.collectionTextQueryString(sb.index.fulltext().getDefaultConfiguration(), 0, false); - post.put("defType", "edismax"); - post.put(CommonParams.Q, solrQ.toString()); - post.put(CommonParams.ROWS, post.remove("num")); - post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100)); - - // set ranking - if (post.containsKey("sort")) { - // if a gsa-style sort attribute is given, use this to set the solr sort attribute - GSAResponseWriter.Sort sort = new GSAResponseWriter.Sort(post.get(CommonParams.SORT, "")); - String sorts = sort.toSolr(); - if (sorts == null) { - post.remove(CommonParams.SORT); - } else { - post.put(CommonParams.SORT, sorts); - } - } else { - // if no such sort attribute is given, use the ranking as configured for YaCy - Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(0); - String bq = ranking.getBoostQuery(); - String bf = ranking.getBoostFunction(); - if (bq.length() > 0) post.put("bq", bq); - if (bf.length() > 0) post.put("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 - } - String daterange[] = post.remove("daterange"); - if (daterange != null) { - String origfq = post.get(CommonParams.FQ); - String datefq = ""; - for (String dr: daterange) { - String from_to[] = dr.endsWith("..") ? new String[]{dr.substring(0, dr.length() - 2), ""} : dr.startsWith("..") ? new String[]{"", dr.substring(2)} : dr.split("\\.\\."); - if (from_to.length != 2) continue; - Date from = HeaderFramework.parseGSAFS(from_to[0]); - if (from == null) from = new Date(0); - Date to = HeaderFramework.parseGSAFS(from_to[1]); - if (to == null) to = new Date(); - to.setTime(to.getTime() + 24L * 60L * 60L * 1000L); // we add a day because the day is inclusive - String z = CollectionSchema.last_modified.getSolrFieldName() + ":[" + ISO8601Formatter.FORMATTER.format(from) + " TO " + ISO8601Formatter.FORMATTER.format(to) + "]"; - datefq = datefq.length() == 0 ? z : " OR " + z; - } - if (datefq.length() > 0) post.put(CommonParams.FQ, origfq == null || origfq.length() == 0 ? datefq : "(" + origfq + ") AND (" + datefq + ")"); - } - post.put(CommonParams.FL, - CollectionSchema.content_type.getSolrFieldName() + ',' + - CollectionSchema.id.getSolrFieldName() + ',' + - CollectionSchema.sku.getSolrFieldName() + ',' + - CollectionSchema.title.getSolrFieldName() + ',' + - CollectionSchema.description_txt.getSolrFieldName() + ',' + - CollectionSchema.load_date_dt.getSolrFieldName() + ',' + - CollectionSchema.last_modified.getSolrFieldName() + ',' + - CollectionSchema.size_i.getSolrFieldName()); - post.put("hl", "true"); - post.put("hl.q", qg.getIncludeString()); - post.put("hl.fl", CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName()); - post.put("hl.alternateField", CollectionSchema.description_txt.getSolrFieldName()); - post.put("hl.simple.pre", ""); - post.put("hl.simple.post", ""); - post.put("hl.fragsize", Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)); - - //String[] access = post.remove("access"); - //String[] entqr = post.remove("entqr"); - - // add sites operator - String[] site = post.remove("site"); // example: col1|col2 - if (site != null && site[0].length() > 0) { - String origfq = post.get(CommonParams.FQ); - String sitefq = QueryModifier.parseCollectionExpression(site[0]); - post.put(CommonParams.FQ, origfq == null || origfq.length() == 0 ? sitefq : "(" + origfq + ") AND (" + sitefq + ")"); - } - - // get the embedded connector - EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector(); - if (connector == null) return null; - - // do the solr request - SolrQueryRequest req = connector.request(post.toSolrParams(null)); - SolrQueryResponse response = null; - Writer ow = null; - try { - response = connector.query(req); - if (response != null) { - Exception e = response.getException(); - if (e != null) { - ConcurrentLog.logException(e); - } else { - - // set some context for the writer - /* - Map context = req.getContext(); - context.put("ip", header.get("CLIENTIP", "")); - context.put("client", "vsm_frontent"); - context.put("sort", sort.sort); - context.put("site", site == null ? "" : site); - context.put("access", access == null ? "p" : access[0]); - context.put("entqr", entqr == null ? "3" : entqr[0]); - */ - - // write the result directly to the output stream - ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset)); - responseWriter.write(ow, req, response); - ow.flush(); - } - } - } catch (final SolrException e) { - ConcurrentLog.logException(e); - } catch (final IOException e1) { - } finally { - req.close(); - SolrRequestInfo.clearRequestInfo(); - if (ow != null) try {ow.close();} catch (final IOException e1) {} - } - if (response == null) return null; - - // log result - Object rv = response.getValues().get("response"); - int matches = 0; - if (rv != null && rv instanceof ResultContext) { - matches = ((ResultContext) rv).docs.matches(); - } else if (rv != null && rv instanceof SolrDocumentList) { - matches = (int) ((SolrDocumentList) rv).getNumFound(); - } - AccessTracker.addToDump(originalQuery, Integer.toString(matches)); - ConcurrentLog.info("GSA Query", "results: " + matches + ", for query:" + post.toString()); - - return null; - } -} diff --git a/htroot/solr/select.java b/htroot/solr/select.java deleted file mode 100644 index 97b5f8e58..000000000 --- a/htroot/solr/select.java +++ /dev/null @@ -1,249 +0,0 @@ -/** - * select - * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 12.08.2012 at http://yacy.net - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; - -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.federate.solr.Ranking; -import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; -import net.yacy.cora.federate.solr.responsewriter.YJsonResponseWriter; -import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter; -import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.http.servlets.SolrServlet; -import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; -import net.yacy.search.query.AccessTracker; -import net.yacy.search.query.QueryGoal; -import net.yacy.search.query.QueryModifier; -import net.yacy.search.query.SearchEvent; -import net.yacy.search.schema.CollectionSchema; -import net.yacy.search.schema.WebgraphSchema; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.MultiMapSolrParams; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.response.BinaryResponseWriter; -import org.apache.solr.response.QueryResponseWriter; -import org.apache.solr.response.ResultContext; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.FastWriter; - - -// try -// http://localhost:8090/solr/select?q=*:*&start=0&rows=10&indent=on - -/** - * this is a standard solr search result formatter as defined in - * http://wiki.apache.org/solr/SolrQuerySyntax - */ -public class select { - - - /** - * get the right mime type for this streamed result page - * @param header - * @param post - * @param env - * @return - */ - public static String mime(final RequestHeader header, final serverObjects post, final serverSwitch env) { - String wt = post == null ? "xml" : post.get(CommonParams.WT, "xml"); - if (wt == null || wt.length() == 0 || "xml".equals(wt) || "exml".equals(wt)) return "text/xml"; - if ("xslt".equals(wt)) { - String tr = post == null ? "" : post.get("tr",""); - if (tr.indexOf("json") >= 0) return "application/json"; - } - if ("rss".equals(wt)) return "application/rss+xml"; - if ("exml".equals(wt)) return "application/rss+xml"; - if ("json".equals(wt)) return "application/json"; - if ("yjson".equals(wt)) return "application/json"; - if ("html".equals(wt) || "grephtml".equals(wt) || "python".equals(wt)) return "text/html"; - if ("php".equals(wt) || "phps".equals(wt)) return "application/x-httpd-php"; - if ("ruby".equals(wt)) return "text/html"; - if ("raw".equals(wt)) return "application/octet-stream"; - if ("javabin".equals(wt)) return "application/octet-stream"; - if ("csv".equals(wt)) return "text/csv"; - return "text/xml"; - } - - /** - * a query to solr, for documentation of parameters see: - * http://lucene.apache.org/solr/api-3_6_0/doc-files/tutorial.html - * and - * http://wiki.apache.org/solr/SolrQuerySyntax - * @param header - * @param post - * @param env - * @param out - * @return - */ - public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env, final OutputStream out) { - - // this uses the methods in the jetty servlet environment and can be removed if jetty in implemented - Switchboard sb = (Switchboard) env; - - // remember the peer contact for peer statistics - final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""); // read an artificial header addendum - final String userAgent = header.get(HeaderFramework.USER_AGENT, ""); - sb.peers.peerActions.setUserAgent(clientip, userAgent); - - // check if user is allowed to search (can be switched in /ConfigPortal.html) - boolean authenticated = sb.adminAuthenticated(header) >= 2; - final boolean searchAllowed = authenticated || sb.getConfigBool("publicSearchpage", true); - if (!searchAllowed) return null; - - // check post - if (post == null) {post = new serverObjects(); post.put(CommonParams.Q, ""); post.put(CommonParams.ROWS, "0");} - if (post.size() > 100) { - ConcurrentLog.warn("select", "rejected bad-formed search request with " + post.size() + " properties from " + header.refererHost()); - return null; // prevent the worst hacks here... - } - sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time - - MultiMapSolrParams mmsp = post.toSolrParams(/*responseWriter instanceof JsonResponseWriter ? new YaCySchema[]{YaCySchema.host_s, YaCySchema.url_file_ext_s, YaCySchema.url_protocol_s} :*/ null); - - // count remote searches if this was part of a p2p search - if (mmsp.getMap().containsKey("partitions")) { - final int partitions = mmsp.getInt("partitions", 30); - sb.searchQueriesGlobal += 1.0f / partitions; // increase query counter - } - - // get the ranking profile id - int profileNr = mmsp.getInt("profileNr", 0); - - // rename post fields according to result style - if (!mmsp.getMap().containsKey(CommonParams.Q) && mmsp.getMap().containsKey("query")) { - String querystring = mmsp.get("query", ""); - mmsp.getMap().remove("query"); - QueryModifier modifier = new QueryModifier(); - querystring = modifier.parse(querystring); - modifier.apply(mmsp); - QueryGoal qg = new QueryGoal(querystring, querystring); - StringBuilder solrQ = qg.collectionTextQueryString(sb.index.fulltext().getDefaultConfiguration(), profileNr, false); - mmsp.getMap().put(CommonParams.Q, new String[]{solrQ.toString()}); // sru patch - } - String q = mmsp.get(CommonParams.Q, ""); - if (!mmsp.getMap().containsKey(CommonParams.START)) { - int startRecord = mmsp.getFieldInt("startRecord", "0"); - mmsp.getMap().remove("startRecord"); - mmsp.getMap().put(CommonParams.START, new String[]{Integer.toString(startRecord)}); // sru patch - } - if (!mmsp.getMap().containsKey(CommonParams.ROWS)) { - int maximumRecords = mmsp.getFieldInt("maximumRecords", "10"); - mmsp.getMap().remove("maximumRecords"); - mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(maximumRecords)}); // sru patch - } - mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(Math.min(mmsp.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100))}); - - // set ranking according to profile number if ranking attributes are not given in the request - if (!mmsp.getMap().containsKey("sort") && !mmsp.getMap().containsKey("bq") && !mmsp.getMap().containsKey("bf") && !mmsp.getMap().containsKey("boost")) { - if (!mmsp.getMap().containsKey("defType")) mmsp.getMap().put("defType", new String[]{"edismax"}); - Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr); - String bq = ranking.getBoostQuery(); - String bf = ranking.getBoostFunction(); - if (bq.length() > 0) mmsp.getMap().put("bq", new String[]{bq}); - if (bf.length() > 0) mmsp.getMap().put("boost", new String[]{bf}); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 - } - - // get a response writer for the result - String wt = mmsp.get(CommonParams.WT, "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml - QueryResponseWriter responseWriter = SolrServlet.RESPONSE_WRITER.get(wt); - if (responseWriter == null) return null; - if (responseWriter instanceof OpensearchResponseWriter) { - // set the title every time, it is possible that it has changed - final String promoteSearchPageGreeting = - (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? env.getConfig( - "network.unit.description", - "") : env.getConfig(SwitchboardConstants.GREETING, ""); - ((OpensearchResponseWriter) responseWriter).setTitle(promoteSearchPageGreeting); - } - - // if this is a call to YaCys special search formats, enhance the query with field assignments - if ((responseWriter instanceof YJsonResponseWriter || responseWriter instanceof OpensearchResponseWriter) && "true".equals(mmsp.get("hl", "true"))) { - // add options for snippet generation - if (!mmsp.getMap().containsKey("hl.q")) mmsp.getMap().put("hl.q", new String[]{q}); - if (!mmsp.getMap().containsKey("hl.fl")) mmsp.getMap().put("hl.fl", new String[]{CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName()}); - if (!mmsp.getMap().containsKey("hl.alternateField")) mmsp.getMap().put("hl.alternateField", new String[]{CollectionSchema.description_txt.getSolrFieldName()}); - if (!mmsp.getMap().containsKey("hl.simple.pre")) mmsp.getMap().put("hl.simple.pre", new String[]{""}); - if (!mmsp.getMap().containsKey("hl.simple.post")) mmsp.getMap().put("hl.simple.post", new String[]{""}); - if (!mmsp.getMap().containsKey("hl.fragsize")) mmsp.getMap().put("hl.fragsize", new String[]{Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)}); - } - - // get the embedded connector - boolean defaultConnector = mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); - EmbeddedSolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME); - if (connector == null) return null; - - // do the solr request, generate facets if we use a special YaCy format - SolrQueryRequest req = connector.request(mmsp); - Writer ow = null; - SolrQueryResponse rsp = null; - try { - rsp = connector.query(req); - if (rsp != null) { - Exception e = rsp.getException(); - if (e != null) { - ConcurrentLog.logException(e); - } else { - - // write the result directly to the output stream - if (responseWriter instanceof BinaryResponseWriter) { - ((BinaryResponseWriter) responseWriter).write(out, req, rsp); - } else { - ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset)); - responseWriter.write(ow, req, rsp); - ow.flush(); - } - } - } - } catch (final SolrException e) { - ConcurrentLog.logException(e); - } catch (final IOException e1) { - } finally { - req.close(); - SolrRequestInfo.clearRequestInfo(); - if (ow != null) try {ow.close();} catch (final IOException e1) {} - } - if (rsp == null) return null; - - // log result - Object rv = rsp.getValues().get("response"); - int matches = 0; - if (rv != null && rv instanceof ResultContext) { - matches = ((ResultContext) rv).docs.matches(); - } else if (rv != null && rv instanceof SolrDocumentList) { - matches = (int) ((SolrDocumentList) rv).getNumFound(); - } - AccessTracker.addToDump(q, Integer.toString(matches)); - ConcurrentLog.info("SOLR Query", "results: " + matches + ", for query:" + mmsp.toString()); - - return null; - } -} diff --git a/htroot/solr/update.java b/htroot/solr/update.java deleted file mode 100644 index fb6d463c7..000000000 --- a/htroot/solr/update.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * update - * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 25.09.2012 at http://yacy.net - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -import java.io.OutputStream; - -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -/** - * this is a dummy class doing nothing. It is called by the solr connector but in YaCy we do not use it (yet) - */ -public class update { - - - public static String mime(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { - return "text/xml"; - } - - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env, @SuppressWarnings("unused") final OutputStream out) { - ConcurrentLog.info("update", "post = " + post == null ? "NULL" : post.toString()); - return null; - } -}