From 8f58c1dcfa557b88e665b7b30ac9b4fa790e5938 Mon Sep 17 00:00:00 2001 From: sgaebel Date: Fri, 4 Jan 2019 18:27:44 +0100 Subject: [PATCH] extend the SolrServlet to be usable as remote solr (incl. update) this feature needs to be enabled by uncomment the url-pattern --- defaults/web.xml | 4 +- .../net/yacy/http/servlets/SolrServlet.java | 175 +++++++++++------- 2 files changed, 115 insertions(+), 64 deletions(-) diff --git a/defaults/web.xml b/defaults/web.xml index ef48d8318..02290d642 100644 --- a/defaults/web.xml +++ b/defaults/web.xml @@ -89,7 +89,9 @@ SolrServlet /solr/collection1/admin/luke - /solr/webgraph/admin/luke + /solr/webgraph/admin/luke + + diff --git a/source/net/yacy/http/servlets/SolrServlet.java b/source/net/yacy/http/servlets/SolrServlet.java index 3b2966154..7795a50e9 100644 --- a/source/net/yacy/http/servlets/SolrServlet.java +++ b/source/net/yacy/http/servlets/SolrServlet.java @@ -20,91 +20,140 @@ package net.yacy.http.servlets; +import java.io.EOFException; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.nio.charset.StandardCharsets; +import java.io.OutputStream; +import java.util.Iterator; +import java.util.Map; import javax.servlet.ServletException; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.params.MultiMapSolrParams; -import org.apache.solr.common.util.ContentStreamBase; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.handler.admin.LukeRequestHandler; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.response.BinaryQueryResponseWriter; +import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.QueryResponseWriterUtil; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.servlet.ResponseUtils; import org.apache.solr.servlet.SolrRequestParsers; import org.apache.solr.servlet.cache.Method; -import org.apache.solr.util.FastWriter; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; +import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; import net.yacy.search.Switchboard; import net.yacy.search.schema.CollectionSchema; import net.yacy.search.schema.WebgraphSchema; public class SolrServlet extends HttpServlet { - - private static final long serialVersionUID = 1L; - - @Override - public void service(ServletRequest request, ServletResponse response) throws IOException, ServletException { - - HttpServletRequest hrequest = (HttpServletRequest) request; - - final Method reqMethod = Method.getMethod(hrequest.getMethod()); - - // get the embedded connector - String requestURI = hrequest.getRequestURI(); - MultiMapSolrParams mmsp = SolrRequestParsers.parseQueryString(hrequest.getQueryString()); - boolean defaultConnector = (requestURI.startsWith("/solr/" + WebgraphSchema.CORE_NAME)) ? false : requestURI.startsWith("/solr/" + CollectionSchema.CORE_NAME) || mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); - mmsp.getMap().remove("core"); - Switchboard sb = Switchboard.getSwitchboard(); - EmbeddedSolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME); - if (connector == null) throw new ServletException("no core"); - - SolrQueryResponse solrRsp = new SolrQueryResponse(); - SolrQueryRequest solrReq = null; - try { - solrReq = connector.request(mmsp); // SolrRequestParsers.DEFAULT.parse(null, hrequest.getServletPath(), hrequest); - solrReq.getContext().put("webapp", hrequest.getContextPath()); - SolrRequestHandler handler = sb.index.fulltext().getEmbeddedInstance().getCoreContainer().getMultiCoreHandler(); - connector.getCore().execute( handler, solrReq, solrRsp ); - - // write response header - QueryResponseWriter responseWriter = connector.getCore().getQueryResponseWriter(solrReq); - - final String ct = responseWriter.getContentType(solrReq, solrRsp); - if (null != ct) response.setContentType(ct); - - if (Method.HEAD != reqMethod) { - if (responseWriter instanceof BinaryQueryResponseWriter) { - BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) responseWriter; - binWriter.write(response.getOutputStream(), solrReq, solrRsp); - } else { - String charset = ContentStreamBase.getCharsetFromContentType(ct); - Writer out = (charset == null || charset.equalsIgnoreCase(StandardCharsets.UTF_8.name())) - ? new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8) - : new OutputStreamWriter(response.getOutputStream(), charset); - out = new FastWriter(out); - responseWriter.write(out, solrReq, solrRsp); - out.flush(); - } - } - - ConcurrentLog.info("luke", solrRsp.getValues().toString()); - - } catch (Exception e) { - ConcurrentLog.logException(e); - } finally { - if (solrReq != null) solrReq.close(); - } - - } + private static final long serialVersionUID = 1L; + + @Override + public void service(ServletRequest request, ServletResponse response) throws IOException, ServletException { + + if (!Domains.isIntranet(request.getRemoteAddr())) { + ((HttpServletResponse) response).sendError(HttpServletResponse.SC_FORBIDDEN, + "SolrServlet use not granted for IP " + request.getRemoteAddr()); + } + + HttpServletRequest hrequest = (HttpServletRequest) request; + + final Method reqMethod = Method.getMethod(hrequest.getMethod()); + + // get the embedded connector + String requestURI = hrequest.getRequestURI(); + MultiMapSolrParams mmsp = SolrRequestParsers.parseQueryString(hrequest.getQueryString()); + boolean defaultConnector = (requestURI.startsWith("/solr/" + WebgraphSchema.CORE_NAME)) ? false + : requestURI.startsWith("/solr/" + CollectionSchema.CORE_NAME) + || mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); + mmsp.getMap().remove("core"); + Switchboard sb = Switchboard.getSwitchboard(); + EmbeddedSolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() + : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME); + + if (connector == null) + throw new ServletException("no core"); + + SolrQueryResponse solrRsp = new SolrQueryResponse(); + SolrQueryRequest solrReq = null; + try { + solrReq = SolrRequestParsers.DEFAULT.parse(connector.getCore(), hrequest.getServletPath(), hrequest); + solrReq.getContext().put("webapp", hrequest.getContextPath()); + SolrRequestHandler handler; + if ("/solr/collection1/update".equals(hrequest.getServletPath()) + || "/solr/webgraph/update".equals(hrequest.getServletPath())) { + handler = new UpdateRequestHandler(); + } else { + handler = new LukeRequestHandler(); + } + handler.init(new NamedList()); + + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp)); + connector.getCore().execute(handler, solrReq, solrRsp); + Iterator> headers = solrRsp.httpHeaders(); + while (headers.hasNext()) { + Map.Entry entry = headers.next(); + ((HttpServletResponse) response).addHeader(entry.getKey(), entry.getValue()); + } + + // write response header + QueryResponseWriter responseWriter = connector.getCore().getQueryResponseWriter(solrReq); + writeResponse(solrReq, solrRsp, (HttpServletResponse) response, responseWriter, reqMethod); + } catch (Exception e) { + ConcurrentLog.logException(e); + } finally { + if (solrReq != null) + solrReq.close(); + SolrRequestInfo.clearRequestInfo(); + } + + } + + private void writeResponse(SolrQueryRequest solrReq, SolrQueryResponse solrRsp, HttpServletResponse response, QueryResponseWriter responseWriter, + Method reqMethod) throws IOException { + try { + Object invalidStates = solrReq.getContext().get(CloudSolrClient.STATE_VERSION); + // This is the last item added to the response and the client would expect it + // that way. + // If that assumption is changed , it would fail. This is done to avoid an O(n) + // scan on + // the response for each request + if (invalidStates != null) + solrRsp.add(CloudSolrClient.STATE_VERSION, invalidStates); + // Now write it out + final String ct = responseWriter.getContentType(solrReq, solrRsp); + // don't call setContentType on null + if (null != ct) + response.setContentType(ct); + + if (solrRsp.getException() != null) { + @SuppressWarnings("rawtypes") + NamedList info = new SimpleOrderedMap(); + int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, null); + solrRsp.add("error", info); + response.setStatus(code); + } + + if (Method.HEAD != reqMethod) { + OutputStream out = response.getOutputStream(); + QueryResponseWriterUtil.writeQueryResponse(out, responseWriter, solrReq, solrRsp, ct); + } + // else http HEAD request, nothing to write out, waited this long just to get + // ContentType + } catch (EOFException e) { + ConcurrentLog.info("SolrServlet", "Unable to write response, client closed connection or we are shutting down", e); + } + } }