Adjusted Solr select servlet output when using an external Solr only

- Use the EnhancedXMLResponseWriter only when requested output is "exml"
- Use the Standard Solr writers when possible, for example for json, xml
or javabin output formats
- Return an error when the requested format can not been rendered with
an external Solr server only

Important : this modification is necessary for peers using exclusively
an external Solr server to be reachable as robinson targets in p2p
search, as the binary format ("javabin") is the default Solr exchange
format for peers.
Before this, when a peer requested a remote one attached only to an
external Solr (no embedded one), it ended with "Invalid type" error, as
the remote peer answered with xml although binary format was requested.

source/net/yacy/cora/federate/solr/responsewriter/EmbeddedSolrResponseWriter.java

@@ -0,0 +1,32 @@
+// EmbeddedSolrResponseWriter.java
+// ---------------------------
+// Copyright 2018 by luccioman; https://github.com/luccioman
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+package net.yacy.cora.federate.solr.responsewriter;
+
+import org.apache.solr.response.QueryResponseWriter;
+
+/**
+ * Interface used only to mark that a {@link QueryResponseWriter} implementation
+ * class is only compatible with results from an embedded Solr server
+ */
+public interface EmbeddedSolrResponseWriter {
+}

source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java

@@ -47,7 +47,7 @@ import org.json.simple.JSONArray;
 import net.yacy.cora.federate.solr.SolrType;
 import net.yacy.cora.util.JSONObject;
 
-public class FlatJSONResponseWriter implements QueryResponseWriter {
+public class FlatJSONResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
 
     private static final char lb = '\n';
     

source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java

@@ -59,7 +59,7 @@ import org.apache.solr.search.SolrIndexSearcher;
  * example: GET /gsa/searchresult?q=chicken+teriyaki&output=xml&client=test&site=test&sort=date:D:S:d1
  * for a xml reference, see https://developers.google.com/search-appliance/documentation/614/xml_reference
  */
-public class GSAResponseWriter implements QueryResponseWriter {
+public class GSAResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
 
     private static String YaCyVer = null;
     private static final char lb = '\n';

source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java

@@ -52,7 +52,7 @@ import org.apache.solr.search.SolrIndexSearcher;
  * text part and each sentence is shown as separate line. grep attributes can be used to
  * show leading and trainling lines.
  */
-public class GrepHTMLResponseWriter implements QueryResponseWriter {
+public class GrepHTMLResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
 
     private static final Set<String> DEFAULT_FIELD_LIST = new HashSet<String>();
     private static final Pattern dqp = Pattern.compile("\"");

source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java

@@ -54,7 +54,7 @@ import net.yacy.cora.lod.vocabulary.DublinCore;
 import net.yacy.search.schema.CollectionSchema;
 import net.yacy.search.schema.WebgraphSchema;
 
-public class HTMLResponseWriter implements QueryResponseWriter {
+public class HTMLResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
 
     public static final Pattern dqp = Pattern.compile("\"");
     

source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java

@@ -58,7 +58,7 @@ import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
 import org.apache.solr.search.SolrIndexSearcher;
 
-public class OpensearchResponseWriter implements QueryResponseWriter {
+public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
 
     // define a list of simple YaCySchema -> RSS Token matchings
     private static final Map<String, String> field2tag = new HashMap<String, String>();

source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java

@@ -20,7 +20,7 @@ import org.apache.solr.search.SolrIndexSearcher;
 /**
  * this writer is supposed to be used to generate iframes. It generates links for the /api/snapshot.jpg servlet.
  */
-public class SnapshotImagesReponseWriter implements QueryResponseWriter  {
+public class SnapshotImagesReponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter  {
 
     private static final Set<String> DEFAULT_FIELD_LIST = new HashSet<>();
     

source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java

@@ -59,7 +59,7 @@ import org.apache.solr.search.SolrIndexSearcher;
  * example:
  * http://localhost:8090/solr/select?hl=false&wt=yjson&facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt&facet.field=collection_sxt&start=0&rows=10&query=www
  */
-public class YJsonResponseWriter implements QueryResponseWriter {
+public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
 
     // define a list of simple YaCySchema -> json Token matchings
     private static final Map<String, String> field2tag = new HashMap<String, String>();

source/net/yacy/http/servlets/SolrSelectServlet.java

@@ -40,6 +40,7 @@ import javax.servlet.http.HttpServletResponse;
 import net.yacy.cora.federate.solr.Ranking;
 import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
 import net.yacy.cora.federate.solr.connector.SolrConnector;
+import net.yacy.cora.federate.solr.responsewriter.EmbeddedSolrResponseWriter;
 import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter;
 import net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter;
 import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter;
@@ -60,17 +61,23 @@ import net.yacy.search.schema.CollectionSchema;
 import net.yacy.search.schema.WebgraphSchema;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.DisMaxParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.MultiMapSolrParams;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequestBase;
 import org.apache.solr.request.SolrRequestInfo;
 import org.apache.solr.response.BinaryResponseWriter;
+import org.apache.solr.response.CSVResponseWriter;
 import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.RawResponseWriter;
 import org.apache.solr.response.ResultContext;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.response.XSLTResponseWriter;
@@ -287,16 +294,55 @@ public class SolrSelectServlet extends HttpServlet {
                     out.flush();
                 }
             } else {
-                // write a 'faked' response using a call to the backend
+				if (responseWriter instanceof EmbeddedSolrResponseWriter || responseWriter instanceof CSVResponseWriter
-                SolrDocumentList sdl = connector.getDocumentListByQuery(
+						|| responseWriter instanceof XSLTResponseWriter || responseWriter instanceof RawResponseWriter) {
-                        mmsp.getMap().get(CommonParams.Q)[0],
+					/* These writers need a non null req.getSearcher(), req.getSchema() and/or req.getCore() */
-                        mmsp.getMap().get(CommonParams.SORT) == null ? null : mmsp.getMap().get(CommonParams.SORT)[0],
+					throw new ServletException("The writer " + responseWriter.getClass().getSimpleName() + " can only process responses from an embedded Solr server.");
-                        Integer.parseInt(mmsp.getMap().get(CommonParams.START)[0]),
+                } 
-                        Integer.parseInt(mmsp.getMap().get(CommonParams.ROWS)[0]),
+            	
-                        mmsp.getMap().get(CommonParams.FL));
+            	QueryResponse queryRsp = connector.getResponseByParams(ModifiableSolrParams.of(mmsp));
-                OutputStreamWriter osw = new OutputStreamWriter(response.getOutputStream());
+            	
-                EnhancedXMLResponseWriter.write(osw, req, sdl);
+                /* Create SolrQueryRequestBase and SolrQueryResponse instances as these types are requited by Solr standard writers.
-                osw.close();
+                 * WARNING : the SolrQueryRequestBase instance will return null for the getSearcher(), getCore() and getSchema() functions.
+                 * Be sure thath the responseWriter instance can handle this properly.  */
+            	req = new SolrQueryRequestBase(null, mmsp) {};
+            	
+            	rsp = new SolrQueryResponse();
+                NamedList<Object> responseHeader = new SimpleOrderedMap<Object>();
+                responseHeader.add("params", mmsp.toNamedList());
+                rsp.add("responseHeader", responseHeader);
+                rsp.setHttpCaching(false);
+                rsp.getValues().addAll(queryRsp.getResponse());
+
+            	
+                // prepare response
+                hresponse.setHeader("Cache-Control", "no-cache, no-store");
+                
+                final SolrDocumentList documentsList = queryRsp.getResults();
+                long numFound = documentsList.getNumFound();
+                AccessTracker.addToDump(querystring, numFound, new Date(), "sq");
+                
+                // write response header
+                final String contentType = responseWriter.getContentType(req, rsp);
+                if (null != contentType) {
+                	response.setContentType(contentType);
+                }
+
+                if (Method.HEAD == reqMethod) {
+                    return;
+                }
+                
+                // write response body
+                if (responseWriter instanceof EnhancedXMLResponseWriter) {
+					out = new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8);
+					EnhancedXMLResponseWriter.write(out, req, documentsList);
+                } else if(responseWriter instanceof BinaryResponseWriter) {
+               		((BinaryResponseWriter) responseWriter).write(response.getOutputStream(), req, rsp);
+               	} else {
+               		out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8));
+               		responseWriter.write(out, req, rsp);
+               		out.flush();
+               	}
             }
         } catch (final Throwable ex) {
             sendError(hresponse, ex);
@@ -305,7 +351,14 @@ public class SolrSelectServlet extends HttpServlet {
                 req.close();
             }
             SolrRequestInfo.clearRequestInfo();
-            if (out != null) try {out.close();} catch (final IOException e1) {}
+            if (out != null) {
+				try {
+					out.close();
+				} catch (final IOException e1) {
+					ConcurrentLog.info("SolrSelect", "Could not close output writer."
+							+ (e1.getMessage() != null ? "Cause : " + e1.getMessage() : ""));
+				}
+            }
         }
     }
 

source/net/yacy/search/query/AccessTracker.java

@@ -151,13 +151,13 @@ public class AccessTracker {
         return 0;
     }
 
-    private static void addToDump(final QueryParams query, int resultCount) {
+    private static void addToDump(final QueryParams query, long resultCount) {
         String queryString = query.getQueryGoal().getQueryString(false);
         if (queryString == null || queryString.isEmpty()) return;
         addToDump(queryString, resultCount, new Date(query.starttime), "qs");
     }
 
-    public static void addToDump(String querystring, int resultCount) {
+    public static void addToDump(String querystring, long resultCount) {
         addToDump(querystring, resultCount, new Date(), "qs");
     }
 
@@ -169,7 +169,7 @@ public class AccessTracker {
      * @param d start time
      * @param querySyntax used syntax (qs=normal querstring, sq=solr querystring,
      */
-    public static void addToDump(String querystring, int resultcount, Date d, String querySyntax) {
+    public static void addToDump(String querystring, long resultcount, Date d, String querySyntax) {
         //if (query.resultcount == 0) return;
         if (querystring == null || querystring.isEmpty()) return;
         final StringBuilder sb = new StringBuilder(40);