From 3961b643a3b3eae52dc4af09fe56a28b6dcca7b0 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 29 Dec 2013 01:25:44 +0100 Subject: [PATCH 1/2] write solr searches to search log --- source/net/yacy/http/servlets/SolrServlet.java | 18 +++++++++++++++++- .../net/yacy/search/query/AccessTracker.java | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/source/net/yacy/http/servlets/SolrServlet.java b/source/net/yacy/http/servlets/SolrServlet.java index a54b931a9..11c5e876b 100644 --- a/source/net/yacy/http/servlets/SolrServlet.java +++ b/source/net/yacy/http/servlets/SolrServlet.java @@ -27,7 +27,9 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; import java.net.URL; +import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.servlet.Filter; @@ -49,8 +51,10 @@ import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter; import net.yacy.cora.federate.solr.responsewriter.HTMLResponseWriter; import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter; import net.yacy.cora.federate.solr.responsewriter.YJsonResponseWriter; +import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; +import net.yacy.search.query.AccessTracker; import net.yacy.search.query.QueryGoal; import net.yacy.search.query.QueryModifier; import net.yacy.search.query.SearchEvent; @@ -62,13 +66,16 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.DisMaxParams; import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.BinaryResponseWriter; import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.XSLTResponseWriter; +import org.apache.solr.search.DocList; import org.apache.solr.servlet.SolrRequestParsers; import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; import org.apache.solr.servlet.cache.Method; @@ -161,8 +168,9 @@ public class SolrServlet implements Filter { int profileNr = mmsp.getInt("profileNr", 0); // rename post fields according to result style + String querystring = ""; if (!mmsp.getMap().containsKey(CommonParams.Q) && mmsp.getMap().containsKey("query")) { - String querystring = mmsp.get("query", ""); + querystring = mmsp.get("query", ""); mmsp.getMap().remove("query"); QueryModifier modifier = new QueryModifier(); querystring = modifier.parse(querystring); @@ -172,6 +180,7 @@ public class SolrServlet implements Filter { mmsp.getMap().put(CommonParams.Q, new String[]{solrQ.toString()}); // sru patch } String q = mmsp.get(CommonParams.Q, ""); + if (querystring.length() == 0) querystring = q; if (!mmsp.getMap().containsKey(CommonParams.START)) { int startRecord = mmsp.getFieldInt("startRecord", "0"); mmsp.getMap().remove("startRecord"); @@ -239,10 +248,17 @@ public class SolrServlet implements Filter { // check error if (rsp.getException() != null) { + AccessTracker.addToDump(querystring, "0", new Date()); sendError(hresponse, rsp.getException()); return; } + + NamedList values = rsp.getValues(); + DocList r = ((ResultContext) values.get("response")).docs; + int numFound = r.matches(); + AccessTracker.addToDump(querystring, Integer.toString(numFound), new Date()); + // write response header final String contentType = responseWriter.getContentType(req, rsp); if (null != contentType) response.setContentType(contentType); diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index eee6e3d38..27d1ef408 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -122,7 +122,7 @@ public class AccessTracker { } } - private static void addToDump(String querystring, String resultcount, Date d) { + public static void addToDump(String querystring, String resultcount, Date d) { //if (query.resultcount == 0) return; if (querystring == null || querystring.isEmpty()) return; final StringBuilder sb = new StringBuilder(40); From 2018e55f8b7b54c76e1dae96797ee8bb1f55707f Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 29 Dec 2013 01:39:30 +0100 Subject: [PATCH 2/2] switched back on index deletion (was accidently off because new jetty framework delivers never null to post arguments .. there may be more of that kind of problems) --- htroot/IndexControlURLs_p.java | 2 +- source/net/yacy/http/servlets/SolrServlet.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index 6bae5637e..c1e4ca30b 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -84,7 +84,7 @@ public class IndexControlURLs_p { List dumpFiles = segment.fulltext().dumpFiles(); prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath()); prop.put("dumprestore_optimizemax", 10); - prop.put("cleanup", post == null ? 1 : 0); + prop.put("cleanup", post == null || post.size() == 0 ? 1 : 0); prop.put("cleanup_solr", segment.fulltext().connectedRemoteSolr() ? 1 : 0); prop.put("cleanup_rwi", segment.termIndex() != null && !segment.termIndex().isEmpty() ? 1 : 0); prop.put("cleanup_citation", segment.connectedCitation() && !segment.urlCitation().isEmpty() ? 1 : 0); diff --git a/source/net/yacy/http/servlets/SolrServlet.java b/source/net/yacy/http/servlets/SolrServlet.java index 11c5e876b..9122d06f6 100644 --- a/source/net/yacy/http/servlets/SolrServlet.java +++ b/source/net/yacy/http/servlets/SolrServlet.java @@ -29,7 +29,6 @@ import java.io.Writer; import java.net.URL; import java.util.Date; import java.util.HashMap; -import java.util.List; import java.util.Map; import javax.servlet.Filter; @@ -51,7 +50,6 @@ import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter; import net.yacy.cora.federate.solr.responsewriter.HTMLResponseWriter; import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter; import net.yacy.cora.federate.solr.responsewriter.YJsonResponseWriter; -import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.AccessTracker; @@ -66,7 +64,6 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.DisMaxParams; import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo;