From cc770512d5d6b1eb5b88e266376ede7c779a70bd Mon Sep 17 00:00:00 2001 From: reger Date: Sun, 19 Feb 2017 05:23:17 +0100 Subject: [PATCH 1/2] add hint of query syntax in AccessTracker log (qs=normal querystring, sq=solr-querystring) to allow to filter simple text queries for processing, remove toString for counter parameter use more predefined constants in solrservlet --- .../yacy/http/servlets/GSAsearchServlet.java | 2 +- .../yacy/http/servlets/SolrSelectServlet.java | 15 ++++----- .../net/yacy/search/query/AccessTracker.java | 32 ++++++++++++------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/source/net/yacy/http/servlets/GSAsearchServlet.java b/source/net/yacy/http/servlets/GSAsearchServlet.java index 7e38e5b65..4885b9f95 100644 --- a/source/net/yacy/http/servlets/GSAsearchServlet.java +++ b/source/net/yacy/http/servlets/GSAsearchServlet.java @@ -261,7 +261,7 @@ public class GSAsearchServlet extends HttpServlet { } else if (rv != null && rv instanceof SolrDocumentList) { matches = (int) ((SolrDocumentList) rv).getNumFound(); } - AccessTracker.addToDump(originalQuery, Integer.toString(matches)); + AccessTracker.addToDump(originalQuery, matches); ConcurrentLog.info("GSA Query", "results: " + matches + ", for query:" + post.toString()); } diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index 16d8a5125..f02f670a9 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -39,7 +39,6 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import net.yacy.cora.federate.FederateSearchManager; import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; @@ -162,16 +161,16 @@ public class SolrSelectServlet extends HttpServlet { String q = mmsp.get(CommonParams.Q, ""); if (querystring.length() == 0) querystring = q; if (!mmsp.getMap().containsKey(CommonParams.START)) { - int startRecord = mmsp.getFieldInt("startRecord", null, 0); + int startRecord = mmsp.getFieldInt("startRecord", null, CommonParams.START_DEFAULT); mmsp.getMap().remove("startRecord"); mmsp.getMap().put(CommonParams.START, new String[]{Integer.toString(startRecord)}); // sru patch } if (!mmsp.getMap().containsKey(CommonParams.ROWS)) { - int maximumRecords = mmsp.getFieldInt("maximumRecords", null, 10); + int maximumRecords = mmsp.getFieldInt("maximumRecords", null, CommonParams.ROWS_DEFAULT); mmsp.getMap().remove("maximumRecords"); mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(maximumRecords)}); // sru patch } - mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(Math.min(mmsp.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100))}); + mmsp.getMap().put(CommonParams.ROWS, new String[]{Integer.toString(Math.min(mmsp.getInt(CommonParams.ROWS, CommonParams.ROWS_DEFAULT), (authenticated) ? 100000000 : 100))}); // set ranking according to profile number if ranking attributes are not given in the request Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr); @@ -207,9 +206,9 @@ public class SolrSelectServlet extends HttpServlet { if (!mmsp.getMap().containsKey("hl.simple.pre")) mmsp.getMap().put("hl.simple.pre", new String[]{""}); if (!mmsp.getMap().containsKey("hl.simple.post")) mmsp.getMap().put("hl.simple.post", new String[]{""}); if (!mmsp.getMap().containsKey("hl.fragsize")) mmsp.getMap().put("hl.fragsize", new String[]{Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)}); - if (!mmsp.getMap().containsKey("fl")) mmsp.getMap().put("fl", new String[]{ + if (!mmsp.getMap().containsKey(CommonParams.FL)) mmsp.getMap().put(CommonParams.FL, new String[]{ CollectionSchema.sku.getSolrFieldName() + "," + - CollectionSchema.title + "," + + CollectionSchema.title.getSolrFieldName() + "," + CollectionSchema.description_txt.getSolrFieldName() + "," + CollectionSchema.id.getSolrFieldName() + "," + CollectionSchema.url_paths_sxt.getSolrFieldName() + "," + @@ -254,7 +253,7 @@ public class SolrSelectServlet extends HttpServlet { // check error if (rsp.getException() != null) { - AccessTracker.addToDump(querystring, "0", new Date()); + AccessTracker.addToDump(querystring, 0, new Date(), "sq"); sendError(hresponse, rsp.getException()); return; } @@ -263,7 +262,7 @@ public class SolrSelectServlet extends HttpServlet { NamedList values = rsp.getValues(); DocList r = ((ResultContext) values.get("response")).docs; int numFound = r.matches(); - AccessTracker.addToDump(querystring, Integer.toString(numFound), new Date()); + AccessTracker.addToDump(querystring, numFound, new Date(), "sq"); // write response header final String contentType = responseWriter.getContentType(req, rsp); diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index 0753bf224..55c3a40cb 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -154,14 +154,22 @@ public class AccessTracker { private static void addToDump(final QueryParams query, int resultCount) { String queryString = query.getQueryGoal().getQueryString(false); if (queryString == null || queryString.isEmpty()) return; - addToDump(queryString, Integer.toString(resultCount), new Date(query.starttime)); + addToDump(queryString, resultCount, new Date(query.starttime), "qs"); } - public static void addToDump(String querystring, String resultcount) { - addToDump(querystring, resultcount, new Date()); + public static void addToDump(String querystring, int resultCount) { + addToDump(querystring, resultCount, new Date(), "qs"); } - public static void addToDump(String querystring, String resultcount, Date d) { + /** + * Add a line to the queries log + * + * @param querystring the original query + * @param resultcount found results + * @param d start time + * @param querySyntax used syntax (qs=normal querstring, sq=solr querystring, + */ + public static void addToDump(String querystring, int resultcount, Date d, String querySyntax) { //if (query.resultcount == 0) return; if (querystring == null || querystring.isEmpty()) return; final StringBuilder sb = new StringBuilder(40); @@ -169,6 +177,8 @@ public class AccessTracker { sb.append(' '); sb.append(resultcount); sb.append(' '); + sb.append(querySyntax); + sb.append(' '); sb.append(querystring); synchronized (log) { log.add(sb.toString()); @@ -332,15 +342,15 @@ public class AccessTracker { public static void main(String[] args) { // i.e. /Users/admin/git/rc1/DATA/LOG/queries.log 20140522135156 20140614223118 - String file = args[0]; + String file = "C:\\src\\github\\yacy_search_server\\DATA\\LOG\\queries.log"; Date from; - try { - from = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1], 0).getTime(); - Date to = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2], 0).getTime(); + // try { + from = new Date(0);//GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1], 0).getTime(); + Date to = new Date();//GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2], 0).getTime(); List dump = readLog(new File(file), from, to); for (EventTracker.Event s: dump) System.out.println(s.toString()); - } catch (ParseException e) { - e.printStackTrace(); - } + // } catch (ParseException e) { + // e.printStackTrace(); + // } } } From 334c70c37af7965d230d8f3c52b50b3060a9774f Mon Sep 17 00:00:00 2001 From: reger Date: Mon, 20 Feb 2017 00:14:14 +0100 Subject: [PATCH 2/2] correct fromDate init value on missing param in api/timeline_p servlet revert test modification from last commit in AccessTracker.main --- htroot/api/timeline_p.java | 6 +++--- source/net/yacy/search/query/AccessTracker.java | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/htroot/api/timeline_p.java b/htroot/api/timeline_p.java index b9e4991b0..c531a510b 100644 --- a/htroot/api/timeline_p.java +++ b/htroot/api/timeline_p.java @@ -75,9 +75,9 @@ public final class timeline_p { // get a time period Date fromDate = new Date(0); Date toDate = new Date(); - try {fromDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("from", "20031215182700"), 0).getTime();} catch (ParseException e) {} - try {toDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("to", GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())), 0).getTime();} catch (ParseException e) {} - + try {fromDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("from", GenericFormatter.SHORT_SECOND_FORMATTER.format(fromDate)), 0).getTime();} catch (ParseException e) {} + try {toDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("to", GenericFormatter.SHORT_SECOND_FORMATTER.format(toDate)), 0).getTime();} catch (ParseException e) {} + // get latest dump; AccessTracker.dumpLog(); diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index 55c3a40cb..46b017626 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -342,15 +342,15 @@ public class AccessTracker { public static void main(String[] args) { // i.e. /Users/admin/git/rc1/DATA/LOG/queries.log 20140522135156 20140614223118 - String file = "C:\\src\\github\\yacy_search_server\\DATA\\LOG\\queries.log"; + String file = args[0]; Date from; - // try { - from = new Date(0);//GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1], 0).getTime(); - Date to = new Date();//GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2], 0).getTime(); + try { + from = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1], 0).getTime(); + Date to = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2], 0).getTime(); List dump = readLog(new File(file), from, to); for (EventTracker.Event s: dump) System.out.println(s.toString()); - // } catch (ParseException e) { - // e.printStackTrace(); - // } + } catch (ParseException e) { + e.printStackTrace(); + } } }