From 35fa718b77ad415fe8c9b02201373994ef45aed6 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 25 Feb 2013 14:31:50 +0100 Subject: [PATCH] testing to use solr for portalsearch caused some bugfixing but no full success: try to comment out the solr search request in yacy-portalsearch.js --- htroot/portalsearch/yacy-portalsearch.js | 5 +++-- htroot/solr/select.java | 17 +++++++++++------ source/net/yacy/kelondro/logging/Log.java | 2 +- source/net/yacy/search/query/QueryGoal.java | 2 ++ source/net/yacy/server/serverObjects.java | 15 +++++++++++++-- .../yacy/upnp/impls/InternetGatewayDevice.java | 3 --- 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/htroot/portalsearch/yacy-portalsearch.js b/htroot/portalsearch/yacy-portalsearch.js index d5ff27736..dbaf9d6ed 100644 --- a/htroot/portalsearch/yacy-portalsearch.js +++ b/htroot/portalsearch/yacy-portalsearch.js @@ -207,6 +207,7 @@ function yrun() { function yacysearch(clear) { var url = yconf.url + '/yacysearch.json?callback=?' // JSONP (cross domain) request URL + //var url = yconf.url + '/solr/select?wt=yjson&jsonp=?' // JSONP (cross domain) request URL if(clear) { $('#ypopup').empty(); @@ -244,7 +245,7 @@ function yacysearch(clear) { $.ajaxSetup({ timeout: 10000, - error: function(x,e) { + error: function(x,e,ex) { var err = 'Unknow Error: '+x.responseText; if(x.status==0) { err = 'Unknown Network Error! I try to reload...'; @@ -254,7 +255,7 @@ function yacysearch(clear) { } else if(x.status==500) { err = x.status + ' - Internel Server Error.'; } else if(e=='parsererror') { - err = 'Parsing JSON Request failed.'; + err = 'Parsing JSON Request failed:' + ex; } else if(e=='timeout') { err = 'Request Time out.'; }; diff --git a/htroot/solr/select.java b/htroot/solr/select.java index 96ba19905..4070be1f1 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -40,6 +40,7 @@ import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.AccessTracker; +import net.yacy.search.query.QueryGoal; import net.yacy.search.query.QueryModifier; import net.yacy.search.query.SearchEvent; import net.yacy.search.schema.CollectionSchema; @@ -142,7 +143,6 @@ public class select { // check post if (post == null) return null; - Log.logInfo("SOLR Query", post.toString()); sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time // rename post fields according to result style @@ -152,12 +152,14 @@ public class select { QueryModifier modifier = new QueryModifier(); querystring = modifier.parse(querystring); modifier.apply(post); - post.put(CommonParams.Q, querystring); // sru patch + QueryGoal qg = new QueryGoal(querystring, querystring); + StringBuilder solrQ = qg.solrQueryString(sb.index.fulltext().getDefaultConfiguration()); + post.put(CommonParams.Q, solrQ.toString()); // sru patch } String q = post.get(CommonParams.Q, ""); - if (!post.containsKey(CommonParams.START)) post.put(CommonParams.START, post.remove("startRecord")); // sru patch - post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, post.getInt("maximumRecords", 10)), (authenticated) ? 5000 : 100)); - post.remove("maximumRecords"); + if (!post.containsKey(CommonParams.START)) post.put(CommonParams.START, post.remove("startRecord", 0)); // sru patch + if (!post.containsKey(CommonParams.ROWS)) post.put(CommonParams.ROWS, post.remove("maximumRecords", 10)); // sru patch + post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 10000 : 100)); // get a response writer for the result String wt = post.get(CommonParams.WT, "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml @@ -212,9 +214,12 @@ public class select { // log result Object rv = response.getValues().get("response"); + int matches = ((ResultContext) rv).docs.matches(); if (rv != null && rv instanceof ResultContext) { - AccessTracker.addToDump(q, Integer.toString(((ResultContext) rv).docs.matches())); + AccessTracker.addToDump(q, Integer.toString(matches)); } + + Log.logInfo("SOLR Query", "results: " + matches + ", for query:" + post.toString()); return null; } } diff --git a/source/net/yacy/kelondro/logging/Log.java b/source/net/yacy/kelondro/logging/Log.java index fc3e0e21c..d76f079de 100644 --- a/source/net/yacy/kelondro/logging/Log.java +++ b/source/net/yacy/kelondro/logging/Log.java @@ -295,7 +295,7 @@ public final class Log { private Throwable thrown; private logEntry(final Level level, final String message) { this.level = level; - this.message = message == null || message.length() <= 512 ? message : message.substring(0, 512); + this.message = message == null || message.length() <= 1024 ? message : message.substring(0, 1024); } public logEntry(final Logger logger, final Level level, final String message, final Throwable thrown) { this(level, message); diff --git a/source/net/yacy/search/query/QueryGoal.java b/source/net/yacy/search/query/QueryGoal.java index 66afd280e..9ec29ae43 100644 --- a/source/net/yacy/search/query/QueryGoal.java +++ b/source/net/yacy/search/query/QueryGoal.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.SortedSet; import net.yacy.cora.federate.solr.Boost; +import net.yacy.cora.federate.solr.SolrType; import net.yacy.cora.storage.HandleSet; import net.yacy.document.parser.html.AbstractScraper; import net.yacy.document.parser.html.CharacterCoding; @@ -234,6 +235,7 @@ public class QueryGoal { CollectionSchema field = entry.getKey(); if (entry.getValue().floatValue() < 0.0f) continue; if (configuration != null && !configuration.contains(field.getSolrFieldName())) continue; + if (field.getType() == SolrType.num_integer) continue; if (wc > 0) q.append(" OR "); q.append('('); q.append(field.getSolrFieldName()).append(':').append(w); diff --git a/source/net/yacy/server/serverObjects.java b/source/net/yacy/server/serverObjects.java index 1bdd545ab..4d4b3e3bd 100644 --- a/source/net/yacy/server/serverObjects.java +++ b/source/net/yacy/server/serverObjects.java @@ -153,11 +153,22 @@ public class serverObjects implements Serializable, Cloneable { public Set keySet() { return this.map.getMap().keySet(); } - + public String[] remove(String key) { return this.map.getMap().remove(key); } + public int remove(String key, int dflt) { + final String result = removeByteOrderMark(get(key)); + this.map.getMap().remove(key); + if (result == null) return dflt; + try { + return Integer.parseInt(result); + } catch (NumberFormatException e) { + return dflt; + } + } + public void putAll(Map m) { for (Map.Entry e: m.entrySet()) { put(e.getKey(), e.getValue()); @@ -372,7 +383,7 @@ public class serverObjects implements Serializable, Cloneable { public String get(String name) { String[] arr = map.getMap().get(name); - return arr==null ? null : arr[0]; + return arr == null || arr.length == 0 ? null : arr[0]; } // new get with default objects diff --git a/source/net/yacy/upnp/impls/InternetGatewayDevice.java b/source/net/yacy/upnp/impls/InternetGatewayDevice.java index 118861fcc..cb97c3780 100644 --- a/source/net/yacy/upnp/impls/InternetGatewayDevice.java +++ b/source/net/yacy/upnp/impls/InternetGatewayDevice.java @@ -65,9 +65,6 @@ import net.yacy.upnp.messages.UPNPMessageFactory; import net.yacy.upnp.messages.UPNPResponseException; import net.yacy.upnp.services.UPNPService; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - /** * This class can be used to access some funtionalities on the * InternetGatewayDevice on your network without having to know