From 3834829b373393eac5368cca809ec44841825e02 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 4 Feb 2013 16:42:10 +0100 Subject: [PATCH] bugfixes and more logging for solr connector --- htroot/api/schema.java | 4 +-- .../solr/connector/EmbeddedSolrConnector.java | 1 + .../solr/connector/RemoteSolrConnector.java | 9 +++++- .../solr/connector/SolrServerConnector.java | 15 +++++----- source/net/yacy/search/query/QueryParams.java | 28 +++++++++++++------ 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/htroot/api/schema.java b/htroot/api/schema.java index 234745388..670450ff1 100644 --- a/htroot/api/schema.java +++ b/htroot/api/schema.java @@ -48,9 +48,7 @@ public class schema { c++; } } - if (solrScheme.contains(YaCySchema.author)) { - addField(prop, c, YaCySchema.author_sxt); - } + //if (solrScheme.contains(YaCySchema.author)) {addField(prop, c, YaCySchema.author_sxt);} prop.put("fields", c); prop.put("copyFieldAuthor", solrScheme.contains(YaCySchema.author) ? 1 : 0); diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java index 3f0ff028e..3b5865a1b 100644 --- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java @@ -96,6 +96,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo this.cores = new CoreContainer(storagePath.getAbsolutePath(), new File(solr_config, "solr.xml")); } this.defaultCoreName = this.cores.getDefaultCoreName(); + Log.logInfo("EmbeddedSolrConnector", "detected default solr core: " + this.defaultCoreName); this.defaultCore = this.cores.getCore(this.defaultCoreName); // should be "collection1" if (this.defaultCore == null) { // try again diff --git a/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java index 48f11261b..faf551a75 100644 --- a/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java @@ -26,6 +26,7 @@ import java.net.InetAddress; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.Domains; +import net.yacy.kelondro.logging.Log; import org.apache.commons.httpclient.HttpException; import org.apache.http.Header; @@ -48,12 +49,15 @@ import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.protocol.HttpContext; import org.apache.solr.client.solrj.ResponseParser; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; public class RemoteSolrConnector extends SolrServerConnector implements SolrConnector { @@ -130,8 +134,11 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope(this.host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(this.solraccount, this.solrpw)); this.client.setCredentialsProvider(credsProvider); - s = new HttpSolrServer("http://" + this.host + ":" + this.port + this.solrpath, this.client); + String p = "http://" + this.host + ":" + this.port + this.solrpath; + Log.logInfo("RemoteSolrConnector", "connecting Solr authenticated with url:" + p); + s = new HttpSolrServer(p, this.client); } else { + Log.logInfo("RemoteSolrConnector", "connecting Solr with url:" + this.solrurl); s = new HttpSolrServer(this.solrurl); } s.setAllowCompression(true); diff --git a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java index 88ef095a5..a26999bd8 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java @@ -30,6 +30,7 @@ import net.yacy.cora.document.UTF8; import net.yacy.cora.federate.solr.YaCySchema; import net.yacy.cora.sorting.ClusteredScoreMap; import net.yacy.cora.sorting.ReversibleScoreMap; +import net.yacy.kelondro.logging.Log; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; @@ -103,8 +104,8 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen public synchronized void commit(final boolean softCommit) { try { this.server.commit(true, true, softCommit); - } catch (SolrServerException e) { - } catch (IOException e) { + } catch (Throwable e) { + Log.logException(e); } } @@ -115,8 +116,8 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen public void optimize(int maxSegments) { try { this.server.optimize(true, true, maxSegments); - } catch (SolrServerException e) { - } catch (IOException e) { + } catch (Throwable e) { + Log.logException(e); } } @@ -125,10 +126,8 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen try { if (this.server != null) synchronized (this.server) {this.server.commit(true, true, false);} this.server = null; - } catch (SolrServerException e) { - log.warn(e); - } catch (IOException e) { - log.warn(e); + } catch (Throwable e) { + Log.logException(e); } } diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 753be5ec8..a5bfc79b5 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -67,6 +67,7 @@ import net.yacy.kelondro.util.Bitfield; import net.yacy.kelondro.util.SetTools; import net.yacy.peers.Seed; import net.yacy.search.index.Segment; +import net.yacy.search.index.SolrConfiguration; import net.yacy.search.ranking.RankingProfile; public final class QueryParams { @@ -83,11 +84,8 @@ public final class QueryParams { } } - private static final String[] defaultfacetfields = new String[]{ - YaCySchema.host_s.getSolrFieldName(), - YaCySchema.url_protocol_s.getSolrFieldName(), - YaCySchema.url_file_ext_s.getSolrFieldName(), - YaCySchema.author_sxt.getSolrFieldName()}; + private static final YaCySchema[] defaultfacetfields = new YaCySchema[]{ + YaCySchema.host_s, YaCySchema.url_protocol_s, YaCySchema.url_file_ext_s, YaCySchema.author_sxt}; private static final int defaultmaxfacets = 30; @@ -146,6 +144,7 @@ public final class QueryParams { public List facetfields; public int maxfacets; private SolrQuery cachedQuery; + private SolrConfiguration solrScheme; // the following values are filled during the search process as statistics for the search public final AtomicInteger local_rwi_available; // the number of hits generated/ranked by the local search in rwi index @@ -158,7 +157,8 @@ public final class QueryParams { public final SortedSet misses; // url hashes that had been sorted out because of constraints in postranking public QueryParams( - final String query_original, final String query_words, + final String query_original, + final String query_words, final int itemsPerPage, final Bitfield constraint, final Segment indexSegment, @@ -210,7 +210,12 @@ public final class QueryParams { this.remote_available = new AtomicInteger(0); // the number of result contributions from all the remote peers this.remote_peerCount = new AtomicInteger(0); // the number of remote peers that have contributed this.misses = Collections.synchronizedSortedSet(new TreeSet(URIMetadataRow.rowdef.objectOrder)); - this.facetfields = new ArrayList(); for (String f: defaultfacetfields) facetfields.add(f); + this.facetfields = new ArrayList(); + + this.solrScheme = indexSegment.fulltext().getSolrScheme(); + for (YaCySchema f: defaultfacetfields) { + if (solrScheme.contains(f)) facetfields.add(f.getSolrFieldName()); + } for (Tagging v: LibraryProvider.autotagging.getVocabularies()) this.facetfields.add(YaCySchema.VOCABULARY_PREFIX + v.getName() + YaCySchema.VOCABULARY_SUFFIX); this.maxfacets = defaultmaxfacets; this.cachedQuery = null; @@ -312,7 +317,12 @@ public final class QueryParams { this.remote_available = new AtomicInteger(0); // the number of result contributions from all the remote peers this.remote_peerCount = new AtomicInteger(0); // the number of remote peers that have contributed this.misses = Collections.synchronizedSortedSet(new TreeSet(URIMetadataRow.rowdef.objectOrder)); - this.facetfields = new ArrayList(); for (String f: defaultfacetfields) facetfields.add(f); + this.facetfields = new ArrayList(); + + this.solrScheme = indexSegment.fulltext().getSolrScheme(); + for (YaCySchema f: defaultfacetfields) { + if (solrScheme.contains(f)) facetfields.add(f.getSolrFieldName()); + } for (Tagging v: LibraryProvider.autotagging.getVocabularies()) this.facetfields.add(YaCySchema.VOCABULARY_PREFIX + v.getName() + YaCySchema.VOCABULARY_SUFFIX); this.maxfacets = defaultmaxfacets; this.cachedQuery = null; @@ -477,7 +487,7 @@ public final class QueryParams { } // add author facets - if (this.author != null && this.author.length() > 0) { + if (this.author != null && this.author.length() > 0 && this.solrScheme.contains(YaCySchema.author_sxt)) { fq.append(" AND ").append(YaCySchema.author_sxt.getSolrFieldName()).append(":\"").append(this.author).append('\"'); }