boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Moved methods from SolrServerConnector to AbstractSolrConnector with the

Browse Source 
result that most of these methods become superfluous in other classes.
This is a generalization step towards multi-indexes in Solr.
pull/1/head
Michael Peter Christen 12 years ago
parent 02fa31b5bf
commit 1db23e9eac
6 changed files with 130 additions and 354 deletions
Show Stats Download Patch File Download Diff File

122
source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
Unescape View File

@ -21,20 +21,31 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.federate.solr.YaCySchema;
import net.yacy.cora.sorting.ClusteredScoreMap;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.cora.util.LookAheadIterator;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.ModifiableSolrParams;
public abstract class AbstractSolrConnector implements SolrConnector {
@ -163,4 +174,115 @@ public abstract class AbstractSolrConnector implements SolrConnector {
};
}
/**
* get a query result from solr
* to get all results set the query String to "*:*"
* @param querystring
* @throws IOException
*/
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
// construct query
final SolrQuery params = new SolrQuery();
params.setQuery(querystring);
params.setRows(count);
params.setStart(offset);
params.setFacet(false);
//params.addSortField( "price", SolrQuery.ORDER.asc );
if (fields.length > 0) params.setFields(fields);
// query the server
QueryResponse rsp = query(params);
final SolrDocumentList docs = rsp.getResults();
return docs;
}
/**
* get the number of results when this query is done.
* This should only be called if the actual result is never used, and only the count is interesting
* @param querystring
* @return the number of results for this query
*/
@Override
public long getQueryCount(String querystring) throws IOException {
// construct query
final SolrQuery params = new SolrQuery();
params.setQuery(querystring);
params.setRows(0);
params.setStart(0);
params.setFacet(false);
//params.setFields(YaCySchema.id.getSolrFieldName());
// query the server
QueryResponse rsp = query(params);
final SolrDocumentList docs = rsp.getResults();
return docs.getNumFound();
}
/**
* get facets of the index: a list of lists with values that are most common in a specific field
* @param query a query which is performed to get the facets
* @param fields the field names which are selected as facet
* @param maxresults the maximum size of the resulting maps
* @return a map with key = facet field name, value = an ordered map of field values for that field
* @throws IOException
*/
@Override
public Map<String, ReversibleScoreMap<String>> getFacets(String query, int maxresults, final String ... fields) throws IOException {
// construct query
assert fields.length > 0;
final SolrQuery params = new SolrQuery();
params.setQuery(query);
params.setRows(0);
params.setStart(0);
params.setFacet(true);
params.setFacetLimit(maxresults);
params.setFacetSort(FacetParams.FACET_SORT_COUNT);
params.setFields(fields);
for (String field: fields) params.addFacetField(field);
// query the server
QueryResponse rsp = query(params);
Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>(fields.length);
for (String field: fields) {
FacetField facet = rsp.getFacetField(field);
ReversibleScoreMap<String> result = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);
List<Count> values = facet.getValues();
if (values == null) continue;
for (Count ff: values) result.set(ff.getName(), (int) ff.getCount());
facets.put(field, result);
}
return facets;
}
@Override
abstract public QueryResponse query(ModifiableSolrParams params) throws IOException;
private final char[] queryIDTemplate = "id:\" \"".toCharArray();
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
final SolrQuery query = new SolrQuery();
assert key.length() == 12;
// construct query
char[] q = new char[17];
System.arraycopy(this.queryIDTemplate, 0, q, 0, 17);
System.arraycopy(key.toCharArray(), 0, q, 4, 12);
query.setQuery(new String(q));
query.setRows(1);
query.setStart(0);
if (fields.length > 0) query.setFields(fields);
// query the server
try {
final QueryResponse rsp = query(query);
final SolrDocumentList docs = rsp.getResults();
if (docs.isEmpty()) return null;
return docs.get(0);
} catch (final Throwable e) {
throw new IOException(e.getMessage(), e);
}
}
}

12
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -23,6 +23,7 @@ package net.yacy.cora.federate.solr.connector;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.MemoryControl;
@ -53,7 +54,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
private final static String[] confFiles = {"solrconfig.xml", "schema.xml", "stopwords.txt", "synonyms.txt", "protwords.txt", "currency.xml", "elevate.xml", "xslt/example.xsl", "xslt/json.xsl", "lang/"};
private CoreContainer cores;
private final String defaultCoreName;
private String defaultCoreName;
private SolrCore defaultCore;
private final SearchHandler requestHandler;
@ -89,7 +90,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
}
}
this.cores = new CoreContainer(storagePath.getAbsolutePath(), new File(solr_config, "solr.xml"));
this.cores = new CoreContainer(storagePath.getAbsolutePath(), new File(solr_config, "solr.xml")); // this may take indefinitely long if solr files are broken
if (this.cores == null) {
// try again
System.gc();
@ -100,8 +101,11 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
this.defaultCore = this.cores.getCore(this.defaultCoreName); // should be "collection1"
if (this.defaultCore == null) {
// try again
System.gc();
this.defaultCore = this.cores.getCore(this.defaultCoreName); // should be "collection1"
Collection<SolrCore> cores = this.cores.getCores();
if (cores.size() > 0) {
this.defaultCore = cores.iterator().next();
this.defaultCoreName = this.defaultCore.getName();
}
}
if (this.defaultCore == null) {
throw new IOException("cannot get the default core; available = " + MemoryControl.available() + ", free = " + MemoryControl.free());

118
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -21,17 +21,13 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import net.yacy.cora.federate.solr.YaCySchema;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.cora.storage.ARC;
import net.yacy.cora.storage.ConcurrentARC;
import net.yacy.kelondro.util.MemoryControl;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
@ -92,7 +88,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
this.documentCache = new ConcurrentARC<String, SolrDocument>(docCacheMax, this.partitions);
}
public HitMissCache getHitMissCache(String field) {
HitMissCache c = this.hitMissCache.get(field);
if (c == null) {
@ -356,57 +351,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
if (this.solr1 != null) this.solr1.add(solrdoc);
}
/**
* get a query result from solr
* to get all results set the query String to "*:*"
* @param querystring
* @throws IOException
*/
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
if (this.solr0 == null && this.solr1 == null) return new SolrDocumentList();
if (offset == 0 && count == 1 && querystring.startsWith("id:")) {
final SolrDocumentList list = new SolrDocumentList();
SolrDocument doc = getById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields);
list.add(doc);
// no addToCache(list) here because that was already handlet in get();
return list;
}
if (this.solr0 != null && this.solr1 == null) {
SolrDocumentList list = this.solr0.query(querystring, offset, count, fields);
addToCache(list, fields.length == 0);
return list;
}
if (this.solr1 != null && this.solr0 == null) {
SolrDocumentList list = this.solr1.query(querystring, offset, count, fields);
addToCache(list, fields.length == 0);
return list;
}
// combine both lists
SolrDocumentList l;
l = this.solr0.query(querystring, offset, count, fields);
if (l.size() >= count) return l;
// at this point we need to know how many results are in solr0
// compute this with a very bad hack; replace with better method later
int size0 = 0;
{ //bad hack - TODO: replace
SolrDocumentList lHack = this.solr0.query(querystring, 0, Integer.MAX_VALUE, fields);
size0 = lHack.size();
}
// now use the size of the first query to do a second query
final SolrDocumentList list = new SolrDocumentList();
for (final SolrDocument d: l) list.add(d);
l = this.solr1.query(querystring, offset + l.size() - size0, count - l.size(), fields);
for (final SolrDocument d: l) list.add(d);
// add caching
addToCache(list, fields.length == 0);
return list;
}
@Override
public QueryResponse query(ModifiableSolrParams query) throws IOException, SolrException {
Integer count0 = query.getInt(CommonParams.ROWS);
@ -450,68 +394,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
// TODO: combine both
return rsp1;
}
@Override
public long getQueryCount(final String querystring) throws IOException {
if (this.solr0 == null && this.solr1 == null) return 0;
if (this.solr0 != null && this.solr1 == null) {
return this.solr0.getQueryCount(querystring);
}
if (this.solr1 != null && this.solr0 == null) {
return this.solr1.getQueryCount(querystring);
}
final AtomicLong count = new AtomicLong(0);
Thread t0 = new Thread() {
@Override
public void run() {
try {
count.addAndGet(MirrorSolrConnector.this.solr0.getQueryCount(querystring));
} catch (IOException e) {}
}
};
t0.start();
Thread t1 = new Thread() {
@Override
public void run() {
try {
count.addAndGet(MirrorSolrConnector.this.solr1.getQueryCount(querystring));
} catch (IOException e) {}
}
};
t1.start();
try {t0.join();} catch (InterruptedException e) {}
try {t1.join();} catch (InterruptedException e) {}
return count.get();
}
@Override
public Map<String, ReversibleScoreMap<String>> getFacets(final String query, final int maxresults, final String ... fields) throws IOException {
if (this.solr0 == null && this.solr1 == null) return new HashMap<String, ReversibleScoreMap<String>>(0);
if (this.solr0 != null && this.solr1 == null) {
return this.solr0.getFacets(query, maxresults, fields);
}
if (this.solr1 != null && this.solr0 == null) {
return this.solr1.getFacets(query, maxresults, fields);
}
Map<String, ReversibleScoreMap<String>> facets0 = this.solr0.getFacets(query, maxresults, fields);
Map<String, ReversibleScoreMap<String>> facets1 = this.solr1.getFacets(query, maxresults, fields);
for (Map.Entry<String, ReversibleScoreMap<String>> facet0: facets0.entrySet()) {
ReversibleScoreMap<String> facet1 = facets1.remove(facet0.getKey());
if (facet1 == null) continue;
for (String key: facet1) facet0.getValue().inc(key, facet1.get(key));
}
for (Map.Entry<String, ReversibleScoreMap<String>> facet1: facets1.entrySet()) {
facets0.put(facet1.getKey(), facet1.getValue());
}
return facets0;
}
private void addToCache(SolrDocumentList list, boolean doccache) {
if (MemoryControl.shortStatus()) clearCache();
for (final SolrDocument solrdoc: list) {
addToCache(solrdoc, doccache);
}
}
private void addToCache(SolrDocument doc, boolean doccach) {
for (Map.Entry<String, HitMissCache> e: this.hitMissCache.entrySet()) {

30
source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
Unescape View File

@ -22,14 +22,9 @@ package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import net.yacy.cora.sorting.ReversibleScoreMap;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
@ -150,16 +145,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
this.solr.deleteByQuery(querystring);
}
@Override
public Object getFieldById(final String key, final String field) throws IOException {
return this.solr.getFieldById(key, field);
}
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
return this.solr.getById(key, fields);
}
@Override
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
try {
@ -169,26 +154,11 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
}
}
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
return this.solr.query(querystring, offset, count, fields);
}
@Override
public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException {
return this.solr.query(query);
}
@Override
public long getQueryCount(final String querystring) throws IOException {
return this.solr.getQueryCount(querystring);
}
@Override
public Map<String, ReversibleScoreMap<String>> getFacets(final String query, final int maxresults, final String ... fields) throws IOException {
return this.solr.getFacets(query, maxresults, fields);
}
@Override
public long getSize() {
return this.solr.getSize();

81
source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java
Unescape View File

@ -21,15 +21,9 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.yacy.cora.sorting.ReversibleScoreMap;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
@ -136,36 +130,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
}
@Override
public boolean exists(final String fieldName, final String key) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
return this.solrConnector.exists(fieldName, key);
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
return false;
}
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
return this.solrConnector.getById(key, fields);
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
return null;
}
@Override
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
@ -181,21 +145,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
}
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
return this.solrConnector.query(querystring, offset, count, fields);
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
return null;
}
@Override
public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
@ -211,36 +160,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon
return null;
}
@Override
public long getQueryCount(final String querystring) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
return this.solrConnector.getQueryCount(querystring);
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
return 0;
}
@Override
public Map<String, ReversibleScoreMap<String>> getFacets(final String query, final int maxresults, final String ... fields) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
return this.solrConnector.getFacets(query, maxresults, fields);
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
return new HashMap<String, ReversibleScoreMap<String>>();
}
@Override
public long getSize() {
final long t = System.currentTimeMillis() + this.retryMaxTime;

121
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -22,33 +22,22 @@ package net.yacy.cora.federate.solr.connector;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.federate.solr.YaCySchema;
import net.yacy.cora.sorting.ClusteredScoreMap;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.kelondro.logging.Log;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
@ -295,114 +284,4 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
}
}
/**
* get a query result from solr
* to get all results set the query String to "*:*"
* @param querystring
* @throws IOException
*/
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
// construct query
final SolrQuery params = new SolrQuery();
params.setQuery(querystring);
params.setRows(count);
params.setStart(offset);
params.setFacet(false);
//params.addSortField( "price", SolrQuery.ORDER.asc );
if (fields.length > 0) params.setFields(fields);
// query the server
QueryResponse rsp = query(params);
final SolrDocumentList docs = rsp.getResults();
return docs;
}
/**
* get the number of results when this query is done.
* This should only be called if the actual result is never used, and only the count is interesting
* @param querystring
* @return the number of results for this query
*/
@Override
public long getQueryCount(String querystring) throws IOException {
// construct query
final SolrQuery params = new SolrQuery();
params.setQuery(querystring);
params.setRows(0);
params.setStart(0);
params.setFacet(false);
//params.setFields(YaCySchema.id.getSolrFieldName());
// query the server
QueryResponse rsp = query(params);
final SolrDocumentList docs = rsp.getResults();
return docs.getNumFound();
}
/**
* get facets of the index: a list of lists with values that are most common in a specific field
* @param query a query which is performed to get the facets
* @param fields the field names which are selected as facet
* @param maxresults the maximum size of the resulting maps
* @return a map with key = facet field name, value = an ordered map of field values for that field
* @throws IOException
*/
public Map<String, ReversibleScoreMap<String>> getFacets(String query, int maxresults, final String ... fields) throws IOException {
// construct query
assert fields.length > 0;
final SolrQuery params = new SolrQuery();
params.setQuery(query);
params.setRows(0);
params.setStart(0);
params.setFacet(true);
params.setFacetLimit(maxresults);
params.setFacetSort(FacetParams.FACET_SORT_COUNT);
params.setFields(fields);
for (String field: fields) params.addFacetField(field);
// query the server
QueryResponse rsp = query(params);
Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>(fields.length);
for (String field: fields) {
FacetField facet = rsp.getFacetField(field);
ReversibleScoreMap<String> result = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);
List<Count> values = facet.getValues();
if (values == null) continue;
for (Count ff: values) result.set(ff.getName(), (int) ff.getCount());
facets.put(field, result);
}
return facets;
}
@Override
abstract public QueryResponse query(ModifiableSolrParams params) throws IOException;
private final char[] queryIDTemplate = "id:\" \"".toCharArray();
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
final SolrQuery query = new SolrQuery();
assert key.length() == 12;
// construct query
char[] q = new char[17];
System.arraycopy(this.queryIDTemplate, 0, q, 0, 17);
System.arraycopy(key.toCharArray(), 0, q, 4, 12);
query.setQuery(new String(q));
query.setRows(1);
query.setStart(0);
if (fields.length > 0) query.setFields(fields);
// query the server
try {
final QueryResponse rsp = query(query);
final SolrDocumentList docs = rsp.getResults();
if (docs.isEmpty()) return null;
return docs.get(0);
} catch (final Throwable e) {
throw new IOException(e.getMessage(), e);
}
}
}

Write Preview
Loading…
Cancel
Save