boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring (renaming) of yacy-solr api

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent 3a0fcfbeda
commit 3502b4c697
14 changed files with 79 additions and 79 deletions
Show Stats Download Patch File Download Diff File

2
htroot/HostBrowser.java
Unescape View File

@ -258,7 +258,7 @@ public class HostBrowser {
q.append(" AND ").append(CollectionSchema.url_paths_sxt.getSolrFieldName()).append(":[* TO *]"); q.append(" AND ").append(CollectionSchema.url_paths_sxt.getSolrFieldName()).append(":[* TO *]");
} }
} }
BlockingQueue<SolrDocument> docs = fulltext.getDefaultConnector().concurrentQuery(q.toString(), 0, 100000, TIMEOUT, 100, BlockingQueue<SolrDocument> docs = fulltext.getDefaultConnector().concurrentDocumentsByQuery(q.toString(), 0, 100000, TIMEOUT, 100,
CollectionSchema.id.getSolrFieldName(), CollectionSchema.id.getSolrFieldName(),
CollectionSchema.sku.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName(),
CollectionSchema.failreason_t.getSolrFieldName(), CollectionSchema.failreason_t.getSolrFieldName(),

4
source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java
Unescape View File

@ -201,7 +201,7 @@ public class OpenSearchConnector {
final long numfound; final long numfound;
try { try {
SolrDocumentList docList = connector.query(webgraphquerystr, 0, 1, webgraphqueryfields); SolrDocumentList docList = connector.getDocumentListByQuery(webgraphquerystr, 0, 1, webgraphqueryfields);
numfound = docList.getNumFound(); numfound = docList.getNumFound();
if (numfound == 0) { if (numfound == 0) {
Log.logInfo("OpenSearchConnector.Discover", "no results found, abort discover job"); Log.logInfo("OpenSearchConnector.Discover", "no results found, abort discover job");
@ -226,7 +226,7 @@ public class OpenSearchConnector {
Set<String> dblmem = new HashSet<String>(); // temp memory for already checked url Set<String> dblmem = new HashSet<String>(); // temp memory for already checked url
while (doloop) { while (doloop) {
Log.logInfo("OpenSearchConnector.Discover", "start Solr query loop at " + Integer.toString(loopnr * 20) + " of " + Long.toString(numfound)); Log.logInfo("OpenSearchConnector.Discover", "start Solr query loop at " + Integer.toString(loopnr * 20) + " of " + Long.toString(numfound));
SolrDocumentList docList = connector.query(webgraphquerystr, loopnr * 20, 20,webgraphqueryfields); // check chunk of 20 result documents SolrDocumentList docList = connector.getDocumentListByQuery(webgraphquerystr, loopnr * 20, 20,webgraphqueryfields); // check chunk of 20 result documents
loopnr++; loopnr++;
if (stoptime < System.currentTimeMillis()) {// stop after max 1h if (stoptime < System.currentTimeMillis()) {// stop after max 1h
doloop = false; doloop = false;

30
source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
Unescape View File

@ -74,7 +74,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
@Override @Override
public boolean existsByQuery(final String query) throws IOException { public boolean existsByQuery(final String query) throws IOException {
try { try {
long count = getQueryCount(query); long count = getCountByQuery(query);
return count > 0; return count > 0;
} catch (final Throwable e) { } catch (final Throwable e) {
return false; return false;
@ -83,7 +83,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
@Override @Override
public Object getFieldById(final String key, final String field) throws IOException { public Object getFieldById(final String key, final String field) throws IOException {
SolrDocument doc = getById(key, field); SolrDocument doc = getDocumentById(key, field);
if (doc == null) return null; if (doc == null) return null;
return doc.getFieldValue(field); return doc.getFieldValue(field);
} }
@ -100,7 +100,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
* @return a blocking queue which is terminated with AbstractSolrConnector.POISON_DOCUMENT as last element * @return a blocking queue which is terminated with AbstractSolrConnector.POISON_DOCUMENT as last element
*/ */
@Override @Override
public BlockingQueue<SolrDocument> concurrentQuery(final String querystring, final int offset, final int maxcount, final long maxtime, final int buffersize, final String ... fields) { public BlockingQueue<SolrDocument> concurrentDocumentsByQuery(final String querystring, final int offset, final int maxcount, final long maxtime, final int buffersize, final String ... fields) {
final BlockingQueue<SolrDocument> queue = buffersize <= 0 ? new LinkedBlockingQueue<SolrDocument>() : new ArrayBlockingQueue<SolrDocument>(buffersize); final BlockingQueue<SolrDocument> queue = buffersize <= 0 ? new LinkedBlockingQueue<SolrDocument>() : new ArrayBlockingQueue<SolrDocument>(buffersize);
final long endtime = System.currentTimeMillis() + maxtime; final long endtime = System.currentTimeMillis() + maxtime;
final Thread t = new Thread() { final Thread t = new Thread() {
@ -110,7 +110,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
int count = 0; int count = 0;
while (System.currentTimeMillis() < endtime && count < maxcount) { while (System.currentTimeMillis() < endtime && count < maxcount) {
try { try {
SolrDocumentList sdl = query(querystring, o, pagesize, fields); SolrDocumentList sdl = getDocumentListByQuery(querystring, o, pagesize, fields);
for (SolrDocument d: sdl) { for (SolrDocument d: sdl) {
try {queue.put(d);} catch (InterruptedException e) {break;} try {queue.put(d);} catch (InterruptedException e) {break;}
count++; count++;
@ -131,7 +131,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
} }
@Override @Override
public BlockingQueue<String> concurrentIDs(final String querystring, final int offset, final int maxcount, final long maxtime) { public BlockingQueue<String> concurrentIDsByQuery(final String querystring, final int offset, final int maxcount, final long maxtime) {
final BlockingQueue<String> queue = new LinkedBlockingQueue<String>(); final BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
final long endtime = System.currentTimeMillis() + maxtime; final long endtime = System.currentTimeMillis() + maxtime;
final Thread t = new Thread() { final Thread t = new Thread() {
@ -140,7 +140,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
int o = offset; int o = offset;
while (System.currentTimeMillis() < endtime) { while (System.currentTimeMillis() < endtime) {
try { try {
SolrDocumentList sdl = query(querystring, o, pagesize, CollectionSchema.id.getSolrFieldName()); SolrDocumentList sdl = getDocumentListByQuery(querystring, o, pagesize, CollectionSchema.id.getSolrFieldName());
for (SolrDocument d: sdl) { for (SolrDocument d: sdl) {
try {queue.put((String) d.getFieldValue(CollectionSchema.id.getSolrFieldName()));} catch (InterruptedException e) {break;} try {queue.put((String) d.getFieldValue(CollectionSchema.id.getSolrFieldName()));} catch (InterruptedException e) {break;}
} }
@ -161,7 +161,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
@Override @Override
public Iterator<String> iterator() { public Iterator<String> iterator() {
final BlockingQueue<String> queue = concurrentIDs("*:*", 0, Integer.MAX_VALUE, 60000); final BlockingQueue<String> queue = concurrentIDsByQuery("*:*", 0, Integer.MAX_VALUE, 60000);
return new LookAheadIterator<String>() { return new LookAheadIterator<String>() {
@Override @Override
protected String next0() { protected String next0() {
@ -184,7 +184,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
* @throws IOException * @throws IOException
*/ */
@Override @Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException { public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
// construct query // construct query
final SolrQuery params = new SolrQuery(); final SolrQuery params = new SolrQuery();
params.setQuery(querystring); params.setQuery(querystring);
@ -196,7 +196,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
if (fields.length > 0) params.setFields(fields); if (fields.length > 0) params.setFields(fields);
// query the server // query the server
QueryResponse rsp = query(params); QueryResponse rsp = getResponseByParams(params);
final SolrDocumentList docs = rsp.getResults(); final SolrDocumentList docs = rsp.getResults();
return docs; return docs;
} }
@ -208,7 +208,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
* @return the number of results for this query * @return the number of results for this query
*/ */
@Override @Override
public long getQueryCount(String querystring) throws IOException { public long getCountByQuery(String querystring) throws IOException {
// construct query // construct query
final SolrQuery params = new SolrQuery(); final SolrQuery params = new SolrQuery();
params.setQuery(querystring); params.setQuery(querystring);
@ -218,7 +218,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
params.setFields(CollectionSchema.id.getSolrFieldName()); params.setFields(CollectionSchema.id.getSolrFieldName());
// query the server // query the server
QueryResponse rsp = query(params); QueryResponse rsp = getResponseByParams(params);
final SolrDocumentList docs = rsp.getResults(); final SolrDocumentList docs = rsp.getResults();
return docs == null ? 0 : docs.getNumFound(); return docs == null ? 0 : docs.getNumFound();
} }
@ -246,7 +246,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
for (String field: fields) params.addFacetField(field); for (String field: fields) params.addFacetField(field);
// query the server // query the server
QueryResponse rsp = query(params); QueryResponse rsp = getResponseByParams(params);
Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>(fields.length); Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>(fields.length);
for (String field: fields) { for (String field: fields) {
FacetField facet = rsp.getFacetField(field); FacetField facet = rsp.getFacetField(field);
@ -260,12 +260,12 @@ public abstract class AbstractSolrConnector implements SolrConnector {
} }
@Override @Override
abstract public QueryResponse query(ModifiableSolrParams params) throws IOException; abstract public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException;
private final char[] queryIDTemplate = "id:\" \"".toCharArray(); private final char[] queryIDTemplate = "id:\" \"".toCharArray();
@Override @Override
public SolrDocument getById(final String key, final String ... fields) throws IOException { public SolrDocument getDocumentById(final String key, final String ... fields) throws IOException {
final SolrQuery query = new SolrQuery(); final SolrQuery query = new SolrQuery();
assert key.length() == 12; assert key.length() == 12;
// construct query // construct query
@ -279,7 +279,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
// query the server // query the server
try { try {
final QueryResponse rsp = query(query); final QueryResponse rsp = getResponseByParams(query);
final SolrDocumentList docs = rsp.getResults(); final SolrDocumentList docs = rsp.getResults();
if (docs.isEmpty()) return null; if (docs.isEmpty()) return null;
return docs.get(0); return docs.get(0);

22
source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java
Unescape View File

@ -149,7 +149,7 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
} }
@Override @Override
public SolrDocument getById(final String id, final String ... fields) throws IOException { public SolrDocument getDocumentById(final String id, final String ... fields) throws IOException {
String q = idQuery(id); String q = idQuery(id);
SolrDocument doc = fields.length == 0 ? this.documentCache.get(q) : null; SolrDocument doc = fields.length == 0 ? this.documentCache.get(q) : null;
if (doc != null) { if (doc != null) {
@ -162,14 +162,14 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
return null; return null;
} }
this.missCache_Miss++; this.missCache_Miss++;
if (solr != null && ((doc = solr.getById(id, fields)) != null)) { if (solr != null && ((doc = solr.getDocumentById(id, fields)) != null)) {
addToCache(doc, fields.length == 0); addToCache(doc, fields.length == 0);
return doc; return doc;
} }
// check if there is a autocommit problem // check if there is a autocommit problem
if (this.hitCache.containsKey(q)) { if (this.hitCache.containsKey(q)) {
// the document should be there, therefore make a commit and check again // the document should be there, therefore make a commit and check again
if (solr != null && ((doc = solr.getById(id, fields)) != null)) { if (solr != null && ((doc = solr.getDocumentById(id, fields)) != null)) {
addToCache(doc, fields.length == 0); addToCache(doc, fields.length == 0);
} }
} }
@ -218,23 +218,23 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
* @throws IOException * @throws IOException
*/ */
@Override @Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException { public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
if (offset == 0 && count == 1 && querystring.startsWith("id:")) { if (offset == 0 && count == 1 && querystring.startsWith("id:")) {
final SolrDocumentList list = new SolrDocumentList(); final SolrDocumentList list = new SolrDocumentList();
SolrDocument doc = getById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields); SolrDocument doc = getDocumentById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields);
list.add(doc); list.add(doc);
// no addToCache(list) here because that was already handlet in get(); // no addToCache(list) here because that was already handlet in get();
return list; return list;
} }
if (this.solr != null) { if (this.solr != null) {
SolrDocumentList list = this.solr.query(querystring, offset, count, fields); SolrDocumentList list = this.solr.getDocumentListByQuery(querystring, offset, count, fields);
addToCache(list, fields.length == 0); addToCache(list, fields.length == 0);
return list; return list;
} }
// combine both lists // combine both lists
SolrDocumentList list; SolrDocumentList list;
list = this.solr.query(querystring, offset, count, fields); list = this.solr.getDocumentListByQuery(querystring, offset, count, fields);
// add caching // add caching
addToCache(list, fields.length == 0); addToCache(list, fields.length == 0);
@ -242,14 +242,14 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
} }
@Override @Override
public QueryResponse query(ModifiableSolrParams query) throws IOException, SolrException { public QueryResponse getResponseByParams(ModifiableSolrParams query) throws IOException, SolrException {
QueryResponse list = this.solr.query(query); QueryResponse list = this.solr.getResponseByParams(query);
return list; return list;
} }
@Override @Override
public long getQueryCount(final String querystring) throws IOException { public long getCountByQuery(final String querystring) throws IOException {
return this.solr.getQueryCount(querystring); return this.solr.getCountByQuery(querystring);
} }
@Override @Override

2
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -183,7 +183,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
} }
@Override @Override
public QueryResponse query(ModifiableSolrParams params) throws IOException { public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException {
if (this.server == null) throw new IOException("server disconnected"); if (this.server == null) throw new IOException("server disconnected");
try { try {
// during the solr query we set the thread name to the query string to get more debugging info in thread dumps // during the solr query we set the thread name to the query string to get more debugging info in thread dumps

40
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -153,9 +153,9 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
} }
@Override @Override
public SolrDocument getById(final String key, final String ... fields) throws IOException { public SolrDocument getDocumentById(final String key, final String ... fields) throws IOException {
SolrDocument doc; SolrDocument doc;
if ((solr0 != null && ((doc = solr0.getById(key, fields)) != null)) || (solr1 != null && ((doc = solr1.getById(key, fields)) != null))) { if ((solr0 != null && ((doc = solr0.getDocumentById(key, fields)) != null)) || (solr1 != null && ((doc = solr1.getDocumentById(key, fields)) != null))) {
return doc; return doc;
} }
return null; return null;
@ -185,48 +185,48 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
* @throws IOException * @throws IOException
*/ */
@Override @Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException { public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
if (this.solr0 == null && this.solr1 == null) return new SolrDocumentList(); if (this.solr0 == null && this.solr1 == null) return new SolrDocumentList();
if (offset == 0 && count == 1 && querystring.startsWith("id:")) { if (offset == 0 && count == 1 && querystring.startsWith("id:")) {
final SolrDocumentList list = new SolrDocumentList(); final SolrDocumentList list = new SolrDocumentList();
SolrDocument doc = getById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields); SolrDocument doc = getDocumentById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields);
list.add(doc); list.add(doc);
// no addToCache(list) here because that was already handlet in get(); // no addToCache(list) here because that was already handlet in get();
return list; return list;
} }
if (this.solr0 != null && this.solr1 == null) { if (this.solr0 != null && this.solr1 == null) {
SolrDocumentList list = this.solr0.query(querystring, offset, count, fields); SolrDocumentList list = this.solr0.getDocumentListByQuery(querystring, offset, count, fields);
return list; return list;
} }
if (this.solr1 != null && this.solr0 == null) { if (this.solr1 != null && this.solr0 == null) {
SolrDocumentList list = this.solr1.query(querystring, offset, count, fields); SolrDocumentList list = this.solr1.getDocumentListByQuery(querystring, offset, count, fields);
return list; return list;
} }
// combine both lists // combine both lists
SolrDocumentList l; SolrDocumentList l;
l = this.solr0.query(querystring, offset, count, fields); l = this.solr0.getDocumentListByQuery(querystring, offset, count, fields);
if (l.size() >= count) return l; if (l.size() >= count) return l;
// at this point we need to know how many results are in solr0 // at this point we need to know how many results are in solr0
// compute this with a very bad hack; replace with better method later // compute this with a very bad hack; replace with better method later
int size0 = 0; int size0 = 0;
{ //bad hack - TODO: replace { //bad hack - TODO: replace
SolrDocumentList lHack = this.solr0.query(querystring, 0, Integer.MAX_VALUE, fields); SolrDocumentList lHack = this.solr0.getDocumentListByQuery(querystring, 0, Integer.MAX_VALUE, fields);
size0 = lHack.size(); size0 = lHack.size();
} }
// now use the size of the first query to do a second query // now use the size of the first query to do a second query
final SolrDocumentList list = new SolrDocumentList(); final SolrDocumentList list = new SolrDocumentList();
for (final SolrDocument d: l) list.add(d); for (final SolrDocument d: l) list.add(d);
l = this.solr1.query(querystring, offset + l.size() - size0, count - l.size(), fields); l = this.solr1.getDocumentListByQuery(querystring, offset + l.size() - size0, count - l.size(), fields);
for (final SolrDocument d: l) list.add(d); for (final SolrDocument d: l) list.add(d);
return list; return list;
} }
@Override @Override
public QueryResponse query(ModifiableSolrParams query) throws IOException, SolrException { public QueryResponse getResponseByParams(ModifiableSolrParams query) throws IOException, SolrException {
Integer count0 = query.getInt(CommonParams.ROWS); Integer count0 = query.getInt(CommonParams.ROWS);
int count = count0 == null ? 10 : count0.intValue(); int count = count0 == null ? 10 : count0.intValue();
Integer start0 = query.getInt(CommonParams.START); Integer start0 = query.getInt(CommonParams.START);
@ -234,16 +234,16 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
if (this.solr0 == null && this.solr1 == null) return new QueryResponse(); if (this.solr0 == null && this.solr1 == null) return new QueryResponse();
if (this.solr0 != null && this.solr1 == null) { if (this.solr0 != null && this.solr1 == null) {
QueryResponse list = this.solr0.query(query); QueryResponse list = this.solr0.getResponseByParams(query);
return list; return list;
} }
if (this.solr1 != null && this.solr0 == null) { if (this.solr1 != null && this.solr0 == null) {
QueryResponse list = this.solr1.query(query); QueryResponse list = this.solr1.getResponseByParams(query);
return list; return list;
} }
// combine both lists // combine both lists
QueryResponse rsp = this.solr0.query(query); QueryResponse rsp = this.solr0.getResponseByParams(query);
final SolrDocumentList l = rsp.getResults(); final SolrDocumentList l = rsp.getResults();
if (l.size() >= count) return rsp; if (l.size() >= count) return rsp;
@ -253,7 +253,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
{ //bad hack - TODO: replace { //bad hack - TODO: replace
query.set(CommonParams.START, 0); query.set(CommonParams.START, 0);
query.set(CommonParams.ROWS, Integer.MAX_VALUE); query.set(CommonParams.ROWS, Integer.MAX_VALUE);
QueryResponse lHack = this.solr0.query(query); QueryResponse lHack = this.solr0.getResponseByParams(query);
query.set(CommonParams.START, start); query.set(CommonParams.START, start);
query.set(CommonParams.ROWS, count); query.set(CommonParams.ROWS, count);
size0 = lHack.getResults().size(); size0 = lHack.getResults().size();
@ -262,7 +262,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
// now use the size of the first query to do a second query // now use the size of the first query to do a second query
query.set(CommonParams.START, start + l.size() - size0); query.set(CommonParams.START, start + l.size() - size0);
query.set(CommonParams.ROWS, count - l.size()); query.set(CommonParams.ROWS, count - l.size());
QueryResponse rsp1 = this.solr1.query(query); QueryResponse rsp1 = this.solr1.getResponseByParams(query);
query.set(CommonParams.START, start); query.set(CommonParams.START, start);
query.set(CommonParams.ROWS, count); query.set(CommonParams.ROWS, count);
// TODO: combine both // TODO: combine both
@ -270,20 +270,20 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
} }
@Override @Override
public long getQueryCount(final String querystring) throws IOException { public long getCountByQuery(final String querystring) throws IOException {
if (this.solr0 == null && this.solr1 == null) return 0; if (this.solr0 == null && this.solr1 == null) return 0;
if (this.solr0 != null && this.solr1 == null) { if (this.solr0 != null && this.solr1 == null) {
return this.solr0.getQueryCount(querystring); return this.solr0.getCountByQuery(querystring);
} }
if (this.solr1 != null && this.solr0 == null) { if (this.solr1 != null && this.solr0 == null) {
return this.solr1.getQueryCount(querystring); return this.solr1.getCountByQuery(querystring);
} }
final AtomicLong count = new AtomicLong(0); final AtomicLong count = new AtomicLong(0);
Thread t0 = new Thread() { Thread t0 = new Thread() {
@Override @Override
public void run() { public void run() {
try { try {
count.addAndGet(MirrorSolrConnector.this.solr0.getQueryCount(querystring)); count.addAndGet(MirrorSolrConnector.this.solr0.getCountByQuery(querystring));
} catch (IOException e) {} } catch (IOException e) {}
} }
}; };
@ -292,7 +292,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
@Override @Override
public void run() { public void run() {
try { try {
count.addAndGet(MirrorSolrConnector.this.solr1.getQueryCount(querystring)); count.addAndGet(MirrorSolrConnector.this.solr1.getCountByQuery(querystring));
} catch (IOException e) {} } catch (IOException e) {}
} }
}; };

2
source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java
Unescape View File

@ -72,7 +72,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
} }
@Override @Override
public QueryResponse query(ModifiableSolrParams params) throws IOException { public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException {
// during the solr query we set the thread name to the query string to get more debugging info in thread dumps // during the solr query we set the thread name to the query string to get more debugging info in thread dumps
String q = params.get("q"); String q = params.get("q");
String threadname = Thread.currentThread().getName(); String threadname = Thread.currentThread().getName();

26
source/net/yacy/cora/federate/solr/connector/SolrConnector.java
Unescape View File

@ -36,6 +36,12 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ModifiableSolrParams;
public interface SolrConnector extends Iterable<String> /* Iterable of document IDs */ { public interface SolrConnector extends Iterable<String> /* Iterable of document IDs */ {
/**
* get the size of the index
* @return number of results if solr is queries with a catch-all pattern
*/
public long getSize();
/** /**
* force a commit * force a commit
@ -121,14 +127,14 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @return one result or null if no result exists * @return one result or null if no result exists
* @throws IOException * @throws IOException
*/ */
public SolrDocument getById(final String key, final String ... fields) throws IOException; public SolrDocument getDocumentById(final String key, final String ... fields) throws IOException;
/** /**
* get a query result from solr * get a query response from solr
* @param query * @param query
* @throws IOException * @throws IOException
*/ */
public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException; public QueryResponse getResponseByParams(final ModifiableSolrParams query) throws IOException, SolrException;
/** /**
* get a query result from solr * get a query result from solr
@ -139,7 +145,7 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @param fields list of fields * @param fields list of fields
* @throws IOException * @throws IOException
*/ */
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException, SolrException; public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException, SolrException;
/** /**
* get the number of results when this query is done. * get the number of results when this query is done.
@ -147,7 +153,7 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @param querystring * @param querystring
* @return the number of results for this query * @return the number of results for this query
*/ */
public long getQueryCount(final String querystring) throws IOException; public long getCountByQuery(final String querystring) throws IOException;
/** /**
* get facets of the index: a list of lists with values that are most common in a specific field * get facets of the index: a list of lists with values that are most common in a specific field
@ -171,7 +177,7 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @param fields list of fields * @param fields list of fields
* @return a blocking queue which is terminated with AbstractSolrConnector.POISON_DOCUMENT as last element * @return a blocking queue which is terminated with AbstractSolrConnector.POISON_DOCUMENT as last element
*/ */
public BlockingQueue<SolrDocument> concurrentQuery(final String querystring, final int offset, final int maxcount, final long maxtime, final int buffersize, final String ... fields); public BlockingQueue<SolrDocument> concurrentDocumentsByQuery(final String querystring, final int offset, final int maxcount, final long maxtime, final int buffersize, final String ... fields);
/** /**
* get a document id result stream from a solr query. * get a document id result stream from a solr query.
@ -182,12 +188,6 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @param maxcount * @param maxcount
* @return * @return
*/ */
public BlockingQueue<String> concurrentIDs(final String querystring, final int offset, final int maxcount, final long maxtime); public BlockingQueue<String> concurrentIDsByQuery(final String querystring, final int offset, final int maxcount, final long maxtime);
/**
* get the size of the index
* @return number of results if solr is queries with a catch-all pattern
*/
public long getSize();
} }

2
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -93,7 +93,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
@Override @Override
public synchronized long getSize() { public synchronized long getSize() {
try { try {
final QueryResponse rsp = query(AbstractSolrConnector.catchSuccessQuery); final QueryResponse rsp = getResponseByParams(AbstractSolrConnector.catchSuccessQuery);
if (rsp == null) return 0; if (rsp == null) return 0;
final SolrDocumentList docs = rsp.getResults(); final SolrDocumentList docs = rsp.getResults();
if (docs == null) return 0; if (docs == null) return 0;

4
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1065,7 +1065,7 @@ public final class Protocol {
if (localsearch) { if (localsearch) {
// search the local index // search the local index
try { try {
rsp = event.getQuery().getSegment().fulltext().getDefaultConnector().query(solrQuery); rsp = event.getQuery().getSegment().fulltext().getDefaultConnector().getResponseByParams(solrQuery);
docList = rsp.getResults(); docList = rsp.getResults();
} catch (Throwable e) { } catch (Throwable e) {
Network.log.logInfo("SEARCH failed (solr), localpeer (" + e.getMessage() + ")", e); Network.log.logInfo("SEARCH failed (solr), localpeer (" + e.getMessage() + ")", e);
@ -1076,7 +1076,7 @@ public final class Protocol {
String address = target == event.peers.mySeed() ? "localhost:" + target.getPort() : target.getPublicAddress(); String address = target == event.peers.mySeed() ? "localhost:" + target.getPort() : target.getPublicAddress();
RemoteInstance instance = new RemoteInstance("http://" + address, null, "solr"); // this is a 'patch configuration' which considers 'solr' as default collection RemoteInstance instance = new RemoteInstance("http://" + address, null, "solr"); // this is a 'patch configuration' which considers 'solr' as default collection
SolrConnector solrConnector = new RemoteSolrConnector(instance, "solr"); SolrConnector solrConnector = new RemoteSolrConnector(instance, "solr");
rsp = solrConnector.query(solrQuery); rsp = solrConnector.getResponseByParams(solrQuery);
docList = rsp.getResults(); docList = rsp.getResults();
solrConnector.close(); solrConnector.close();
instance.close(); instance.close();

10
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -375,7 +375,7 @@ public final class Fulltext {
// get the metadata from Solr // get the metadata from Solr
try { try {
SolrDocument doc = this.getDefaultConnector().getById(u); SolrDocument doc = this.getDefaultConnector().getDocumentById(u);
if (doc != null) { if (doc != null) {
if (this.urlIndexFile != null) this.urlIndexFile.remove(urlHash); // migration if (this.urlIndexFile != null) this.urlIndexFile.remove(urlHash); // migration
return new URIMetadataNode(doc, wre, weight); return new URIMetadataNode(doc, wre, weight);
@ -479,7 +479,7 @@ public final class Fulltext {
try { try {
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb); if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
// because node entries are richer than metadata entries we must check if they exist to prevent that they are overwritten // because node entries are richer than metadata entries we must check if they exist to prevent that they are overwritten
SolrDocument sd = this.getDefaultConnector().getById(id); SolrDocument sd = this.getDefaultConnector().getDocumentById(id);
if (sd == null || (new URIMetadataNode(sd)).isOlder(entry)) { if (sd == null || (new URIMetadataNode(sd)).isOlder(entry)) {
putDocumentLater(getDefaultConfiguration().metadata2solr(entry)); putDocumentLater(getDefaultConfiguration().metadata2solr(entry));
} }
@ -506,7 +506,7 @@ public final class Fulltext {
try { try {
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb); if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
// because node entries are richer than metadata entries we must check if they exist to prevent that they are overwritten // because node entries are richer than metadata entries we must check if they exist to prevent that they are overwritten
SolrDocument sd = this.getDefaultConnector().getById(id); SolrDocument sd = this.getDefaultConnector().getDocumentById(id);
if (sd == null || (new URIMetadataNode(sd)).isOlder(entry)) { if (sd == null || (new URIMetadataNode(sd)).isOlder(entry)) {
putDocument(getDefaultConfiguration().metadata2solr(entry)); putDocument(getDefaultConfiguration().metadata2solr(entry));
} }
@ -656,7 +656,7 @@ public final class Fulltext {
final AtomicInteger count = new AtomicInteger(0); final AtomicInteger count = new AtomicInteger(0);
Thread t = new Thread(){ Thread t = new Thread(){
public void run() { public void run() {
final BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentQuery(collectionQuery, 0, 1000000, 600000, -1, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName()); final BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(collectionQuery, 0, 1000000, 600000, -1, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
try { try {
SolrDocument doc; SolrDocument doc;
while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) { while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) {
@ -887,7 +887,7 @@ public final class Fulltext {
this.count++; this.count++;
} }
} else { } else {
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentQuery(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", 0, 100000000, 10 * 60 * 60 * 1000, 100, BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", 0, 100000000, 10 * 60 * 60 * 1000, 100,
CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(),
CollectionSchema.author.getSolrFieldName(), CollectionSchema.description.getSolrFieldName(), CollectionSchema.size_i.getSolrFieldName(), CollectionSchema.last_modified.getSolrFieldName()); CollectionSchema.author.getSolrFieldName(), CollectionSchema.description.getSolrFieldName(), CollectionSchema.size_i.getSolrFieldName(), CollectionSchema.last_modified.getSolrFieldName());
SolrDocument doc; SolrDocument doc;

8
source/net/yacy/search/index/Segment.java
Unescape View File

@ -290,7 +290,7 @@ public class Segment {
if (count > 0) return count; if (count > 0) return count;
} }
try { try {
return (int) this.fulltext.getDefaultConnector().getQueryCount(CollectionSchema.text_t.getSolrFieldName() + ":\"" + word + "\""); return (int) this.fulltext.getDefaultConnector().getCountByQuery(CollectionSchema.text_t.getSolrFieldName() + ":\"" + word + "\"");
} catch (Throwable e) { } catch (Throwable e) {
Log.logException(e); Log.logException(e);
return 0; return 0;
@ -310,12 +310,12 @@ public class Segment {
final BlockingQueue<SolrDocument> docQueue; final BlockingQueue<SolrDocument> docQueue;
final String urlstub; final String urlstub;
if (stub == null) { if (stub == null) {
docQueue = this.fulltext.getDefaultConnector().concurrentQuery("*:*", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName()); docQueue = this.fulltext.getDefaultConnector().concurrentDocumentsByQuery("*:*", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
urlstub = null; urlstub = null;
} else { } else {
final String host = stub.getHost(); final String host = stub.getHost();
String hh = DigestURI.hosthash(host); String hh = DigestURI.hosthash(host);
docQueue = this.fulltext.getDefaultConnector().concurrentQuery(CollectionSchema.host_id_s + ":\"" + hh + "\"", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName()); docQueue = this.fulltext.getDefaultConnector().concurrentDocumentsByQuery(CollectionSchema.host_id_s + ":\"" + hh + "\"", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
urlstub = stub.toNormalform(true); urlstub = stub.toNormalform(true);
} }
@ -530,7 +530,7 @@ public class Segment {
// switch unique attribute in new document // switch unique attribute in new document
vector.setField(uniquefield.getSolrFieldName(), false); vector.setField(uniquefield.getSolrFieldName(), false);
// switch attribute also in all existing documents (which should be exactly only one!) // switch attribute also in all existing documents (which should be exactly only one!)
SolrDocumentList docs = this.fulltext.getDefaultConnector().query(CollectionSchema.host_id_s + ":\"" + hostid + "\" AND " + signaturefield.getSolrFieldName() + ":\"" + checkhash.toString() + "\" AND " + uniquefield.getSolrFieldName() + ":true", 0, 1000); SolrDocumentList docs = this.fulltext.getDefaultConnector().getDocumentListByQuery(CollectionSchema.host_id_s + ":\"" + hostid + "\" AND " + signaturefield.getSolrFieldName() + ":\"" + checkhash.toString() + "\" AND " + uniquefield.getSolrFieldName() + ":true", 0, 1000);
for (SolrDocument doc: docs) { for (SolrDocument doc: docs) {
SolrInputDocument sid = this.fulltext.getDefaultConfiguration().toSolrInputDocument(doc); SolrInputDocument sid = this.fulltext.getDefaultConfiguration().toSolrInputDocument(doc);
sid.setField(uniquefield.getSolrFieldName(), false); sid.setField(uniquefield.getSolrFieldName(), false);

4
source/net/yacy/search/schema/CollectionConfiguration.java
Unescape View File

@ -820,7 +820,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
// that means we must search for those entries. // that means we must search for those entries.
connector.commit(true); // make sure that we have latest information that can be found connector.commit(true); // make sure that we have latest information that can be found
//BlockingQueue<SolrDocument> docs = index.fulltext().getSolr().concurrentQuery("*:*", 0, 1000, 60000, 10); //BlockingQueue<SolrDocument> docs = index.fulltext().getSolr().concurrentQuery("*:*", 0, 1000, 60000, 10);
BlockingQueue<SolrDocument> docs = connector.concurrentQuery(CollectionSchema.process_sxt.getSolrFieldName() + ":[* TO *]", 0, 10000, 60000, 50); BlockingQueue<SolrDocument> docs = connector.concurrentDocumentsByQuery(CollectionSchema.process_sxt.getSolrFieldName() + ":[* TO *]", 0, 10000, 60000, 50);
SolrDocument doc; SolrDocument doc;
int proccount = 0, proccount_clickdepthchange = 0, proccount_referencechange = 0; int proccount = 0, proccount_clickdepthchange = 0, proccount_referencechange = 0;
@ -847,7 +847,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
if (!hostExtentCache.containsKey(hosthash)) { if (!hostExtentCache.containsKey(hosthash)) {
StringBuilder q = new StringBuilder(); StringBuilder q = new StringBuilder();
q.append(CollectionSchema.host_id_s.getSolrFieldName()).append(":\"").append(hosthash).append("\" AND ").append(CollectionSchema.httpstatus_i.getSolrFieldName()).append(":200"); q.append(CollectionSchema.host_id_s.getSolrFieldName()).append(":\"").append(hosthash).append("\" AND ").append(CollectionSchema.httpstatus_i.getSolrFieldName()).append(":200");
long count = segment.fulltext().getDefaultConnector().getQueryCount(q.toString()); long count = segment.fulltext().getDefaultConnector().getCountByQuery(q.toString());
hostExtentCache.put(hosthash, count); hostExtentCache.put(hosthash, count);
} }
if (postprocessing_references(segment, doc, sid, url, hostExtentCache)) proccount_referencechange++; if (postprocessing_references(segment, doc, sid, url, hostExtentCache)) proccount_referencechange++;

2
source/net/yacy/search/schema/WebgraphConfiguration.java
Unescape View File

@ -290,7 +290,7 @@ public class WebgraphConfiguration extends SchemaConfiguration implements Serial
// that means we must search for those entries. // that means we must search for those entries.
connector.commit(true); // make sure that we have latest information that can be found connector.commit(true); // make sure that we have latest information that can be found
//BlockingQueue<SolrDocument> docs = index.fulltext().getSolr().concurrentQuery("*:*", 0, 1000, 60000, 10); //BlockingQueue<SolrDocument> docs = index.fulltext().getSolr().concurrentQuery("*:*", 0, 1000, 60000, 10);
BlockingQueue<SolrDocument> docs = connector.concurrentQuery(WebgraphSchema.process_sxt.getSolrFieldName() + ":[* TO *]", 0, 100000, 60000, 50); BlockingQueue<SolrDocument> docs = connector.concurrentDocumentsByQuery(WebgraphSchema.process_sxt.getSolrFieldName() + ":[* TO *]", 0, 100000, 60000, 50);
SolrDocument doc; SolrDocument doc;
String protocol, urlstub, id; String protocol, urlstub, id;

Write Preview
Loading…
Cancel
Save