boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring (renaming) of yacy-solr api

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent 3a0fcfbeda
commit 3502b4c697
14 changed files with 79 additions and 79 deletions
Show Stats Download Patch File Download Diff File

2
htroot/HostBrowser.java
Unescape View File

@ -258,7 +258,7 @@ public class HostBrowser {
q.append(" AND ").append(CollectionSchema.url_paths_sxt.getSolrFieldName()).append(":[* TO *]");
}
}
BlockingQueue<SolrDocument> docs = fulltext.getDefaultConnector().concurrentQuery(q.toString(), 0, 100000, TIMEOUT, 100,
BlockingQueue<SolrDocument> docs = fulltext.getDefaultConnector().concurrentDocumentsByQuery(q.toString(), 0, 100000, TIMEOUT, 100,
CollectionSchema.id.getSolrFieldName(),
CollectionSchema.sku.getSolrFieldName(),
CollectionSchema.failreason_t.getSolrFieldName(),

4
source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java
Unescape View File

@ -201,7 +201,7 @@ public class OpenSearchConnector {
final long numfound;
try {
SolrDocumentList docList = connector.query(webgraphquerystr, 0, 1, webgraphqueryfields);
SolrDocumentList docList = connector.getDocumentListByQuery(webgraphquerystr, 0, 1, webgraphqueryfields);
numfound = docList.getNumFound();
if (numfound == 0) {
Log.logInfo("OpenSearchConnector.Discover", "no results found, abort discover job");
@ -226,7 +226,7 @@ public class OpenSearchConnector {
Set<String> dblmem = new HashSet<String>(); // temp memory for already checked url
while (doloop) {
Log.logInfo("OpenSearchConnector.Discover", "start Solr query loop at " + Integer.toString(loopnr * 20) + " of " + Long.toString(numfound));
SolrDocumentList docList = connector.query(webgraphquerystr, loopnr * 20, 20,webgraphqueryfields); // check chunk of 20 result documents
SolrDocumentList docList = connector.getDocumentListByQuery(webgraphquerystr, loopnr * 20, 20,webgraphqueryfields); // check chunk of 20 result documents
loopnr++;
if (stoptime < System.currentTimeMillis()) {// stop after max 1h
doloop = false;

30
source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
Unescape View File

@ -74,7 +74,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
@Override
public boolean existsByQuery(final String query) throws IOException {
try {
long count = getQueryCount(query);
long count = getCountByQuery(query);
return count > 0;
} catch (final Throwable e) {
return false;
@ -83,7 +83,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
@Override
public Object getFieldById(final String key, final String field) throws IOException {
SolrDocument doc = getById(key, field);
SolrDocument doc = getDocumentById(key, field);
if (doc == null) return null;
return doc.getFieldValue(field);
}
@ -100,7 +100,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
* @return a blocking queue which is terminated with AbstractSolrConnector.POISON_DOCUMENT as last element
*/
@Override
public BlockingQueue<SolrDocument> concurrentQuery(final String querystring, final int offset, final int maxcount, final long maxtime, final int buffersize, final String ... fields) {
public BlockingQueue<SolrDocument> concurrentDocumentsByQuery(final String querystring, final int offset, final int maxcount, final long maxtime, final int buffersize, final String ... fields) {
final BlockingQueue<SolrDocument> queue = buffersize <= 0 ? new LinkedBlockingQueue<SolrDocument>() : new ArrayBlockingQueue<SolrDocument>(buffersize);
final long endtime = System.currentTimeMillis() + maxtime;
final Thread t = new Thread() {
@ -110,7 +110,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
int count = 0;
while (System.currentTimeMillis() < endtime && count < maxcount) {
try {
SolrDocumentList sdl = query(querystring, o, pagesize, fields);
SolrDocumentList sdl = getDocumentListByQuery(querystring, o, pagesize, fields);
for (SolrDocument d: sdl) {
try {queue.put(d);} catch (InterruptedException e) {break;}
count++;
@ -131,7 +131,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
}
@Override
public BlockingQueue<String> concurrentIDs(final String querystring, final int offset, final int maxcount, final long maxtime) {
public BlockingQueue<String> concurrentIDsByQuery(final String querystring, final int offset, final int maxcount, final long maxtime) {
final BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
final long endtime = System.currentTimeMillis() + maxtime;
final Thread t = new Thread() {
@ -140,7 +140,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
int o = offset;
while (System.currentTimeMillis() < endtime) {
try {
SolrDocumentList sdl = query(querystring, o, pagesize, CollectionSchema.id.getSolrFieldName());
SolrDocumentList sdl = getDocumentListByQuery(querystring, o, pagesize, CollectionSchema.id.getSolrFieldName());
for (SolrDocument d: sdl) {
try {queue.put((String) d.getFieldValue(CollectionSchema.id.getSolrFieldName()));} catch (InterruptedException e) {break;}
}
@ -161,7 +161,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
@Override
public Iterator<String> iterator() {
final BlockingQueue<String> queue = concurrentIDs("*:*", 0, Integer.MAX_VALUE, 60000);
final BlockingQueue<String> queue = concurrentIDsByQuery("*:*", 0, Integer.MAX_VALUE, 60000);
return new LookAheadIterator<String>() {
@Override
protected String next0() {
@ -184,7 +184,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
* @throws IOException
*/
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
// construct query
final SolrQuery params = new SolrQuery();
params.setQuery(querystring);
@ -196,7 +196,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
if (fields.length > 0) params.setFields(fields);
// query the server
QueryResponse rsp = query(params);
QueryResponse rsp = getResponseByParams(params);
final SolrDocumentList docs = rsp.getResults();
return docs;
}
@ -208,7 +208,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
* @return the number of results for this query
*/
@Override
public long getQueryCount(String querystring) throws IOException {
public long getCountByQuery(String querystring) throws IOException {
// construct query
final SolrQuery params = new SolrQuery();
params.setQuery(querystring);
@ -218,7 +218,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
params.setFields(CollectionSchema.id.getSolrFieldName());
// query the server
QueryResponse rsp = query(params);
QueryResponse rsp = getResponseByParams(params);
final SolrDocumentList docs = rsp.getResults();
return docs == null ? 0 : docs.getNumFound();
}
@ -246,7 +246,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
for (String field: fields) params.addFacetField(field);
// query the server
QueryResponse rsp = query(params);
QueryResponse rsp = getResponseByParams(params);
Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>(fields.length);
for (String field: fields) {
FacetField facet = rsp.getFacetField(field);
@ -260,12 +260,12 @@ public abstract class AbstractSolrConnector implements SolrConnector {
}
@Override
abstract public QueryResponse query(ModifiableSolrParams params) throws IOException;
abstract public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException;
private final char[] queryIDTemplate = "id:\" \"".toCharArray();
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
public SolrDocument getDocumentById(final String key, final String ... fields) throws IOException {
final SolrQuery query = new SolrQuery();
assert key.length() == 12;
// construct query
@ -279,7 +279,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
// query the server
try {
final QueryResponse rsp = query(query);
final QueryResponse rsp = getResponseByParams(query);
final SolrDocumentList docs = rsp.getResults();
if (docs.isEmpty()) return null;
return docs.get(0);

22
source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java
Unescape View File

@ -149,7 +149,7 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
}
@Override
public SolrDocument getById(final String id, final String ... fields) throws IOException {
public SolrDocument getDocumentById(final String id, final String ... fields) throws IOException {
String q = idQuery(id);
SolrDocument doc = fields.length == 0 ? this.documentCache.get(q) : null;
if (doc != null) {
@ -162,14 +162,14 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
return null;
}
this.missCache_Miss++;
if (solr != null && ((doc = solr.getById(id, fields)) != null)) {
if (solr != null && ((doc = solr.getDocumentById(id, fields)) != null)) {
addToCache(doc, fields.length == 0);
return doc;
}
// check if there is a autocommit problem
if (this.hitCache.containsKey(q)) {
// the document should be there, therefore make a commit and check again
if (solr != null && ((doc = solr.getById(id, fields)) != null)) {
if (solr != null && ((doc = solr.getDocumentById(id, fields)) != null)) {
addToCache(doc, fields.length == 0);
}
}
@ -218,23 +218,23 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
* @throws IOException
*/
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
if (offset == 0 && count == 1 && querystring.startsWith("id:")) {
final SolrDocumentList list = new SolrDocumentList();
SolrDocument doc = getById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields);
SolrDocument doc = getDocumentById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields);
list.add(doc);
// no addToCache(list) here because that was already handlet in get();
return list;
}
if (this.solr != null) {
SolrDocumentList list = this.solr.query(querystring, offset, count, fields);
SolrDocumentList list = this.solr.getDocumentListByQuery(querystring, offset, count, fields);
addToCache(list, fields.length == 0);
return list;
}
// combine both lists
SolrDocumentList list;
list = this.solr.query(querystring, offset, count, fields);
list = this.solr.getDocumentListByQuery(querystring, offset, count, fields);
// add caching
addToCache(list, fields.length == 0);
@ -242,14 +242,14 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
}
@Override
public QueryResponse query(ModifiableSolrParams query) throws IOException, SolrException {
QueryResponse list = this.solr.query(query);
public QueryResponse getResponseByParams(ModifiableSolrParams query) throws IOException, SolrException {
QueryResponse list = this.solr.getResponseByParams(query);
return list;
}
@Override
public long getQueryCount(final String querystring) throws IOException {
return this.solr.getQueryCount(querystring);
public long getCountByQuery(final String querystring) throws IOException {
return this.solr.getCountByQuery(querystring);
}
@Override

2
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -183,7 +183,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
}
@Override
public QueryResponse query(ModifiableSolrParams params) throws IOException {
public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException {
if (this.server == null) throw new IOException("server disconnected");
try {
// during the solr query we set the thread name to the query string to get more debugging info in thread dumps

40
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -153,9 +153,9 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
}
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
public SolrDocument getDocumentById(final String key, final String ... fields) throws IOException {
SolrDocument doc;
if ((solr0 != null && ((doc = solr0.getById(key, fields)) != null)) || (solr1 != null && ((doc = solr1.getById(key, fields)) != null))) {
if ((solr0 != null && ((doc = solr0.getDocumentById(key, fields)) != null)) || (solr1 != null && ((doc = solr1.getDocumentById(key, fields)) != null))) {
return doc;
}
return null;
@ -185,48 +185,48 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
* @throws IOException
*/
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
if (this.solr0 == null && this.solr1 == null) return new SolrDocumentList();
if (offset == 0 && count == 1 && querystring.startsWith("id:")) {
final SolrDocumentList list = new SolrDocumentList();
SolrDocument doc = getById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields);
SolrDocument doc = getDocumentById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields);
list.add(doc);
// no addToCache(list) here because that was already handlet in get();
return list;
}
if (this.solr0 != null && this.solr1 == null) {
SolrDocumentList list = this.solr0.query(querystring, offset, count, fields);
SolrDocumentList list = this.solr0.getDocumentListByQuery(querystring, offset, count, fields);
return list;
}
if (this.solr1 != null && this.solr0 == null) {
SolrDocumentList list = this.solr1.query(querystring, offset, count, fields);
SolrDocumentList list = this.solr1.getDocumentListByQuery(querystring, offset, count, fields);
return list;
}
// combine both lists
SolrDocumentList l;
l = this.solr0.query(querystring, offset, count, fields);
l = this.solr0.getDocumentListByQuery(querystring, offset, count, fields);
if (l.size() >= count) return l;
// at this point we need to know how many results are in solr0
// compute this with a very bad hack; replace with better method later
int size0 = 0;
{ //bad hack - TODO: replace
SolrDocumentList lHack = this.solr0.query(querystring, 0, Integer.MAX_VALUE, fields);
SolrDocumentList lHack = this.solr0.getDocumentListByQuery(querystring, 0, Integer.MAX_VALUE, fields);
size0 = lHack.size();
}
// now use the size of the first query to do a second query
final SolrDocumentList list = new SolrDocumentList();
for (final SolrDocument d: l) list.add(d);
l = this.solr1.query(querystring, offset + l.size() - size0, count - l.size(), fields);
l = this.solr1.getDocumentListByQuery(querystring, offset + l.size() - size0, count - l.size(), fields);
for (final SolrDocument d: l) list.add(d);
return list;
}
@Override
public QueryResponse query(ModifiableSolrParams query) throws IOException, SolrException {
public QueryResponse getResponseByParams(ModifiableSolrParams query) throws IOException, SolrException {
Integer count0 = query.getInt(CommonParams.ROWS);
int count = count0 == null ? 10 : count0.intValue();
Integer start0 = query.getInt(CommonParams.START);
@ -234,16 +234,16 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
if (this.solr0 == null && this.solr1 == null) return new QueryResponse();
if (this.solr0 != null && this.solr1 == null) {
QueryResponse list = this.solr0.query(query);
QueryResponse list = this.solr0.getResponseByParams(query);
return list;
}
if (this.solr1 != null && this.solr0 == null) {
QueryResponse list = this.solr1.query(query);
QueryResponse list = this.solr1.getResponseByParams(query);
return list;
}
// combine both lists
QueryResponse rsp = this.solr0.query(query);
QueryResponse rsp = this.solr0.getResponseByParams(query);
final SolrDocumentList l = rsp.getResults();
if (l.size() >= count) return rsp;
@ -253,7 +253,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
{ //bad hack - TODO: replace
query.set(CommonParams.START, 0);
query.set(CommonParams.ROWS, Integer.MAX_VALUE);
QueryResponse lHack = this.solr0.query(query);
QueryResponse lHack = this.solr0.getResponseByParams(query);
query.set(CommonParams.START, start);
query.set(CommonParams.ROWS, count);
size0 = lHack.getResults().size();
@ -262,7 +262,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
// now use the size of the first query to do a second query
query.set(CommonParams.START, start + l.size() - size0);
query.set(CommonParams.ROWS, count - l.size());
QueryResponse rsp1 = this.solr1.query(query);
QueryResponse rsp1 = this.solr1.getResponseByParams(query);
query.set(CommonParams.START, start);
query.set(CommonParams.ROWS, count);
// TODO: combine both
@ -270,20 +270,20 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
}
@Override
public long getQueryCount(final String querystring) throws IOException {
public long getCountByQuery(final String querystring) throws IOException {
if (this.solr0 == null && this.solr1 == null) return 0;
if (this.solr0 != null && this.solr1 == null) {
return this.solr0.getQueryCount(querystring);
return this.solr0.getCountByQuery(querystring);
}
if (this.solr1 != null && this.solr0 == null) {
return this.solr1.getQueryCount(querystring);
return this.solr1.getCountByQuery(querystring);
}
final AtomicLong count = new AtomicLong(0);
Thread t0 = new Thread() {
@Override
public void run() {
try {
count.addAndGet(MirrorSolrConnector.this.solr0.getQueryCount(querystring));
count.addAndGet(MirrorSolrConnector.this.solr0.getCountByQuery(querystring));
} catch (IOException e) {}
}
};
@ -292,7 +292,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
@Override
public void run() {
try {
count.addAndGet(MirrorSolrConnector.this.solr1.getQueryCount(querystring));
count.addAndGet(MirrorSolrConnector.this.solr1.getCountByQuery(querystring));
} catch (IOException e) {}
}
};

2
source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java
Unescape View File

@ -72,7 +72,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
}
@Override
public QueryResponse query(ModifiableSolrParams params) throws IOException {
public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException {
// during the solr query we set the thread name to the query string to get more debugging info in thread dumps
String q = params.get("q");
String threadname = Thread.currentThread().getName();

26
source/net/yacy/cora/federate/solr/connector/SolrConnector.java
Unescape View File

@ -37,6 +37,12 @@ import org.apache.solr.common.params.ModifiableSolrParams;
public interface SolrConnector extends Iterable<String> /* Iterable of document IDs */ {
/**
* get the size of the index
* @return number of results if solr is queries with a catch-all pattern
*/
public long getSize();
/**
* force a commit
*/
@ -121,14 +127,14 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @return one result or null if no result exists
* @throws IOException
*/
public SolrDocument getById(final String key, final String ... fields) throws IOException;
public SolrDocument getDocumentById(final String key, final String ... fields) throws IOException;
/**
* get a query result from solr
* get a query response from solr
* @param query
* @throws IOException
*/
public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException;
public QueryResponse getResponseByParams(final ModifiableSolrParams query) throws IOException, SolrException;
/**
* get a query result from solr
@ -139,7 +145,7 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @param fields list of fields
* @throws IOException
*/
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException, SolrException;
public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException, SolrException;
/**
* get the number of results when this query is done.
@ -147,7 +153,7 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @param querystring
* @return the number of results for this query
*/
public long getQueryCount(final String querystring) throws IOException;
public long getCountByQuery(final String querystring) throws IOException;
/**
* get facets of the index: a list of lists with values that are most common in a specific field
@ -171,7 +177,7 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @param fields list of fields
* @return a blocking queue which is terminated with AbstractSolrConnector.POISON_DOCUMENT as last element
*/
public BlockingQueue<SolrDocument> concurrentQuery(final String querystring, final int offset, final int maxcount, final long maxtime, final int buffersize, final String ... fields);
public BlockingQueue<SolrDocument> concurrentDocumentsByQuery(final String querystring, final int offset, final int maxcount, final long maxtime, final int buffersize, final String ... fields);
/**
* get a document id result stream from a solr query.
@ -182,12 +188,6 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @param maxcount
* @return
*/
public BlockingQueue<String> concurrentIDs(final String querystring, final int offset, final int maxcount, final long maxtime);
/**
* get the size of the index
* @return number of results if solr is queries with a catch-all pattern
*/
public long getSize();
public BlockingQueue<String> concurrentIDsByQuery(final String querystring, final int offset, final int maxcount, final long maxtime);
}

2
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -93,7 +93,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
@Override
public synchronized long getSize() {
try {
final QueryResponse rsp = query(AbstractSolrConnector.catchSuccessQuery);
final QueryResponse rsp = getResponseByParams(AbstractSolrConnector.catchSuccessQuery);
if (rsp == null) return 0;
final SolrDocumentList docs = rsp.getResults();
if (docs == null) return 0;

4
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1065,7 +1065,7 @@ public final class Protocol {
if (localsearch) {
// search the local index
try {
rsp = event.getQuery().getSegment().fulltext().getDefaultConnector().query(solrQuery);
rsp = event.getQuery().getSegment().fulltext().getDefaultConnector().getResponseByParams(solrQuery);
docList = rsp.getResults();
} catch (Throwable e) {
Network.log.logInfo("SEARCH failed (solr), localpeer (" + e.getMessage() + ")", e);
@ -1076,7 +1076,7 @@ public final class Protocol {
String address = target == event.peers.mySeed() ? "localhost:" + target.getPort() : target.getPublicAddress();
RemoteInstance instance = new RemoteInstance("http://" + address, null, "solr"); // this is a 'patch configuration' which considers 'solr' as default collection
SolrConnector solrConnector = new RemoteSolrConnector(instance, "solr");
rsp = solrConnector.query(solrQuery);
rsp = solrConnector.getResponseByParams(solrQuery);
docList = rsp.getResults();
solrConnector.close();
instance.close();

10
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -375,7 +375,7 @@ public final class Fulltext {
// get the metadata from Solr
try {
SolrDocument doc = this.getDefaultConnector().getById(u);
SolrDocument doc = this.getDefaultConnector().getDocumentById(u);
if (doc != null) {
if (this.urlIndexFile != null) this.urlIndexFile.remove(urlHash); // migration
return new URIMetadataNode(doc, wre, weight);
@ -479,7 +479,7 @@ public final class Fulltext {
try {
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
// because node entries are richer than metadata entries we must check if they exist to prevent that they are overwritten
SolrDocument sd = this.getDefaultConnector().getById(id);
SolrDocument sd = this.getDefaultConnector().getDocumentById(id);
if (sd == null || (new URIMetadataNode(sd)).isOlder(entry)) {
putDocumentLater(getDefaultConfiguration().metadata2solr(entry));
}
@ -506,7 +506,7 @@ public final class Fulltext {
try {
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
// because node entries are richer than metadata entries we must check if they exist to prevent that they are overwritten
SolrDocument sd = this.getDefaultConnector().getById(id);
SolrDocument sd = this.getDefaultConnector().getDocumentById(id);
if (sd == null || (new URIMetadataNode(sd)).isOlder(entry)) {
putDocument(getDefaultConfiguration().metadata2solr(entry));
}
@ -656,7 +656,7 @@ public final class Fulltext {
final AtomicInteger count = new AtomicInteger(0);
Thread t = new Thread(){
public void run() {
final BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentQuery(collectionQuery, 0, 1000000, 600000, -1, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
final BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(collectionQuery, 0, 1000000, 600000, -1, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
try {
SolrDocument doc;
while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) {
@ -887,7 +887,7 @@ public final class Fulltext {
this.count++;
}
} else {
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentQuery(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", 0, 100000000, 10 * 60 * 60 * 1000, 100,
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", 0, 100000000, 10 * 60 * 60 * 1000, 100,
CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(),
CollectionSchema.author.getSolrFieldName(), CollectionSchema.description.getSolrFieldName(), CollectionSchema.size_i.getSolrFieldName(), CollectionSchema.last_modified.getSolrFieldName());
SolrDocument doc;

8
source/net/yacy/search/index/Segment.java
Unescape View File

@ -290,7 +290,7 @@ public class Segment {
if (count > 0) return count;
}
try {
return (int) this.fulltext.getDefaultConnector().getQueryCount(CollectionSchema.text_t.getSolrFieldName() + ":\"" + word + "\"");
return (int) this.fulltext.getDefaultConnector().getCountByQuery(CollectionSchema.text_t.getSolrFieldName() + ":\"" + word + "\"");
} catch (Throwable e) {
Log.logException(e);
return 0;
@ -310,12 +310,12 @@ public class Segment {
final BlockingQueue<SolrDocument> docQueue;
final String urlstub;
if (stub == null) {
docQueue = this.fulltext.getDefaultConnector().concurrentQuery("*:*", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
docQueue = this.fulltext.getDefaultConnector().concurrentDocumentsByQuery("*:*", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
urlstub = null;
} else {
final String host = stub.getHost();
String hh = DigestURI.hosthash(host);
docQueue = this.fulltext.getDefaultConnector().concurrentQuery(CollectionSchema.host_id_s + ":\"" + hh + "\"", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
docQueue = this.fulltext.getDefaultConnector().concurrentDocumentsByQuery(CollectionSchema.host_id_s + ":\"" + hh + "\"", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
urlstub = stub.toNormalform(true);
}
@ -530,7 +530,7 @@ public class Segment {
// switch unique attribute in new document
vector.setField(uniquefield.getSolrFieldName(), false);
// switch attribute also in all existing documents (which should be exactly only one!)
SolrDocumentList docs = this.fulltext.getDefaultConnector().query(CollectionSchema.host_id_s + ":\"" + hostid + "\" AND " + signaturefield.getSolrFieldName() + ":\"" + checkhash.toString() + "\" AND " + uniquefield.getSolrFieldName() + ":true", 0, 1000);
SolrDocumentList docs = this.fulltext.getDefaultConnector().getDocumentListByQuery(CollectionSchema.host_id_s + ":\"" + hostid + "\" AND " + signaturefield.getSolrFieldName() + ":\"" + checkhash.toString() + "\" AND " + uniquefield.getSolrFieldName() + ":true", 0, 1000);
for (SolrDocument doc: docs) {
SolrInputDocument sid = this.fulltext.getDefaultConfiguration().toSolrInputDocument(doc);
sid.setField(uniquefield.getSolrFieldName(), false);

4
source/net/yacy/search/schema/CollectionConfiguration.java
Unescape View File

@ -820,7 +820,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
// that means we must search for those entries.
connector.commit(true); // make sure that we have latest information that can be found
//BlockingQueue<SolrDocument> docs = index.fulltext().getSolr().concurrentQuery("*:*", 0, 1000, 60000, 10);
BlockingQueue<SolrDocument> docs = connector.concurrentQuery(CollectionSchema.process_sxt.getSolrFieldName() + ":[* TO *]", 0, 10000, 60000, 50);
BlockingQueue<SolrDocument> docs = connector.concurrentDocumentsByQuery(CollectionSchema.process_sxt.getSolrFieldName() + ":[* TO *]", 0, 10000, 60000, 50);
SolrDocument doc;
int proccount = 0, proccount_clickdepthchange = 0, proccount_referencechange = 0;
@ -847,7 +847,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
if (!hostExtentCache.containsKey(hosthash)) {
StringBuilder q = new StringBuilder();
q.append(CollectionSchema.host_id_s.getSolrFieldName()).append(":\"").append(hosthash).append("\" AND ").append(CollectionSchema.httpstatus_i.getSolrFieldName()).append(":200");
long count = segment.fulltext().getDefaultConnector().getQueryCount(q.toString());
long count = segment.fulltext().getDefaultConnector().getCountByQuery(q.toString());
hostExtentCache.put(hosthash, count);
}
if (postprocessing_references(segment, doc, sid, url, hostExtentCache)) proccount_referencechange++;

2
source/net/yacy/search/schema/WebgraphConfiguration.java
Unescape View File

@ -290,7 +290,7 @@ public class WebgraphConfiguration extends SchemaConfiguration implements Serial
// that means we must search for those entries.
connector.commit(true); // make sure that we have latest information that can be found
//BlockingQueue<SolrDocument> docs = index.fulltext().getSolr().concurrentQuery("*:*", 0, 1000, 60000, 10);
BlockingQueue<SolrDocument> docs = connector.concurrentQuery(WebgraphSchema.process_sxt.getSolrFieldName() + ":[* TO *]", 0, 100000, 60000, 50);
BlockingQueue<SolrDocument> docs = connector.concurrentDocumentsByQuery(WebgraphSchema.process_sxt.getSolrFieldName() + ":[* TO *]", 0, 100000, 60000, 50);
SolrDocument doc;
String protocol, urlstub, id;

Write Preview
Loading…
Cancel
Save