boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

concurrently initialize the error cache; extended also the cache by

Browse Source 
factor 10 up to 1000 entries. This error cache is only used to catch up
paused crawls between shutdown+startup
pull/1/head
Michael Peter Christen 10 years ago
parent 8098a86f1d
commit 77662e08e1
1 changed files with 25 additions and 18 deletions
Show Stats Download Patch File Download Diff File

43
source/net/yacy/search/index/ErrorCache.java
Unescape View File

@ -58,25 +58,32 @@ public class ErrorCache {
public ErrorCache(final Fulltext fulltext) {
this.fulltext = fulltext;
this.cache = new LinkedHashMap<String, CollectionConfiguration.FailDoc>();
try {
// fill stack with latest values
final SolrQuery params = new SolrQuery();
params.setParam("defType", "edismax");
params.setStart(0);
params.setRows(100);
params.setFacet(false);
params.setSort(new SortClause(CollectionSchema.last_modified.getSolrFieldName(), SolrQuery.ORDER.desc));
params.setFields(CollectionSchema.id.getSolrFieldName());
params.setQuery(CollectionSchema.failreason_s.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM);
params.set(CommonParams.DF, CollectionSchema.id.getSolrFieldName()); // DisMaxParams.QF or CommonParams.DF must be given
SolrDocumentList docList = fulltext.getDefaultConnector().getDocumentListByParams(params);
if (docList != null) for (int i = docList.size() - 1; i >= 0; i--) {
SolrDocument doc = docList.get(i);
String hash = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());
this.cache.put(hash, null);
// concurrently fill stack with latest values
new Thread() {
@Override
public void run() {
final SolrQuery params = new SolrQuery();
params.setParam("defType", "edismax");
params.setStart(0);
params.setRows(1000);
params.setFacet(false);
params.setSort(new SortClause(CollectionSchema.last_modified.getSolrFieldName(), SolrQuery.ORDER.desc));
params.setFields(CollectionSchema.id.getSolrFieldName());
params.setQuery(CollectionSchema.failreason_s.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM);
params.set(CommonParams.DF, CollectionSchema.id.getSolrFieldName()); // DisMaxParams.QF or CommonParams.DF must be given
SolrDocumentList docList;
try {
docList = fulltext.getDefaultConnector().getDocumentListByParams(params);
if (docList != null) for (int i = docList.size() - 1; i >= 0; i--) {
SolrDocument doc = docList.get(i);
String hash = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());
cache.put(hash, null);
}
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
} catch (final Throwable e) {
}
}.start();
}
public void clearCache() {

Write Preview
Loading…
Cancel
Save