boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced solr caching:

Browse Source 
- increased cache size which is needed for longer solr commit time
- speed hacks on cache write code
pull/1/head
Michael Peter Christen 12 years ago
parent a33e2742cb
commit 799d71bc67
4 changed files with 45 additions and 55 deletions
Show Stats Download Patch File Download Diff File

18
htroot/PerformanceMemory_p.java
Unescape View File

@ -205,19 +205,19 @@ public class PerformanceMemory_p {
// other caching structures // other caching structures
final MirrorSolrConnector solr = (MirrorSolrConnector) Switchboard.getSwitchboard().index.fulltext().getSolr(); final MirrorSolrConnector solr = (MirrorSolrConnector) Switchboard.getSwitchboard().index.fulltext().getSolr();
prop.putNum("solrcacheHit.size", solr.nameCacheHitSize()); prop.putNum("solrcacheHit.size", solr.nameCacheHitSize());
prop.putNum("solrcacheHit.Hit", solr.cacheHit_Hit); prop.putNum("solrcacheHit.Hit", solr.hitCache_Hit);
prop.putNum("solrcacheHit.Miss", solr.cacheHit_Miss); prop.putNum("solrcacheHit.Miss", solr.hitCache_Miss);
prop.putNum("solrcacheHit.Insert", solr.cacheHit_Insert); prop.putNum("solrcacheHit.Insert", solr.hitCache_Insert);
prop.putNum("solrcacheMiss.size", solr.nameCacheMissSize()); prop.putNum("solrcacheMiss.size", solr.nameCacheMissSize());
prop.putNum("solrcacheMiss.Hit", solr.cacheMiss_Hit); prop.putNum("solrcacheMiss.Hit", solr.missCache_Hit);
prop.putNum("solrcacheMiss.Miss", solr.cacheMiss_Miss); prop.putNum("solrcacheMiss.Miss", solr.missCache_Miss);
prop.putNum("solrcacheMiss.Insert", solr.cacheMiss_Insert); prop.putNum("solrcacheMiss.Insert", solr.missCache_Insert);
prop.putNum("solrcacheDocument.size", solr.nameCacheDocumentSize()); prop.putNum("solrcacheDocument.size", solr.nameCacheDocumentSize());
prop.putNum("solrcacheDocument.Hit", solr.cacheDocument_Hit); prop.putNum("solrcacheDocument.Hit", solr.documentCache_Hit);
prop.putNum("solrcacheDocument.Miss", solr.cacheDocument_Miss); prop.putNum("solrcacheDocument.Miss", solr.documentCache_Miss);
prop.putNum("solrcacheDocument.Insert", solr.cacheDocument_Insert); prop.putNum("solrcacheDocument.Insert", solr.documentCache_Insert);
prop.putNum("namecacheHit.size", Domains.nameCacheHitSize()); prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());
prop.putNum("namecacheHit.Hit", Domains.cacheHit_Hit); prop.putNum("namecacheHit.Hit", Domains.cacheHit_Hit);

76
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -59,9 +59,9 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
private SolrConnector solr1; private SolrConnector solr1;
private final ARC<String, Object> hitCache, missCache; private final ARC<String, Object> hitCache, missCache;
private final ARC<String, SolrDocument> documentCache; private final ARC<String, SolrDocument> documentCache;
public long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write public long hitCache_Hit = 0, hitCache_Miss = 0, hitCache_Insert = 0; // for statistics only; do not write
public long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write public long missCache_Hit = 0, missCache_Miss = 0, missCache_Insert = 0; // for statistics only; do not write
public long cacheDocument_Hit = 0, cacheDocument_Miss = 0, cacheDocument_Insert = 0; // for statistics only; do not write public long documentCache_Hit = 0, documentCache_Miss = 0, documentCache_Insert = 0; // for statistics only; do not write
public MirrorSolrConnector(int hitCacheMax, int missCacheMax, int docCacheMax) { public MirrorSolrConnector(int hitCacheMax, int missCacheMax, int docCacheMax) {
@ -164,7 +164,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
public void delete(final String id) throws IOException { public void delete(final String id) throws IOException {
this.hitCache.remove(id); this.hitCache.remove(id);
this.missCache.put(id, EXIST); this.missCache.put(id, EXIST);
cacheMiss_Insert++; this.missCache_Insert++;
this.documentCache.remove(id); this.documentCache.remove(id);
if (this.solr0 != null) this.solr0.delete(id); if (this.solr0 != null) this.solr0.delete(id);
if (this.solr1 != null) this.solr1.delete(id); if (this.solr1 != null) this.solr1.delete(id);
@ -180,7 +180,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
for (String id: ids) { for (String id: ids) {
this.hitCache.remove(id); this.hitCache.remove(id);
this.missCache.put(id, EXIST); this.missCache.put(id, EXIST);
cacheMiss_Insert++; this.missCache_Insert++;
this.documentCache.remove(id); this.documentCache.remove(id);
} }
if (this.solr0 != null) this.solr0.delete(ids); if (this.solr0 != null) this.solr0.delete(ids);
@ -203,31 +203,28 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
@Override @Override
public boolean exists(final String id) throws IOException { public boolean exists(final String id) throws IOException {
if (this.hitCache.containsKey(id)) { if (this.hitCache.containsKey(id)) {
cacheHit_Hit++; this.hitCache_Hit++;
return true; return true;
} }
cacheHit_Miss++; this.hitCache_Miss++;
if (this.documentCache.containsKey(id)) { if (this.documentCache.containsKey(id)) {
cacheDocument_Hit++; this.documentCache_Hit++;
return true; return true;
} }
cacheDocument_Miss++; this.documentCache_Miss++;
if (this.missCache.containsKey(id)) { if (this.missCache.containsKey(id)) {
cacheMiss_Hit++; this.missCache_Hit++;
return false; return false;
} }
cacheMiss_Miss++; this.missCache_Miss++;
for (SolrConnector solr: new SolrConnector[]{this.solr0, this.solr1}) { if ((solr0 != null && solr0.exists(id)) || (solr1 != null && solr1.exists(id))) {
if (solr != null) { this.missCache.remove(id);
if (solr.exists(id)) { this.hitCache.put(id, EXIST);
this.hitCache.put(id, EXIST); this.hitCache_Insert++;
cacheHit_Insert++; return true;
return true;
}
}
} }
this.missCache.put(id, EXIST); this.missCache.put(id, EXIST);
cacheMiss_Insert++; this.missCache_Insert++;
return false; return false;
} }
@ -235,30 +232,25 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
public SolrDocument get(String id) throws IOException { public SolrDocument get(String id) throws IOException {
SolrDocument doc = this.documentCache.get(id); SolrDocument doc = this.documentCache.get(id);
if (doc != null) { if (doc != null) {
cacheDocument_Hit++; this.documentCache_Hit++;
return doc; return doc;
} }
cacheDocument_Miss++; documentCache_Miss++;
if (this.missCache.containsKey(id)) { if (this.missCache.containsKey(id)) {
cacheMiss_Hit++; this.missCache_Hit++;
return null; return null;
} }
cacheMiss_Miss++; missCache_Miss++;
if ((solr0 != null && ((doc = solr0.get(id)) != null)) || (solr1 != null && ((doc = solr1.get(id)) != null))) {
for (SolrConnector solr: new SolrConnector[]{this.solr0, this.solr1}) { this.missCache.remove(id);
if (solr != null) { this.hitCache.put(id, EXIST);
doc = solr.get(id); this.hitCache_Insert++;
if (doc != null) { this.documentCache.put(id, doc);
this.hitCache.put(id, EXIST); this.documentCache_Insert++;
cacheHit_Insert++; return doc;
this.documentCache.put(id, doc);
cacheDocument_Insert++;
return doc;
}
}
} }
this.missCache.put(id, EXIST); this.missCache.put(id, EXIST);
cacheMiss_Insert++; this.missCache_Insert++;
return null; return null;
} }
@ -274,11 +266,11 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
if (id == null) return; if (id == null) return;
this.missCache.remove(id); this.missCache.remove(id);
this.documentCache.put(id, ClientUtils.toSolrDocument(solrdoc)); this.documentCache.put(id, ClientUtils.toSolrDocument(solrdoc));
cacheDocument_Insert++; this.documentCache_Insert++;
this.hitCache.put(id, EXIST);
this.hitCache_Insert++;
if (this.solr0 != null) this.solr0.add(solrdoc); if (this.solr0 != null) this.solr0.add(solrdoc);
if (this.solr1 != null) this.solr1.add(solrdoc); if (this.solr1 != null) this.solr1.add(solrdoc);
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
} }
@Override @Override
@ -441,9 +433,9 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
String id = (String) solrdoc.getFieldValue(YaCySchema.id.getSolrFieldName()); String id = (String) solrdoc.getFieldValue(YaCySchema.id.getSolrFieldName());
if (id != null) { if (id != null) {
this.hitCache.put(id, EXIST); this.hitCache.put(id, EXIST);
cacheHit_Insert++; hitCache_Insert++;
this.documentCache.put(id, solrdoc); this.documentCache.put(id, solrdoc);
cacheDocument_Insert++; documentCache_Insert++;
} }
} }
} }

4
source/net/yacy/search/Switchboard.java
Unescape View File

@ -1512,9 +1512,7 @@ public final class Switchboard extends serverSwitch
// tests if hash occurrs in any database // tests if hash occurrs in any database
// if it exists, the name of the database is returned, // if it exists, the name of the database is returned,
// if it not exists, null is returned // if it not exists, null is returned
if ( this.index.fulltext().exists(hash) ) { if (this.index.exists(hash)) return "loaded";
return "loaded";
}
return this.crawlQueues.urlExists(hash); return this.crawlQueues.urlExists(hash);
} }

2
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -87,7 +87,7 @@ public final class Fulltext implements Iterable<byte[]> {
this.urlIndexFile = null; this.urlIndexFile = null;
this.exportthread = null; // will have a export thread assigned if exporter is running this.exportthread = null; // will have a export thread assigned if exporter is running
this.statsDump = null; this.statsDump = null;
this.solr = new MirrorSolrConnector(1000, 1000, 100); this.solr = new MirrorSolrConnector(10000, 10000, 100);
this.solrScheme = solrScheme; this.solrScheme = solrScheme;
this.forcedCommitTime = 0; this.forcedCommitTime = 0;
} }

Write Preview
Loading…
Cancel
Save