boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added a field cache for solr queries which call only for a single

Browse Source 
value
- fixed a version conflict exception within a solr add request
pull/1/head
Michael Peter Christen 12 years ago
parent bf42179982
commit b7004043ea
9 changed files with 114 additions and 43 deletions
Show Stats Download Patch File Download Diff File

2
htroot/PerformanceMemory_p.java
Unescape View File

@ -205,7 +205,7 @@ public class PerformanceMemory_p {
// other caching structures
final MirrorSolrConnector solr = (MirrorSolrConnector) Switchboard.getSwitchboard().index.fulltext().getSolr();
final MirrorSolrConnector.HitMissCache hitMissCache = solr.getCache(YaCySchema.id.getSolrFieldName());
final MirrorSolrConnector.HitMissCache hitMissCache = solr.getHitMissCache(YaCySchema.id.getSolrFieldName());
prop.putNum("solrcacheHit.size", solr.nameCacheHitSize());
prop.putNum("solrcacheHit.Hit", hitMissCache.hitCache_Hit);
prop.putNum("solrcacheHit.Miss", hitMissCache.hitCache_Miss);

2
htroot/js/Crawler.js
Unescape View File

@ -74,7 +74,7 @@ function handleStatus(){
var ppmSpan = document.getElementById("ppmSpan");
removeAllChildren(ppmSpan);
for(i = 0; i < ppm / 10; i++){
for(i = 0; i < ppm / 25; i++){
img=document.createElement("img");
img.setAttribute("src", BAR_IMG1);
ppmSpan.appendChild(img);

7
source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
Unescape View File

@ -74,6 +74,13 @@ public abstract class AbstractSolrConnector implements SolrConnector {
}
}
@Override
public Object getFieldById(final String key, final String field) throws IOException {
SolrDocument doc = getById(key, field);
if (doc == null) return null;
return doc.getFieldValue(field);
}
/**
* Get a query result from solr as a stream of documents.
* The result queue is considered as terminated if AbstractSolrConnector.POISON_DOCUMENT is returned.

65
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -25,6 +25,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import net.yacy.cora.federate.solr.YaCySchema;
@ -59,6 +60,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
private SolrConnector solr1;
private int hitCacheMax, missCacheMax, partitions;
private final Map<String, HitMissCache> hitMissCache;
private final Map<String, ARC<String, Object>> fieldCache; // a map from a field name to a id-key/value object cache
private final ARC<String, SolrDocument> documentCache;
public long documentCache_Hit = 0, documentCache_Miss = 0, documentCache_Insert = 0; // for statistics only; do not write
@ -86,12 +88,13 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
this.hitCacheMax = hitCacheMax;
this.missCacheMax = missCacheMax;
this.partitions = Runtime.getRuntime().availableProcessors() * 2;
this.hitMissCache = new HashMap<String, HitMissCache>();
this.hitMissCache = new ConcurrentHashMap<String, HitMissCache>();
this.fieldCache = new ConcurrentHashMap<String, ARC<String, Object>>();
this.documentCache = new ConcurrentARC<String, SolrDocument>(docCacheMax, this.partitions);
}
public HitMissCache getCache(String field) {
public HitMissCache getHitMissCache(String field) {
HitMissCache c = this.hitMissCache.get(field);
if (c == null) {
c = new HitMissCache(this.hitCacheMax, this.missCacheMax, this.partitions);
@ -100,6 +103,15 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
return c;
}
public ARC<String, Object> getFieldCache(String field) {
ARC<String, Object> c = this.fieldCache.get(field);
if (c == null) {
c = new ConcurrentARC<String, Object>(this.hitCacheMax, this.partitions);
this.fieldCache.put(field, c);
}
return c;
}
public boolean isConnected0() {
return this.solr0 != null;
}
@ -137,7 +149,8 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
}
public void clearCache() {
for (HitMissCache c: hitMissCache.values()) c.clearCache();
for (HitMissCache c: this.hitMissCache.values()) c.clearCache();
for (ARC<String, Object> c: this.fieldCache.values()) c.clear();
this.documentCache.clear();
}
@ -189,7 +202,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
@Override
public void delete(final String id) throws IOException {
this.documentCache.remove(id);
HitMissCache c = getCache("id");
HitMissCache c = getHitMissCache("id");
c.hitCache.remove(id);
c.missCache.put(id, EXIST);
c.missCache_Insert++;
@ -206,7 +219,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
public void delete(final List<String> ids) throws IOException {
for (String id: ids) {
this.documentCache.remove(id);
HitMissCache c = getCache("id");
HitMissCache c = getHitMissCache("id");
c.hitCache.remove(id);
c.missCache.put(id, EXIST);
c.missCache_Insert++;
@ -226,7 +239,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
@Override
public boolean exists(final String fieldName, final String key) throws IOException {
HitMissCache c = getCache(fieldName);
HitMissCache c = getHitMissCache(fieldName);
if (c.hitCache.containsKey(key)) {
c.hitCache_Hit++;
return true;
@ -253,15 +266,31 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
return false;
}
@Override
public Object getFieldById(final String key, final String field) throws IOException {
// try to get this from this cache
ARC<String, Object> c = getFieldCache(field);
Object o = c.get(key);
if (o != null) return o;
// load the document
o = super.getFieldById(key, field);
// use result to fill the cache
if (o == null) return null;
c.put(key, o);
return o;
}
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
SolrDocument doc = fields.length == 0 ? this.documentCache.get(key) : null;
SolrDocument doc = this.documentCache.get(key);
if (doc != null) {
this.documentCache_Hit++;
return doc;
}
documentCache_Miss++;
HitMissCache c = this.getCache(YaCySchema.id.getSolrFieldName());
HitMissCache c = this.getHitMissCache(YaCySchema.id.getSolrFieldName());
if (c.missCache.containsKey(key)) {
c.missCache_Hit++;
return null;
@ -277,6 +306,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
this.commit();
if ((solr0 != null && ((doc = solr0.getById(key, fields)) != null)) || (solr1 != null && ((doc = solr1.getById(key, fields)) != null))) {
addToCache(doc, fields.length == 0);
return doc;
}
}
c.missCache.put(key, EXIST);
@ -467,6 +497,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
private void addToCache(SolrDocument doc, boolean doccach) {
for (Map.Entry<String, HitMissCache> e: this.hitMissCache.entrySet()) {
Object keyo = doc.getFieldValue(e.getKey());
if (keyo == null) continue;
String key = null;
if (keyo instanceof String) key = (String) keyo;
if (keyo instanceof Integer) key = ((Integer) keyo).toString();
@ -478,29 +509,37 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
c.hitCache_Insert++;
}
}
String id = (String) doc.getFieldValue(YaCySchema.id.getSolrFieldName());
if (id != null) {
for (Map.Entry<String, ARC<String, Object>> e: this.fieldCache.entrySet()) {
Object keyo = doc.getFieldValue(e.getKey());
if (keyo != null) e.getValue().put(id, keyo);
}
if (doccach) {
this.documentCache.put((String) doc.getFieldValue(YaCySchema.id.getSolrFieldName()), doc);
this.documentCache.put(id, doc);
this.documentCache_Insert++;
}
}
}
@Override
public long getSize() {
long s = 0;
if (this.solr0 != null) s += this.solr0.getSize();
if (this.solr1 != null) s += this.solr1.getSize();
HitMissCache c = getCache("id");
HitMissCache c = getHitMissCache("id");
return Math.max(this.documentCache.size(), Math.max(c.hitCache.size(), s));
}
public int nameCacheHitSize() {
HitMissCache c = getCache("id");
HitMissCache c = getHitMissCache("id");
return c.hitCache.size();
}
public int nameCacheMissSize() {
HitMissCache c = getCache("id");
HitMissCache c = getHitMissCache("id");
return c.missCache.size();
}

5
source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
Unescape View File

@ -143,6 +143,11 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
return this.solr.deleteByQuery(querystring);
}
@Override
public Object getFieldById(final String key, final String field) throws IOException {
return this.solr.getFieldById(key, field);
}
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
return this.solr.getById(key, fields);

9
source/net/yacy/cora/federate/solr/connector/SolrConnector.java
Unescape View File

@ -105,6 +105,15 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException;
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException;
/**
* get a field value from solr by given key for the id-field and a field name
* @param key
* @param field one field
* @return one result or null if no result exists
* @throws IOException
*/
public Object getFieldById(final String key, final String field) throws IOException;
/**
* get a document from solr by given key for the id-field
* @param key

25
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -109,6 +109,14 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
@Override
public long getSize() {
/*
if (this.server instanceof EmbeddedSolrServer) {
EmbeddedSolrServer ess = (EmbeddedSolrServer) this.server;
CoreContainer coreContainer = ess.getCoreContainer();
String coreName = coreContainer.getDefaultCoreName();
SolrCore core = coreContainer.getCore(coreName);
}
*/
try {
final QueryResponse rsp = query(AbstractSolrConnector.catchSuccessQuery);
if (rsp == null) return 0;
@ -190,7 +198,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try {
synchronized (this.server) {
this.server.request(up);
this.server.commit();
//this.server.commit();
}
} catch (final Throwable e) {
throw new IOException(e);
@ -204,11 +212,20 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
synchronized (this.server) {
//this.server.deleteById((String) solrdoc.getFieldValue(YaCySchema.id.getSolrFieldName()));
this.server.add(solrdoc, this.commitWithinMs);
//this.server.commit();
}
} catch (SolrServerException e) {
log.warn(e.getMessage() + " DOC=" + solrdoc.toString());
throw new IOException(e);
// ok try this again and delete the document in advance
try {
this.server.deleteById((String) solrdoc.getFieldValue(YaCySchema.id.getSolrFieldName()));
} catch (SolrServerException e1) {}
try {
synchronized (this.server) {
this.server.add(solrdoc, this.commitWithinMs);
}
} catch (SolrServerException ee) {
log.warn(e.getMessage() + " DOC=" + solrdoc.toString());
throw new IOException(ee);
}
}
}

31
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -215,29 +215,23 @@ public final class Fulltext implements Iterable<byte[]> {
public Date getLoadDate(final String urlHash) {
if (urlHash == null) return null;
SolrDocument doc;
Date x;
try {
doc = this.solr.getById(urlHash, YaCySchema.load_date_dt.getSolrFieldName());
x = (Date) this.solr.getFieldById(urlHash, YaCySchema.load_date_dt.getSolrFieldName());
} catch (IOException e) {
return null;
}
if (doc == null) return null;
Date x = (Date) doc.getFieldValue(YaCySchema.load_date_dt.getSolrFieldName());
if (x == null) return new Date(0);
Date now = new Date();
return x.after(now) ? now : x;
return x;
}
public DigestURI getURL(final byte[] urlHash) {
if (urlHash == null) return null;
SolrDocument doc;
String x;
try {
doc = this.solr.getById(ASCII.String(urlHash), YaCySchema.sku.getSolrFieldName());
x = (String) this.solr.getFieldById(ASCII.String(urlHash), YaCySchema.sku.getSolrFieldName());
} catch (IOException e) {
return null;
}
if (doc == null) return null;
String x = (String) doc.getFieldValue(YaCySchema.sku.getSolrFieldName());
if (x == null) return null;
try {
DigestURI uri = new DigestURI(x, urlHash);
@ -291,13 +285,9 @@ public final class Fulltext implements Iterable<byte[]> {
byte[] idb = ASCII.getBytes(id);
try {
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
SolrDocument sd = this.solr.getById(id, YaCySchema.last_modified.getSolrFieldName());
Date now = new Date();
Date sdDate = sd == null ? null : URIMetadataNode.getDate(sd, YaCySchema.last_modified);
if (sdDate == null || sdDate.after(now)) sdDate = now;
Date docDate = SolrConfiguration.getDate(doc, YaCySchema.last_modified);
if (docDate.after(now)) docDate = now;
if (sd == null || sdDate.before(docDate)) {
Date sdDate = (Date) this.solr.getFieldById(id, YaCySchema.last_modified.getSolrFieldName());
Date docDate = null;
if (sdDate == null || (docDate = SolrConfiguration.getDate(doc, YaCySchema.last_modified)) == null || sdDate.before(docDate)) {
if (this.solrScheme.contains(YaCySchema.ip_s)) {
// ip_s needs a dns lookup which causes blockings during search here
this.solr.add(doc);
@ -492,9 +482,8 @@ public final class Fulltext implements Iterable<byte[]> {
public String failReason(final String urlHash) throws IOException {
if (urlHash == null) return null;
SolrDocument doc = this.solr.getById(urlHash, YaCySchema.failreason_t.getSolrFieldName());
if (doc == null) return null;
String reason = (String) doc.getFieldValue(YaCySchema.failreason_t.getSolrFieldName());
String reason = (String) this.solr.getFieldById(urlHash, YaCySchema.failreason_t.getSolrFieldName());
if (reason == null) return null;
return reason == null ? null : reason.length() == 0 ? null : reason;
}

9
source/net/yacy/search/index/SolrConfiguration.java
Unescape View File

@ -383,8 +383,13 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
}
if (allAttr || contains(YaCySchema.text_t)) add(doc, YaCySchema.text_t, content);
if (allAttr || contains(YaCySchema.wordcount_i)) {
final int contentwc = content.split(" ").length;
add(doc, YaCySchema.wordcount_i, contentwc);
if (content.length() == 0) {
add(doc, YaCySchema.wordcount_i, 0);
} else {
int contentwc = 1;
for (int i = content.length() - 1; i >= 0; i--) if (content.charAt(i) == ' ') contentwc++;
add(doc, YaCySchema.wordcount_i, contentwc);
}
}
if (allAttr || contains(YaCySchema.synonyms_sxt)) {
List<String> synonyms = condenser.synonyms();

Write Preview
Loading…
Cancel
Save