boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- removed ip_s from default profile since that needs a DNS lookup to

Browse Source 
create an document entry. This makes remote search much slower.
- removed synchronization of add method if ip_s is activated to prevent
that a user configuration causes bad behavior. The disadvantage of that
is, that a index dump can cause data loss if an indexing is running
during index dump
- catched more exceptions and more NPE
- better abstraction in MirrorSolrConnector
- slight performance enhancement when only the index count is requested
(rows=0 is sufficient to get a total count)
pull/1/head
Michael Peter Christen 13 years ago
parent 24f4ca4d85
commit 562183932b
8 changed files with 53 additions and 50 deletions
Show Stats Download Patch File Download Diff File

10
defaults/solr.keys.list
Unescape View File

@ -77,12 +77,12 @@ applinkscount_i
### optional but highly recommended values, not part of the index distribution process
## tags that are attached to crawls/index generation to separate the search result into user-defined subsets
collection_sxt
## point in degrees of latitude,longitude as declared in WSG84, location
coordinate_p
## ip of host of url (after DNS lookup), string
ip_s
## content of author-tag, texgen
author
@ -140,8 +140,8 @@ h6_txt
### optional values, not part of standard YaCy handling (but useful for external applications)
## tags that are attached to crawls/index generation to separate the search result into user-defined subsets
#collection_sxt
## ip of host of url (after DNS lookup), string
#ip_s
## tags of css entries, normalized with absolute URL
#css_tag_txt

14
htroot/IndexFederated_p.java
Unescape View File

@ -119,7 +119,11 @@ public class IndexFederated_p {
if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {
// switch off
sb.index.fulltext().disconnectRemoteSolr();
try {
sb.index.fulltext().disconnectRemoteSolr();
} catch (Throwable e) {
Log.logException(e);
}
}
if (solrRemoteIsOnAfterwards) {
@ -134,9 +138,13 @@ public class IndexFederated_p {
} else {
sb.index.fulltext().disconnectRemoteSolr();
}
} catch (final IOException e) {
} catch (final Throwable e) {
Log.logException(e);
sb.index.fulltext().disconnectRemoteSolr();
try {
sb.index.fulltext().disconnectRemoteSolr();
} catch (Throwable ee) {
Log.logException(ee);
}
}
}

1
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -166,6 +166,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
@Override
public QueryResponse query(SolrParams params) throws IOException {
if (this.server == null) throw new IOException("server disconnected");
try {
return this.server.query(params);
} catch (SolrServerException e) {

48
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -205,18 +205,13 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
return false;
}
cacheMiss_Miss++;
if (this.solr0 != null) {
if (this.solr0.exists(id)) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
return true;
}
}
if (this.solr1 != null) {
if (this.solr1.exists(id)) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
return true;
for (SolrConnector solr: new SolrConnector[]{this.solr0, this.solr1}) {
if (solr != null) {
if (solr.exists(id)) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
return true;
}
}
}
this.missCache.put(id, EXIST);
@ -237,24 +232,17 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
return null;
}
cacheMiss_Miss++;
if (this.solr0 != null) {
doc = this.solr0.get(id);
if (doc != null) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
this.documentCache.put(id, doc);
cacheDocument_Insert++;
return doc;
}
}
if (this.solr1 != null) {
doc = this.solr1.get(id);
if (doc != null) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
this.documentCache.put(id, doc);
cacheDocument_Insert++;
return doc;
for (SolrConnector solr: new SolrConnector[]{this.solr0, this.solr1}) {
if (solr != null) {
doc = solr.get(id);
if (doc != null) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
this.documentCache.put(id, doc);
cacheDocument_Insert++;
return doc;
}
}
}
this.missCache.put(id, EXIST);

2
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -227,7 +227,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
// construct query
final SolrQuery params = new SolrQuery();
params.setQuery(querystring);
params.setRows(1);
params.setRows(0);
params.setStart(0);
// query the server

4
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1059,7 +1059,7 @@ public final class Protocol
// evaluate result
if (docList.size() > 0) {// create containers
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " documents from peer " + target.hash + ":" + target.getName());
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))) ;
final List<ReferenceContainer<WordReference>> container = new ArrayList<ReferenceContainer<WordReference>>(wordhashes.size());
for (byte[] hash: wordhashes) {
try {
@ -1087,7 +1087,7 @@ public final class Protocol
if (localsearch) {
Network.log.logInfo("local search (solr): filtered blacklisted url " + urlEntry.url());
} else {
Network.log.logInfo("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from peer " + target.getName());
Network.log.logInfo("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
}
}
continue; // block with backlist

2
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -170,7 +170,7 @@ public class RemoteSearch extends Thread {
for (Seed s: nodes) {
solrRemoteSearch(event, count, time, s, blacklist);
}
// start search to YaCy peers
final int targets = targetPeers.length;
if (targets == 0) return;

22
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -293,11 +293,14 @@ public final class Fulltext implements Iterable<byte[]> {
if (this.connectedSolr()) {
try {
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
synchronized (this.solr) {
SolrDocument sd = this.solr.get(id);
if (sd == null || this.solrScheme.getDate(sd, YaCySchema.last_modified).before(this.solrScheme.getDate(doc, YaCySchema.last_modified))) {
this.solr.add(doc);
}
SolrDocument sd = this.solr.get(id);
if (sd == null || this.solrScheme.getDate(sd, YaCySchema.last_modified).before(this.solrScheme.getDate(doc, YaCySchema.last_modified))) {
if (this.solrScheme.contains(YaCySchema.ip_s)) {
// ip_s needs a dns lookup which causes blockings during search here
this.solr.add(doc);
} else synchronized (this.solr) {
this.solr.add(doc);
}
}
} catch (SolrException e) {
throw new IOException(e.getMessage(), e);
@ -336,9 +339,12 @@ public final class Fulltext implements Iterable<byte[]> {
if (this.connectedSolr()) {
try {
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
synchronized (this.solr) {
SolrDocument sd = this.solr.get(id);
if (sd == null || (new URIMetadataNode(sd)).isOlder(row)) {
SolrDocument sd = this.solr.get(id);
if (sd == null || (new URIMetadataNode(sd)).isOlder(row)) {
if (this.solrScheme.contains(YaCySchema.ip_s)) {
// ip_s needs a dns lookup which causes blockings during search here
this.solr.add(getSolrScheme().metadata2solr(row));
} else synchronized (this.solr) {
this.solr.add(getSolrScheme().metadata2solr(row));
}
}

Write Preview
Loading…
Cancel
Save