boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

using the BinaryResponseWriter which is supported within the YaCy solr

Browse Source 
servlet since YaCy 1.63. This is much more performant for the client
than using the XMLResponseWriter because parsing of XML data is very CPU
intensive. Older YaCy peers are still requested using the
XMLResponseWriter but the majority of YaCy peers already respond with
the binary writer. This makes remote searches much faster and less CPU
intensive.
pull/1/head
orbiter 11 years ago
parent 61409788eb
commit 037cd0a57c
4 changed files with 18 additions and 13 deletions
Show Stats Download Patch File Download Diff File

18
source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java
Unescape View File

@ -30,6 +30,7 @@ import net.yacy.cora.federate.solr.instance.ShardInstance;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
@ -38,25 +39,28 @@ import org.apache.solr.common.util.NamedList;
public class RemoteSolrConnector extends SolrServerConnector implements SolrConnector {
SolrInstance instance;
String corename;
private final SolrInstance instance;
private final String corename;
private final boolean useBinaryResponseWriter;
/**
* create a new solr connector
* @param instance the instance of the remote solr url, like http://192.168.1.60:8983/solr/ or http://admin:pw@192.168.1.60:8983/solr/
* @throws IOException
*/
public RemoteSolrConnector(final SolrInstance instance) throws IOException {
public RemoteSolrConnector(final SolrInstance instance, final boolean useBinaryResponseWriter) throws IOException {
super();
this.instance = instance;
this.useBinaryResponseWriter = useBinaryResponseWriter;
this.corename = this.instance.getDefaultCoreName();
SolrServer s = instance.getServer(this.corename);
super.init(s);
}
public RemoteSolrConnector(final SolrInstance instance, String corename) {
public RemoteSolrConnector(final SolrInstance instance, final boolean useBinaryResponseWriter, String corename) {
super();
this.instance = instance;
this.useBinaryResponseWriter = useBinaryResponseWriter;
this.corename = corename == null ? this.instance.getDefaultCoreName() : corename;
SolrServer s = instance.getServer(this.corename);
super.init(s);
@ -84,14 +88,14 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
if (q != null) Thread.currentThread().setName("solr query: q = " + q);
QueryRequest request = new QueryRequest(params);
ResponseParser responseParser = new XMLResponseParser();
ResponseParser responseParser = useBinaryResponseWriter ? new BinaryResponseParser() : new XMLResponseParser();
request.setResponseParser(responseParser);
long t = System.currentTimeMillis();
NamedList<Object> result = null;
try {
result = this.server.request(request);
} catch (final Throwable e) {
//Log.logException(e);
//ConcurrentLog.logException(e);
throw new IOException(e.getMessage());
/*
Log.logException(e);
@ -117,7 +121,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000);
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
instances.add(instance);
solr = new RemoteSolrConnector(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5), "solr");
solr = new RemoteSolrConnector(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5), true, "solr");
solr.clear();
final File exampleDir = new File("test/parsertest/");
long t, t0, a = 0;

4
source/net/yacy/cora/federate/solr/instance/InstanceMirror.java
Unescape View File

@ -144,7 +144,7 @@ public class InstanceMirror {
String defaultCoreName = this.getDefaultCoreName();
if (defaultCoreName == null) return null;
EmbeddedSolrConnector esc = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, defaultCoreName);
RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, defaultCoreName);
RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, true, defaultCoreName);
this.defaultConnector = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), 100, 1000000);
this.connectorCache.put(defaultCoreName, this.defaultConnector);
return this.defaultConnector;
@ -154,7 +154,7 @@ public class InstanceMirror {
SolrConnector msc = this.connectorCache.get(corename);
if (msc != null) return msc;
EmbeddedSolrConnector esc = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, corename);
RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, corename);
RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, true, corename);
msc = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), 100, 1000000);
this.connectorCache.put(corename, msc);
return msc;

5
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1054,10 +1054,11 @@ public final class Protocol {
}
} else {
try {
String address = target == event.peers.mySeed() ? "localhost:" + target.getPort() : target.getPublicAddress();
boolean myseed = target == event.peers.mySeed();
String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress();
final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000);
instance = new RemoteInstance("http://" + address, null, "solr", solrtimeout); // this is a 'patch configuration' which considers 'solr' as default collection
solrConnector = new RemoteSolrConnector(instance, "solr");
solrConnector = new RemoteSolrConnector(instance, myseed ? true : target.getVersion() >= 1.63, "solr");
rsp = solrConnector.getResponseByParams(solrQuery);
docList = rsp.getResults();
solrConnector.close();

4
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -206,7 +206,7 @@ public final class Fulltext {
public RemoteSolrConnector getDefaultRemoteSolrConnector() {
if (this.solrInstances.getSolr1() == null) return null;
try {
return new RemoteSolrConnector(this.solrInstances.getSolr1());
return new RemoteSolrConnector(this.solrInstances.getSolr1(), true);
} catch (final IOException e) {
return null;
}
@ -258,7 +258,7 @@ public final class Fulltext {
synchronized (this.solrInstances) {
ShardInstance instance = this.solrInstances.getSolr1();
if (instance != null) {
for (String name: instance.getCoreNames()) new RemoteSolrConnector(instance, name).clear();
for (String name: instance.getCoreNames()) new RemoteSolrConnector(instance, true, name).clear();
}
this.solrInstances.clearCaches();
}

Write Preview
Loading…
Cancel
Save