diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index 17302b3fb..c4ad7f5ec 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -18,10 +18,10 @@ network.unit.description = Public YaCy Community network.unit.domain = global # maximum search time for remote queries (milliseconds) -network.unit.remotesearch.maxtime = 1000 +network.unit.remotesearch.maxtime = 3000 # maximum number of results per remote query -network.unit.remotesearch.maxcount = 100 +network.unit.remotesearch.maxcount = 10 # flag to switch on dht transmission # if the dht transmission is set to 'false' then for a global diff --git a/defaults/yacy.network.metager.unit b/defaults/yacy.network.metager.unit index e75494eef..ea2461db4 100644 --- a/defaults/yacy.network.metager.unit +++ b/defaults/yacy.network.metager.unit @@ -18,10 +18,10 @@ network.unit.description = YaCy freeworld enhanced for MetaGer feed network.unit.domain = global # maximum search time for remote queries (milliseconds) -network.unit.remotesearch.maxtime = 1000 +network.unit.remotesearch.maxtime = 3000 # maximum number of results per remote query -network.unit.remotesearch.maxcount = 100 +network.unit.remotesearch.maxcount = 10 # flag to switch on dht transmission # if the dht transmission is set to 'false' then for a global diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 1ba923448..b2a488f47 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -28,7 +28,6 @@ import org.apache.solr.common.SolrException; import net.yacy.cora.document.UTF8; import net.yacy.cora.federate.solr.connector.RemoteSolrConnector; -import net.yacy.cora.federate.solr.connector.ShardSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.federate.solr.instance.ShardInstance; @@ -130,7 +129,7 @@ public class IndexFederated_p { final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0; try { if (usesolr) { - ArrayList instances = ShardSolrConnector.getShardInstances(solrurls, null, null); + ArrayList instances = RemoteInstance.getShardInstances(solrurls, null, null); sb.index.fulltext().connectRemoteSolr(instances); } else { sb.index.fulltext().disconnectRemoteSolr(); @@ -157,8 +156,8 @@ public class IndexFederated_p { } else { prop.put("table", 1); final SolrConnector solr = sb.index.fulltext().getDefaultRemoteSolrConnector(); - final long[] size = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getSizeList() : new long[]{((RemoteSolrConnector) solr).getSize()}; - final ArrayList urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaces() : ((ShardInstance) ((RemoteSolrConnector) solr).getInstance()).getAdminInterfaces(); + final long[] size = new long[]{((RemoteSolrConnector) solr).getSize()}; + final ArrayList urls = ((ShardInstance) ((RemoteSolrConnector) solr).getInstance()).getAdminInterfaces(); boolean dark = false; for (int i = 0; i < size.length; i++) { prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark; diff --git a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java index fa8e1c2e3..ed56b0b5c 100644 --- a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java @@ -21,7 +21,6 @@ package net.yacy.cora.federate.solr.connector; import java.io.IOException; -import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -44,7 +43,6 @@ import org.apache.solr.client.solrj.response.FacetField.Count; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.params.ModifiableSolrParams; @@ -286,11 +284,4 @@ public abstract class AbstractSolrConnector implements SolrConnector { throw new IOException(e.getMessage(), e); } } - - @Override - public void add(final Collection solrdocs) throws IOException, SolrException { - for (SolrInputDocument solrdoc: solrdocs) { - add(solrdoc); - } - } } diff --git a/source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java index 01bc4d8ca..77cd12420 100644 --- a/source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java @@ -21,6 +21,7 @@ package net.yacy.cora.federate.solr.connector; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -222,6 +223,20 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo if (this.solr != null) this.solr.add(solrdoc); } + @Override + public void add(final Collection solrdocs) throws IOException, SolrException { + for (SolrInputDocument solrdoc: solrdocs) { + String id = (String) solrdoc.getFieldValue(CollectionSchema.id.getSolrFieldName()); + assert id != null; + if (id == null) continue; + SolrDocument doc = ClientUtils.toSolrDocument(solrdoc); + addToCache(doc, true); + this.documentCache.put(id, doc); + this.documentCache_Insert++; + } + if (this.solr != null) this.solr.add(solrdocs); + } + /** * get a query result from solr * to get all results set the query String to "*:*" diff --git a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java index c9dd8268f..c4c35b7f5 100644 --- a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java @@ -21,13 +21,13 @@ package net.yacy.cora.federate.solr.connector; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import net.yacy.cora.sorting.ReversibleScoreMap; -import net.yacy.search.schema.CollectionSchema; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -168,12 +168,15 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo */ @Override public void add(final SolrInputDocument solrdoc) throws IOException { - String id = (String) solrdoc.getFieldValue(CollectionSchema.id.getSolrFieldName()); - assert id != null; - if (id == null) return; if (this.solr0 != null) this.solr0.add(solrdoc); if (this.solr1 != null) this.solr1.add(solrdoc); } + + @Override + public void add(final Collection solrdocs) throws IOException, SolrException { + if (this.solr0 != null) this.solr0.add(solrdocs); + if (this.solr1 != null) this.solr1.add(solrdocs); + } /** * get a query result from solr diff --git a/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java deleted file mode 100644 index bfa73104a..000000000 --- a/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java +++ /dev/null @@ -1,150 +0,0 @@ -/** - * MultipleSolrConnector - * Copyright 2011 by Michael Peter Christen - * First released 08.11.2011 at http://yacy.net - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package net.yacy.cora.federate.solr.connector; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.ArrayBlockingQueue; - -import net.yacy.cora.federate.solr.instance.SolrInstance; - -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.ModifiableSolrParams; - -public class MultipleSolrConnector extends AbstractSolrConnector implements SolrConnector { - - private final static SolrInputDocument POISON_DOC = new SolrInputDocument(); - - private final ArrayBlockingQueue queue; - private final AddWorker[] worker; - private final SolrConnector solr; - - public MultipleSolrConnector(final SolrInstance instance, final String corename, final int connections) { - this.solr = new RemoteSolrConnector(instance, corename); - this.queue = new ArrayBlockingQueue(1000); - this.worker = new AddWorker[connections]; - for (int i = 0; i < connections; i++) { - this.worker[i] = new AddWorker(instance, corename); - this.worker[i].start(); - } - } - - private class AddWorker extends Thread { - private final SolrConnector solr; - public AddWorker(final SolrInstance instance, final String corename) { - this.solr = new RemoteSolrConnector(instance, corename); - } - @Override - public void run() { - SolrInputDocument doc; - try { - while ((doc = MultipleSolrConnector.this.queue.take()) != POISON_DOC) { - try { - this.solr.add(doc); - } catch (SolrException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } catch (InterruptedException e) { - } finally { - this.solr.close(); - } - } - } - - @Override - public void commit(boolean softCommit) { - this.solr.commit(softCommit); - } - - /** - * force an explicit merge of segments - * @param maxSegments the maximum number of segments. Set to 1 for maximum optimization - */ - public void optimize(int maxSegments) { - this.solr.optimize(maxSegments); - } - - @Override - public void close() { - // send termination signal to worker - for (@SuppressWarnings("unused") AddWorker element : this.worker) { - try { - this.queue.put(POISON_DOC); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - // wait for termination - for (AddWorker element : this.worker) { - try { - element.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - this.solr.close(); - } - - @Override - public void clear() throws IOException { - this.solr.clear(); - } - - @Override - public void delete(final String id) throws IOException { - this.solr.delete(id); - } - - @Override - public void delete(final List ids) throws IOException { - this.solr.delete(ids); - } - - @Override - public void deleteByQuery(final String querystring) throws IOException { - this.solr.deleteByQuery(querystring); - } - - @Override - public void add(final SolrInputDocument solrdoc) throws IOException, SolrException { - try { - this.queue.put(solrdoc); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - @Override - public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException { - return this.solr.query(query); - } - - @Override - public long getSize() { - return this.solr.getSize(); - } - -} diff --git a/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java b/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java deleted file mode 100644 index 5756fbf5d..000000000 --- a/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * SolrRetryConnector - * Copyright 2011 by Michael Peter Christen - * First released 08.11.2011 at http://yacy.net - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package net.yacy.cora.federate.solr.connector; - -import java.io.IOException; -import java.util.List; - -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.ModifiableSolrParams; - -public class RetrySolrConnector extends AbstractSolrConnector implements SolrConnector { - - private final SolrConnector solrConnector; - private final long retryMaxTime; - - public RetrySolrConnector(final SolrConnector solrConnector, final long retryMaxTime) { - this.solrConnector = solrConnector; - this.retryMaxTime = retryMaxTime; - } - - @Override - public void commit(boolean softCommit) { - this.solrConnector.commit(softCommit); - } - - /** - * force an explicit merge of segments - * @param maxSegments the maximum number of segments. Set to 1 for maximum optimization - */ - public void optimize(int maxSegments) { - this.solrConnector.optimize(maxSegments); - } - - @Override - public synchronized void close() { - this.solrConnector.close(); - } - - @Override - public void clear() throws IOException { - final long t = System.currentTimeMillis() + this.retryMaxTime; - Throwable ee = null; - while (System.currentTimeMillis() < t) try { - this.solrConnector.clear(); - return; - } catch (final Throwable e) { - ee = e; - try {Thread.sleep(10);} catch (final InterruptedException e1) {} - continue; - } - if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); - } - - @Override - public void delete(final String id) throws IOException { - final long t = System.currentTimeMillis() + this.retryMaxTime; - Throwable ee = null; - while (System.currentTimeMillis() < t) try { - this.solrConnector.delete(id); - return; - } catch (final Throwable e) { - ee = e; - try {Thread.sleep(10);} catch (final InterruptedException e1) {} - continue; - } - if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); - } - - @Override - public void delete(final List ids) throws IOException { - final long t = System.currentTimeMillis() + this.retryMaxTime; - Throwable ee = null; - while (System.currentTimeMillis() < t) try { - this.solrConnector.delete(ids); - return; - } catch (final Throwable e) { - ee = e; - try {Thread.sleep(10);} catch (final InterruptedException e1) {} - continue; - } - if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); - } - - @Override - public void deleteByQuery(final String querystring) throws IOException { - final long t = System.currentTimeMillis() + this.retryMaxTime; - Throwable ee = null; - while (System.currentTimeMillis() < t) try { - this.solrConnector.deleteByQuery(querystring); - return; - } catch (final Throwable e) { - ee = e; - try {Thread.sleep(10);} catch (final InterruptedException e1) {} - continue; - } - if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); - } - - @Override - public void add(final SolrInputDocument solrdoc) throws IOException, SolrException { - final long t = System.currentTimeMillis() + this.retryMaxTime; - Throwable ee = null; - while (System.currentTimeMillis() < t) try { - this.solrConnector.add(solrdoc); - return; - } catch (final Throwable e) { - ee = e; - try {Thread.sleep(10);} catch (final InterruptedException e1) {} - continue; - } - if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); - } - - @Override - public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException { - final long t = System.currentTimeMillis() + this.retryMaxTime; - Throwable ee = null; - while (System.currentTimeMillis() < t) try { - return this.solrConnector.query(query); - } catch (final Throwable e) { - ee = e; - try {Thread.sleep(10);} catch (final InterruptedException e1) {} - continue; - } - if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); - return null; - } - - @Override - public long getSize() { - final long t = System.currentTimeMillis() + this.retryMaxTime; - while (System.currentTimeMillis() < t) try { - return this.solrConnector.getSize(); - } catch (final Throwable e) { - try {Thread.sleep(10);} catch (final InterruptedException e1) {} - continue; - } - return 0; - } - -} diff --git a/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java deleted file mode 100644 index 70b0c7e84..000000000 --- a/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java +++ /dev/null @@ -1,295 +0,0 @@ -/** - * ShardSolrConnector - * Copyright 2011 by Michael Peter Christen - * First released 25.05.2011 at http://yacy.net - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package net.yacy.cora.federate.solr.connector; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicLong; - -import net.yacy.cora.sorting.ReversibleScoreMap; -import net.yacy.cora.federate.solr.instance.ResponseAccumulator; -import net.yacy.cora.federate.solr.instance.RemoteInstance; - -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.ModifiableSolrParams; - -public class ShardSolrConnector extends AbstractSolrConnector implements SolrConnector { - - private final ArrayList instances; - private final ArrayList connectors; - private final ShardSelection sharding; - private final ArrayList adminInterfaces; - - public ShardSolrConnector( - ArrayList instances, - final ShardSelection.Method method, boolean multipleConnections) { - this.instances = instances; - this.connectors = new ArrayList(); - SolrConnector s; - this.adminInterfaces = new ArrayList(instances.size()); - String defaultCoreName = instances.get(0).getDefaultCoreName(); - for (final RemoteInstance instance: instances) { - adminInterfaces.add(instance.getAdminInterface()); - s = multipleConnections ? new MultipleSolrConnector(instance, defaultCoreName, 2) : new RemoteSolrConnector(instance, defaultCoreName); - this.connectors.add(s /*new RetrySolrConnector(s, timeout)*/); - } - this.sharding = new ShardSelection(method, this.connectors.size()); - } - - public static ArrayList getShardInstances(final String urlList, Collection coreNames, String defaultCoreName) throws IOException { - urlList.replace(' ', ','); - String[] urls = urlList.split(","); - ArrayList instances = new ArrayList(); - for (final String u: urls) { - RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName); - instances.add(instance); - } - return instances; - } - - public ArrayList getInstances() { - return this.instances; - } - - @Override - public void commit(boolean softCommit) { - for (final SolrConnector connector: this.connectors) connector.commit(softCommit); - } - - /** - * force an explicit merge of segments - * @param maxSegments the maximum number of segments. Set to 1 for maximum optimization - */ - @Override - public void optimize(int maxSegments) { - for (final SolrConnector connector: this.connectors) connector.optimize(maxSegments); - } - - @Override - public synchronized void close() { - for (final SolrConnector connector: this.connectors) connector.close(); - } - - /** - * delete everything in the solr index - * @throws IOException - */ - @Override - public void clear() throws IOException { - for (final SolrConnector connector: this.connectors) connector.clear(); - } - - /** - * delete an entry from solr - * @param id the url hash of the entry - * @throws IOException - */ - @Override - public void delete(final String id) throws IOException { - for (final SolrConnector connector: this.connectors) connector.delete(id); - } - - /** - * delete a set of entries from solr; entries are identified by their url hash - * @param ids a list of url hashes - * @throws IOException - */ - @Override - public void delete(final List ids) throws IOException { - for (final SolrConnector connector: this.connectors) connector.delete(ids); - } - - @Override - public void deleteByQuery(final String querystring) throws IOException { - for (final SolrConnector connector: this.connectors) connector.deleteByQuery(querystring); - } - - /** - * check if a given id exists in solr - * @param id - * @return true if any entry in solr exists - * @throws IOException - */ - @Override - public boolean exists(final String fieldName, final String key) throws IOException { - for (final SolrConnector connector: this.connectors) { - if (connector.exists(fieldName, key)) return true; - } - return false; - } - - @Override - public SolrDocument getById(final String key, final String ... fields) throws IOException { - for (final SolrConnector connector: this.connectors) { - SolrDocument doc = connector.getById(key, fields); - if (doc != null) return doc; - } - return null; - } - - /** - * add a Solr document - * @param solrdoc - * @throws IOException - */ - @Override - public void add(final SolrInputDocument solrdoc) throws IOException { - this.connectors.get(this.sharding.select(solrdoc)).add(solrdoc); - } - - /** - * add a collection of Solr documents - * @param docs - * @throws IOException - */ - protected void addSolr(final Collection docs) throws IOException { - for (final SolrInputDocument doc: docs) add(doc); - } - - /** - * get a query result from solr - * to get all results set the query String to "*:*" - * @param querystring - * @throws IOException - */ - @Override - public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException { - final SolrDocumentList list = new SolrDocumentList(); - List t = new ArrayList(); - for (final SolrConnector connector: this.connectors) { - Thread t0 = new Thread() { - @Override - public void run() { - try { - final SolrDocumentList l = connector.query(querystring, offset, count, fields); - for (final SolrDocument d: l) { - list.add(d); - } - } catch (IOException e) {} - } - }; - t0.start(); - t.add(t0); - } - for (Thread t0: t) { - try {t0.join();} catch (InterruptedException e) {} - } - return list; - } - - @Override - public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException { - final Collection qrl = new ConcurrentLinkedQueue(); - // concurrently call all shards - List t = new ArrayList(); - for (final SolrConnector connector: this.connectors) { - Thread t0 = new Thread() { - @Override - public void run() { - QueryResponse rsp; - try { - rsp = connector.query(query); - } catch (Throwable e) {return;} - qrl.add(rsp); - } - }; - t0.start(); - t.add(t0); - } - for (Thread t0: t) { - try {t0.join();} catch (InterruptedException e) {} - } - - // prepare combined response - return ResponseAccumulator.combineResponses(qrl); - } - - @Override - public long getQueryCount(final String querystring) throws IOException { - final AtomicLong count = new AtomicLong(0); - List t = new ArrayList(); - for (final SolrConnector connector: this.connectors) { - Thread t0 = new Thread() { - @Override - public void run() { - try { - count.addAndGet(connector.getQueryCount(querystring)); - } catch (IOException e) {} - } - }; - t0.start(); - t.add(t0); - } - for (Thread t0: t) { - try {t0.join();} catch (InterruptedException e) {} - } - return count.get(); - } - - @Override - public Map> getFacets(String query, int maxresults, final String ... fields) throws IOException { - Map> facets = new HashMap>(); - for (final SolrConnector connector: this.connectors) { - Map> peer = connector.getFacets(query, maxresults, fields); - innerloop: for (Map.Entry> facet: facets.entrySet()) { - ReversibleScoreMap peerfacet = peer.remove(facet.getKey()); - if (peerfacet == null) continue innerloop; - for (String key: peerfacet) facet.getValue().inc(key, peerfacet.get(key)); - } - for (Map.Entry> peerfacet: peer.entrySet()) { - facets.put(peerfacet.getKey(), peerfacet.getValue()); - } - } - return facets; - } - - - public long[] getSizeList() { - final long[] size = new long[this.connectors.size()]; - int i = 0; - for (final SolrConnector connector: this.connectors) { - size[i++] = connector.getSize(); - } - return size; - } - - @Override - public long getSize() { - final long[] size = getSizeList(); - long s = 0; - for (final long l: size) s += l; - return s; - } - - public ArrayList getAdminInterfaces() { - return this.adminInterfaces; - } - -} diff --git a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java index 4d9d79ab8..10902cd48 100644 --- a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java @@ -67,11 +67,16 @@ public class RemoteInstance implements SolrInstance { private final Collection coreNames; private final Map server; - /* - public RemoteInstance(final String url) throws IOException { - this(url, null, url.endsWith("solr/") || url.endsWith("solr") ? "solr" : CollectionSchema.CORE_NAME); + public static ArrayList getShardInstances(final String urlList, Collection coreNames, String defaultCoreName) throws IOException { + urlList.replace(' ', ','); + String[] urls = urlList.split(","); + ArrayList instances = new ArrayList(); + for (final String u: urls) { + RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName); + instances.add(instance); + } + return instances; } - */ public RemoteInstance(final String url, final Collection coreNames, final String defaultCoreName) throws IOException { this.server= new HashMap(); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 0d38e52e1..49b8635dd 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -100,7 +100,6 @@ import net.yacy.cora.federate.solr.Boost; import net.yacy.cora.federate.solr.SchemaConfiguration; import net.yacy.cora.federate.solr.ProcessType; import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; -import net.yacy.cora.federate.solr.connector.ShardSolrConnector; import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.lod.JenaTripleStore; @@ -490,7 +489,7 @@ public final class Switchboard extends serverSwitch { if (usesolr && solrurls != null && solrurls.length() > 0) { try { - ArrayList instances = ShardSolrConnector.getShardInstances(solrurls, null, null); + ArrayList instances = RemoteInstance.getShardInstances(solrurls, null, null); this.index.fulltext().connectRemoteSolr(instances); } catch ( final IOException e ) { Log.logException(e); @@ -1329,7 +1328,7 @@ public final class Switchboard extends serverSwitch { if (usesolr && solrurls != null && solrurls.length() > 0) { try { - ArrayList instances = ShardSolrConnector.getShardInstances(solrurls, null, null); + ArrayList instances = RemoteInstance.getShardInstances(solrurls, null, null); this.index.fulltext().connectRemoteSolr(instances); } catch ( final IOException e ) { Log.logException(e); diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index bb36133dc..c6015a1e2 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -61,7 +61,6 @@ import net.yacy.document.parser.html.CharacterCoding; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.meta.URIMetadataRow; -import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.data.word.WordReferenceVars; import net.yacy.kelondro.index.Cache; import net.yacy.kelondro.index.Index; diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 1935473f8..304e9a83b 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -146,7 +146,7 @@ public final class SearchEvent { private final long maxtime; private final ConcurrentHashMap> doubleDomCache; // key = domhash (6 bytes); value = like stack private final int[] flagcount; // flag counter - private final AtomicInteger feedersAlive, feedersTerminated; + private final AtomicInteger feedersAlive, feedersTerminated, snippetFetchAlive; private boolean addRunning; private final AtomicInteger receivedRemoteReferences; private final ReferenceOrder order; @@ -245,6 +245,7 @@ public final class SearchEvent { } this.feedersAlive = new AtomicInteger(0); this.feedersTerminated = new AtomicInteger(0); + this.snippetFetchAlive = new AtomicInteger(0); this.addRunning = true; this.receivedRemoteReferences = new AtomicInteger(0); this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang)); @@ -1084,17 +1085,25 @@ public final class SearchEvent { addResult(getSnippet(localEntry, null)); success = true; } - if (localEntry == null) { + if (SearchEvent.this.snippetFetchAlive.get() >= 10) { + // too many concurrent processes URIMetadataNode p2pEntry = pullOneFilteredFromRWI(true); if (p2pEntry != null) { addResult(getSnippet(p2pEntry, null)); success = true; } } else { - new Thread() { + final URIMetadataNode p2pEntry = pullOneFilteredFromRWI(true); + if (p2pEntry != null) new Thread() { public void run() { - URIMetadataNode p2pEntry = pullOneFilteredFromRWI(true); - if (p2pEntry != null) addResult(getSnippet(p2pEntry, null)); + SearchEvent.this.oneFeederStarted(); + SearchEvent.this.snippetFetchAlive.incrementAndGet(); + try { + addResult(getSnippet(p2pEntry, null)); + } catch (Throwable e) {} finally { + SearchEvent.this.oneFeederTerminated(); + SearchEvent.this.snippetFetchAlive.decrementAndGet(); + } } }.start(); } @@ -1310,7 +1319,7 @@ public final class SearchEvent { } public void oneFeederStarted() { - this.feedersAlive.addAndGet(1); + this.feedersAlive.incrementAndGet(); } public QueryParams getQuery() {