boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 1be0025a9c
commit a5eb91fa60
9 changed files with 31 additions and 63 deletions
Show Stats Download Patch File Download Diff File

12
htroot/IndexFederated_p.java
Unescape View File

@ -31,9 +31,9 @@ import java.util.Iterator;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.solr.SolrConnector;
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
import net.yacy.cora.services.federated.solr.SolrShardingSelection;
import net.yacy.cora.services.federated.solr.SolrSingleConnector;
import net.yacy.cora.services.federated.solr.ShardSolrConnector;
import net.yacy.cora.services.federated.solr.ShardSelection;
import net.yacy.cora.services.federated.solr.SingleSolrConnector;
import net.yacy.cora.storage.ConfigurationSet;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
@ -89,7 +89,7 @@ public class IndexFederated_p {
// switch on
final boolean usesolr = sb.getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0;
try {
sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr((usesolr) ? new SolrShardingConnector(solrurls, SolrShardingSelection.Method.MODULO_HOST_MD5, 10000, true) : null);
sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr((usesolr) ? new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true) : null);
} catch (final IOException e) {
Log.logException(e);
sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr(null);
@ -132,8 +132,8 @@ public class IndexFederated_p {
} else {
prop.put("table", 1);
final SolrConnector solr = sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).getSolr();
final long[] size = (solr instanceof SolrShardingConnector) ? ((SolrShardingConnector) solr).getSizeList() : new long[]{((SolrSingleConnector) solr).getSize()};
final String[] urls = (solr instanceof SolrShardingConnector) ? ((SolrShardingConnector) solr).getAdminInterfaceList() : new String[]{((SolrSingleConnector) solr).getAdminInterface()};
final long[] size = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getSizeList() : new long[]{((SingleSolrConnector) solr).getSize()};
final String[] urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaceList() : new String[]{((SingleSolrConnector) solr).getAdminInterface()};
boolean dark = false;
for (int i = 0; i < size.length; i++) {
prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark;

4
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -38,7 +38,7 @@ import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.services.federated.solr.SolrConnector;
import net.yacy.cora.services.federated.solr.SolrDoc;
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
import net.yacy.cora.services.federated.solr.ShardSolrConnector;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.Index;
@ -114,7 +114,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
this.stack = new LinkedBlockingQueue<byte[]>();
}
public ZURL(final SolrShardingConnector solrConnector,
public ZURL(final ShardSolrConnector solrConnector,
final SolrConfiguration solrConfiguration) {
this.solrConnector = solrConnector;
this.solrConfiguration = solrConfiguration;

10
source/net/yacy/cora/services/federated/solr/SolrMultipleConnector.java → source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java
Unescape View File

@ -8,7 +8,7 @@ import java.util.concurrent.ArrayBlockingQueue;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
public class SolrMultipleConnector implements SolrConnector {
public class MultipleSolrConnector implements SolrConnector {
private final static SolrDoc POISON_DOC = new SolrDoc();
@ -16,8 +16,8 @@ public class SolrMultipleConnector implements SolrConnector {
private final AddWorker[] worker;
private final SolrConnector solr;
public SolrMultipleConnector(final String url, int connections) throws IOException {
this.solr = new SolrSingleConnector(url);
public MultipleSolrConnector(final String url, int connections) throws IOException {
this.solr = new SingleSolrConnector(url);
this.queue = new ArrayBlockingQueue<SolrDoc>(1000);
this.worker = new AddWorker[connections];
for (int i = 0; i < connections; i++) {
@ -29,13 +29,13 @@ public class SolrMultipleConnector implements SolrConnector {
private class AddWorker extends Thread {
private final SolrConnector solr;
public AddWorker(final String url) throws IOException {
this.solr = new SolrSingleConnector(url);
this.solr = new SingleSolrConnector(url);
}
@Override
public void run() {
SolrDoc doc;
try {
while ((doc = SolrMultipleConnector.this.queue.take()) != POISON_DOC) {
while ((doc = MultipleSolrConnector.this.queue.take()) != POISON_DOC) {
try {
this.solr.add(doc);
} catch (SolrException e) {

4
source/net/yacy/cora/services/federated/solr/SolrRetryConnector.java → source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java
Unescape View File

@ -31,12 +31,12 @@ import java.util.List;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
public class SolrRetryConnector implements SolrConnector {
public class RetrySolrConnector implements SolrConnector {
private final SolrConnector solrConnector;
private final long retryMaxTime;
public SolrRetryConnector(final SolrConnector solrConnector, final long retryMaxTime) {
public RetrySolrConnector(final SolrConnector solrConnector, final long retryMaxTime) {
this.solrConnector = solrConnector;
this.retryMaxTime = retryMaxTime;
}

4
source/net/yacy/cora/services/federated/solr/SolrShardingSelection.java → source/net/yacy/cora/services/federated/solr/ShardSelection.java
Unescape View File

@ -33,7 +33,7 @@ import java.security.NoSuchAlgorithmException;
import java.util.concurrent.atomic.AtomicLong;
import net.yacy.search.index.SolrField;
public class SolrShardingSelection {
public class ShardSelection {
public final static Charset charsetUTF8;
static {
@ -47,7 +47,7 @@ public class SolrShardingSelection {
MODULO_HOST_MD5, ROUND_ROBIN;
}
public SolrShardingSelection(final Method method, final int dimension) {
public ShardSelection(final Method method, final int dimension) {
this.method = method;
this.dimension = dimension;
this.chardID = new AtomicLong(0);

12
source/net/yacy/cora/services/federated/solr/SolrShardingConnector.java → source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java
Unescape View File

@ -37,22 +37,22 @@ import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
public class SolrShardingConnector implements SolrConnector {
public class ShardSolrConnector implements SolrConnector {
private final List<SolrConnector> connectors;
private final SolrShardingSelection sharding;
private final ShardSelection sharding;
private final String[] urls;
public SolrShardingConnector(final String urlList, final SolrShardingSelection.Method method, final long timeout, boolean multipleConnections) throws IOException {
public ShardSolrConnector(final String urlList, final ShardSelection.Method method, final long timeout, boolean multipleConnections) throws IOException {
urlList.replace(' ', ',');
this.urls = urlList.split(",");
this.connectors = new ArrayList<SolrConnector>();
SolrConnector s;
for (final String u: this.urls) {
s = multipleConnections ? new SolrMultipleConnector(u.trim(), 2) : new SolrSingleConnector(u.trim());
this.connectors.add(new SolrRetryConnector(s, timeout));
s = multipleConnections ? new MultipleSolrConnector(u.trim(), 2) : new SingleSolrConnector(u.trim());
this.connectors.add(new RetrySolrConnector(s, timeout));
}
this.sharding = new SolrShardingSelection(method, this.urls.length);
this.sharding = new ShardSelection(method, this.urls.length);
}
@Override

8
source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java → source/net/yacy/cora/services/federated/solr/SingleSolrConnector.java
Unescape View File

@ -44,7 +44,7 @@ import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
public class SolrSingleConnector extends AbstractSolrConnector implements SolrConnector {
public class SingleSolrConnector extends AbstractSolrConnector implements SolrConnector {
private final String solrurl, host, solrpath, solraccount, solrpw;
private final int port;
@ -55,7 +55,7 @@ public class SolrSingleConnector extends AbstractSolrConnector implements SolrCo
* @param scheme
* @throws IOException
*/
public SolrSingleConnector(final String url) throws IOException {
public SingleSolrConnector(final String url) throws IOException {
super();
this.solrurl = url;
@ -113,9 +113,9 @@ public class SolrSingleConnector extends AbstractSolrConnector implements SolrCo
}
public static void main(final String args[]) {
SolrSingleConnector solr;
SingleSolrConnector solr;
try {
solr = new SolrSingleConnector("http://127.0.0.1:8983/solr");
solr = new SingleSolrConnector("http://127.0.0.1:8983/solr");
solr.clear();
final File exampleDir = new File("test/parsertest/");
long t, t0, a = 0;

32
source/net/yacy/cora/services/federated/solr/SolrScheme.java
Unescape View File

@ -1,32 +0,0 @@
/**
* SolrScheme
* Copyright 2011 by Michael Peter Christen
* First released 09.05.2012 at http://yacy.net
*
* $LastChangedDate: 2011-04-14 22:05:04 +0200 (Do, 14 Apr 2011) $
* $LastChangedRevision: 7654 $
* $LastChangedBy: orbiter $
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.services.federated.solr;
public interface SolrScheme {
public SolrDoc toSolr();
}

8
source/net/yacy/search/Switchboard.java
Unescape View File

@ -94,8 +94,8 @@ import net.yacy.cora.protocol.TimeoutRequest;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.protocol.http.ProxySettings;
import net.yacy.cora.services.federated.solr.SolrDoc;
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
import net.yacy.cora.services.federated.solr.SolrShardingSelection;
import net.yacy.cora.services.federated.solr.ShardSolrConnector;
import net.yacy.cora.services.federated.solr.ShardSelection;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.document.Condenser;
import net.yacy.document.Document;
@ -407,9 +407,9 @@ public final class Switchboard extends serverSwitch
try {
this.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr(
(usesolr) ? new SolrShardingConnector(
(usesolr) ? new ShardSolrConnector(
solrurls,
SolrShardingSelection.Method.MODULO_HOST_MD5,
ShardSelection.Method.MODULO_HOST_MD5,
10000, true) : null);
} catch ( final IOException e ) {
Log.logException(e);

Write Preview
Loading…
Cancel
Save