boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- removed multi-add of documents (no used)

Browse Source 
- inserted specialized code for size request
pull/1/head
Michael Peter Christen 12 years ago
parent a1a4d9aa94
commit d465773a37
6 changed files with 62 additions and 55 deletions
Show Stats Download Patch File Download Diff File

6
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -21,7 +21,6 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -332,11 +331,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
if (this.solr1 != null) this.solr1.add(solrdoc);
}
@Override
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
for (SolrInputDocument solrdoc: solrdocs) add(solrdoc);
}
/**
* get a query result from solr
* to get all results set the query String to "*:*"

12
source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
Unescape View File

@ -21,7 +21,6 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
@ -162,17 +161,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
}
}
@Override
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
for (SolrInputDocument d: solrdocs) {
try {
this.queue.put(d);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
return this.solr.query(querystring, offset, count, fields);

6
source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java
Unescape View File

@ -21,7 +21,6 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -174,11 +173,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
}
@Override
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
for (SolrInputDocument d: solrdocs) add(d);
}
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;

5
source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java
Unescape View File

@ -151,11 +151,6 @@ public class ShardSolrConnector extends AbstractSolrConnector implements SolrCon
this.connectors.get(this.sharding.select(solrdoc)).add(solrdoc);
}
@Override
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
for (SolrInputDocument d: solrdocs) this.connectors.get(this.sharding.select(d)).add(d);
}
/**
* add a collection of Solr documents
* @param docs

3
source/net/yacy/cora/federate/solr/connector/SolrConnector.java
Unescape View File

@ -21,7 +21,6 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
@ -103,8 +102,6 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
* @throws SolrException
*/
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException;
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException;
/**
* get a field value from solr by given key for the id-field and a field name
* @param key

85
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -22,8 +22,6 @@ package net.yacy.cora.federate.solr.connector;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -37,7 +35,9 @@ import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
@ -47,6 +47,18 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.QueryComponent;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.handler.component.SearchHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.servlet.SolrRequestParsers;
public abstract class SolrServerConnector extends AbstractSolrConnector implements SolrConnector {
@ -107,8 +119,52 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
}
}
private static final SolrRequestParsers _parser = new SolrRequestParsers(null);
@Override
public long getSize() {
String threadname = Thread.currentThread().getName();
Thread.currentThread().setName("solr query: size");
if (this.server instanceof EmbeddedSolrServer) {
EmbeddedSolrServer ess = (EmbeddedSolrServer) this.server;
CoreContainer coreContainer = ess.getCoreContainer();
String coreName = coreContainer.getDefaultCoreName();
SolrCore core = coreContainer.getCore(coreName);
if (core == null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No such core: " + coreName);
try {
SolrParams params = AbstractSolrConnector.catchSuccessQuery;
QueryRequest request = new QueryRequest(AbstractSolrConnector.catchSuccessQuery);
SolrQueryRequest req = _parser.buildRequestFrom(core, params, request.getContentStreams());
String path = "/select";
req.getContext().put("path", path);
SolrQueryResponse rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
SolrRequestHandler handler = core.getRequestHandler(path);
SearchHandler sh = (SearchHandler) handler;
List<SearchComponent> components = sh.getComponents();
ResponseBuilder rb = new ResponseBuilder(req, rsp, components);
QueryComponent qc = (QueryComponent) components.get(0);
qc.prepare(rb);
qc.process(rb);
qc.finishStage(rb);
int hits = rb.getResults().docList.matches();
if (req != null) req.close();
core.close();
SolrRequestInfo.clearRequestInfo();
Thread.currentThread().setName(threadname);
return hits;
} catch (final Throwable e) {
log.warn(e);
Thread.currentThread().setName(threadname);
return 0;
}
}
Thread.currentThread().setName(threadname);
return getSize0();
}
public long getSize0() {
/*
if (this.server instanceof EmbeddedSolrServer) {
EmbeddedSolrServer ess = (EmbeddedSolrServer) this.server;
@ -213,40 +269,23 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
//this.server.deleteById((String) solrdoc.getFieldValue(YaCySchema.id.getSolrFieldName()));
this.server.add(solrdoc, this.commitWithinMs);
}
} catch (SolrServerException e) {
// ok try this again and delete the document in advance
} catch (Throwable e) {
// catches "version conflict for": try this again and delete the document in advance
try {
this.server.deleteById((String) solrdoc.getFieldValue(YaCySchema.id.getSolrFieldName()));
//this.server.commit();
} catch (SolrServerException e1) {}
try {
synchronized (this.server) {
this.server.add(solrdoc, this.commitWithinMs);
}
} catch (SolrServerException ee) {
} catch (Throwable ee) {
log.warn(e.getMessage() + " DOC=" + solrdoc.toString());
throw new IOException(ee);
}
}
}
@Override
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
ArrayList<SolrInputDocument> l = new ArrayList<SolrInputDocument>();
try {
synchronized (this.server) {
for (SolrInputDocument d: solrdocs) {
//this.server.deleteById((String) d.getFieldValue(YaCySchema.id.getSolrFieldName()));
l.add(d);
}
this.server.add(l, this.commitWithinMs);
//this.server.commit();
}
} catch (SolrServerException e) {
log.warn(e.getMessage() + " DOC=" + solrdocs.toString());
throw new IOException(e);
}
}
/**
* get a query result from solr
* to get all results set the query String to "*:*"

Write Preview
Loading…
Cancel
Save