boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- used reger's LukeRequest to generalize the index info in

Browse Source 
SolrServerConnector
- used the LukeRequest in SolrServerConnector to replace the index size
method by a getNumDocs request to a LukeRequest result
pull/1/head
Michael Peter Christen 12 years ago
parent 39fceb5ccf
commit 36035e0a0a
3 changed files with 28 additions and 75 deletions
Show Stats Download Patch File Download Diff File

57
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -22,34 +22,25 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.List;
import net.yacy.cora.federate.solr.instance.EmbeddedInstance;
import net.yacy.cora.federate.solr.instance.SolrInstance;
import net.yacy.cora.util.ConcurrentLog;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.QueryComponent;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.handler.component.SearchHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.servlet.SolrRequestParsers;
public class EmbeddedSolrConnector extends SolrServerConnector implements SolrConnector {
@ -58,7 +49,6 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
private final SearchHandler requestHandler;
private final EmbeddedInstance instance;
private final String coreName;
private SolrCore core;
public EmbeddedSolrConnector(EmbeddedInstance instance) {
@ -69,7 +59,6 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
this.requestHandler.init(new NamedList<Object>());
this.requestHandler.inform(this.core);
super.init(this.instance.getDefaultServer());
this.coreName = ((EmbeddedSolrServer) this.server).getCoreContainer().getDefaultCoreName();
}
public EmbeddedSolrConnector(EmbeddedInstance instance, String coreName) {
@ -80,7 +69,6 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
this.requestHandler.init(new NamedList<Object>());
this.requestHandler.inform(this.core);
super.init(this.instance.getServer(coreName));
this.coreName = coreName;
}
public SolrInstance getInstance() {
@ -95,51 +83,6 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
return this.core.getSolrConfig();
}
private static final SolrRequestParsers _parser = new SolrRequestParsers(null);
/**
* get the size of the index. We override the implementation in SolrServerConnector
* because we can do this with more efficiently in a different way for embedded indexes.
*/
@Override
public long getSize() {
if (this.server == null) return 0;
String threadname = Thread.currentThread().getName();
Thread.currentThread().setName("solr query: size");
EmbeddedSolrServer ess = (EmbeddedSolrServer) this.server;
CoreContainer coreContainer = ess.getCoreContainer();
SolrCore core = coreContainer.getCore(this.coreName);
if (core == null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No such core: " + this.coreName);
try {
SolrParams params = AbstractSolrConnector.catchSuccessQuery;
QueryRequest request = new QueryRequest(AbstractSolrConnector.catchSuccessQuery);
SolrQueryRequest req = _parser.buildRequestFrom(core, params, request.getContentStreams());
String path = "/select";
req.getContext().put("path", path);
SolrQueryResponse rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
SolrRequestHandler handler = core.getRequestHandler(path);
SearchHandler sh = (SearchHandler) handler;
List<SearchComponent> components = sh.getComponents();
ResponseBuilder rb = new ResponseBuilder(req, rsp, components);
QueryComponent qc = (QueryComponent) components.get(0);
qc.prepare(rb);
qc.process(rb);
qc.finishStage(rb);
int hits = rb.getResults().docList.matches();
if (req != null) req.close();
core.close();
SolrRequestInfo.clearRequestInfo();
Thread.currentThread().setName(threadname);
return hits;
} catch (final Throwable e) {
log.warn(e);
Thread.currentThread().setName(threadname);
return 0;
}
}
@Override
public synchronized void close() {
try {this.commit(false);} catch (final Throwable e) {ConcurrentLog.logException(e);}

33
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -30,14 +30,15 @@ import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.schema.CollectionSchema;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;
import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.client.solrj.response.LukeResponse;
public abstract class SolrServerConnector extends AbstractSolrConnector implements SolrConnector {
@ -104,11 +105,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public long getSize() {
if (this.server == null) return 0;
try {
final QueryResponse rsp = getResponseByParams(AbstractSolrConnector.catchSuccessQuery);
if (rsp == null) return 0;
final SolrDocumentList docs = rsp.getResults();
if (docs == null) return 0;
return docs.getNumFound();
return getIndexBrowser().getNumDocs();
} catch (final Throwable e) {
log.warn(e);
return 0;
@ -261,5 +258,23 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
}
}
}
public Collection<FieldInfo> getFields() throws SolrServerException {
// get all fields contained in index
return getIndexBrowser().getFieldInfo().values();
}
private LukeResponse getIndexBrowser() throws SolrServerException {
// get all fields contained in index
final LukeRequest lukeRequest = new LukeRequest();
lukeRequest.setNumTerms(1);
LukeResponse lukeResponse = null;
try {
lukeResponse = lukeRequest.process(this.server);
} catch (IOException e) {
throw new SolrServerException(e.getMessage());
}
return lukeResponse;
}
}

13
source/net/yacy/migration.java
Unescape View File

@ -24,6 +24,7 @@ package net.yacy;
import net.yacy.search.index.ReindexSolrBusyThread;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@ -46,8 +47,7 @@ import net.yacy.search.index.Fulltext;
import net.yacy.search.schema.CollectionConfiguration;
import net.yacy.search.schema.CollectionSchema;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.client.solrj.response.LukeResponse;
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;
public class migration {
//SVN constants
@ -371,11 +371,8 @@ public class migration {
ReindexSolrBusyThread reidx = new ReindexSolrBusyThread(null); // ("*:*" would reindex all);
try { // get all fields contained in index
LukeRequest lukeRequest = new LukeRequest();
lukeRequest.setNumTerms(1);
LukeResponse lukeResponse = lukeRequest.process(Switchboard.getSwitchboard().index.fulltext().getDefaultEmbeddedConnector().getServer());
for (LukeResponse.FieldInfo solrfield : lukeResponse.getFieldInfo().values()) {
Collection<FieldInfo> solrfields = Switchboard.getSwitchboard().index.fulltext().getDefaultEmbeddedConnector().getFields();
for (FieldInfo solrfield : solrfields) {
if (!colcfg.contains(solrfield.getName()) && !omitFields.contains(solrfield.getName())) { // add found fields not in config for reindexing
reidx.addSelectFieldname(solrfield.getName());
}
@ -383,8 +380,6 @@ public class migration {
lukeCheckok = true;
} catch (final SolrServerException ex) {
ConcurrentLog.logException(ex);
} catch (final IOException ex) {
ConcurrentLog.logException(ex);
}
if (!lukeCheckok) { // if luke failed alternatively use config and manual list

Write Preview
Loading…
Cancel
Save