follow later) - added embedded solr connector which can connect to solr programmatically (without using a server in between)pull/1/head
Michael Peter Christen
13 years ago
parent
15f4551d88
commit
3f55dc7c1e
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,191 @@
|
||||
/**
|
||||
* AbstractSolrConnector
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 21.06.2012 at http://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.services.federated.solr;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.search.index.SolrField;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
|
||||
public class AbstractSolrConnector implements SolrConnector {
|
||||
|
||||
protected SolrServer server;
|
||||
|
||||
protected AbstractSolrConnector() {
|
||||
this.server = null;
|
||||
}
|
||||
|
||||
protected void init(SolrServer server) {
|
||||
this.server = server;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() {
|
||||
try {
|
||||
this.server.commit();
|
||||
} catch (SolrServerException e) {
|
||||
Log.logException(e);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSize() {
|
||||
try {
|
||||
final SolrDocumentList list = get("*:*", 0, 1);
|
||||
return list.getNumFound();
|
||||
} catch (final Throwable e) {
|
||||
Log.logException(e);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* delete everything in the solr index
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public void clear() throws IOException {
|
||||
try {
|
||||
this.server.deleteByQuery("*:*");
|
||||
this.server.commit();
|
||||
} catch (final Throwable e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(final String id) throws IOException {
|
||||
try {
|
||||
this.server.deleteById(id);
|
||||
} catch (final Throwable e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(final List<String> ids) throws IOException {
|
||||
try {
|
||||
this.server.deleteById(ids);
|
||||
} catch (final Throwable e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exists(final String id) throws IOException {
|
||||
try {
|
||||
final SolrDocumentList list = get(SolrField.id.getSolrFieldName() + ":" + id, 0, 1);
|
||||
return list.getNumFound() > 0;
|
||||
} catch (final Throwable e) {
|
||||
Log.logException(e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public void add(final File file, final String solrId) throws IOException {
|
||||
final ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update/extract");
|
||||
up.addFile(file);
|
||||
up.setParam("literal.id", solrId);
|
||||
up.setParam("uprefix", "attr_");
|
||||
up.setParam("fmap.content", "attr_content");
|
||||
//up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
|
||||
try {
|
||||
this.server.request(up);
|
||||
this.server.commit();
|
||||
} catch (final Throwable e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(final SolrDoc solrdoc) throws IOException, SolrException {
|
||||
try {
|
||||
this.server.add(solrdoc,180000); // commitWithIn 180s
|
||||
//this.server.commit();
|
||||
} catch (SolrServerException e) {
|
||||
Log.logWarning("SolrConnector", e.getMessage() + " DOC=" + solrdoc.toString());
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(final Collection<SolrDoc> solrdocs) throws IOException, SolrException {
|
||||
ArrayList<SolrInputDocument> l = new ArrayList<SolrInputDocument>();
|
||||
for (SolrDoc d: solrdocs) l.add(d);
|
||||
try {
|
||||
this.server.add(l,180000); // commitWithIn 120s
|
||||
//this.server.commit();
|
||||
} catch (SolrServerException e) {
|
||||
Log.logWarning("SolrConnector", e.getMessage() + " DOC=" + solrdocs.toString());
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get a query result from solr
|
||||
* to get all results set the query String to "*:*"
|
||||
* @param querystring
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public SolrDocumentList get(final String querystring, final int offset, final int count) throws IOException {
|
||||
// construct query
|
||||
final SolrQuery query = new SolrQuery();
|
||||
query.setQuery(querystring);
|
||||
query.setRows(count);
|
||||
query.setStart(offset);
|
||||
//query.addSortField( "price", SolrQuery.ORDER.asc );
|
||||
|
||||
// query the server
|
||||
//SearchResult result = new SearchResult(count);
|
||||
try {
|
||||
final QueryResponse rsp = this.server.query( query );
|
||||
final SolrDocumentList docs = rsp.getResults();
|
||||
return docs;
|
||||
// add the docs into the YaCy search result container
|
||||
/*
|
||||
for (SolrDocument doc: docs) {
|
||||
result.put(element)
|
||||
}
|
||||
*/
|
||||
} catch (final Throwable e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
|
||||
//return result;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
/**
|
||||
* EmbeddedSolrConnector
|
||||
* Copyright 2012 by Michael Peter Christen
|
||||
* First released 21.06.2012 at http://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.search.solr;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
|
||||
import net.yacy.cora.services.federated.solr.AbstractSolrConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrConnector;
|
||||
|
||||
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
public class EmbeddedSolrConnector extends AbstractSolrConnector implements SolrConnector {
|
||||
|
||||
private final CoreContainer core;
|
||||
|
||||
public EmbeddedSolrConnector(File storagePath, File configFile) throws IOException {
|
||||
super();
|
||||
try {
|
||||
this.core = new CoreContainer(storagePath.getAbsolutePath(), configFile);
|
||||
} catch (ParserConfigurationException e) {
|
||||
throw new IOException(e.getMessage(), e);
|
||||
} catch (SAXException e) {
|
||||
throw new IOException(e.getMessage(), e);
|
||||
}
|
||||
super.init(new EmbeddedSolrServer(this.core, "metadata"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
super.close();
|
||||
this.core.shutdown();
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue