Michael Peter Christen
13 years ago
parent
d54b80327a
commit
0904afe8fb
@ -0,0 +1,261 @@
|
|||||||
|
/**
|
||||||
|
* SolrServerConnector
|
||||||
|
* Copyright 2012 by Michael Peter Christen
|
||||||
|
* First released 21.06.2012 at http://yacy.net
|
||||||
|
*
|
||||||
|
* This library is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU Lesser General Public
|
||||||
|
* License as published by the Free Software Foundation; either
|
||||||
|
* version 2.1 of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This library is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* Lesser General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Lesser General Public License
|
||||||
|
* along with this program in the file lgpl21.txt
|
||||||
|
* If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package net.yacy.cora.services.federated.solr;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import net.yacy.kelondro.logging.Log;
|
||||||
|
import net.yacy.search.index.YaCySchema;
|
||||||
|
|
||||||
|
import org.apache.solr.client.solrj.SolrQuery;
|
||||||
|
import org.apache.solr.client.solrj.SolrServer;
|
||||||
|
import org.apache.solr.client.solrj.SolrServerException;
|
||||||
|
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||||
|
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||||
|
import org.apache.solr.common.SolrDocument;
|
||||||
|
import org.apache.solr.common.SolrDocumentList;
|
||||||
|
import org.apache.solr.common.SolrException;
|
||||||
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
|
|
||||||
|
public class SolrServerConnector extends AbstractSolrConnector implements SolrConnector {
|
||||||
|
|
||||||
|
protected SolrServer server;
|
||||||
|
protected int commitWithinMs; // max time (in ms) before a commit will happen
|
||||||
|
|
||||||
|
protected SolrServerConnector() {
|
||||||
|
this.server = null;
|
||||||
|
this.commitWithinMs = 180000;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void init(SolrServer server) {
|
||||||
|
this.server = server;
|
||||||
|
}
|
||||||
|
|
||||||
|
public SolrServer getServer() {
|
||||||
|
return this.server;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get the solr autocommit delay
|
||||||
|
* @return the maximum waiting time after a solr command until it is transported to the server
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public int getCommitWithinMs() {
|
||||||
|
return this.commitWithinMs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* set the solr autocommit delay
|
||||||
|
* @param c the maximum waiting time after a solr command until it is transported to the server
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void setCommitWithinMs(int c) {
|
||||||
|
this.commitWithinMs = c;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized void close() {
|
||||||
|
try {
|
||||||
|
this.server.commit();
|
||||||
|
this.server = null;
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
Log.logException(e);
|
||||||
|
} catch (IOException e) {
|
||||||
|
Log.logException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getSize() {
|
||||||
|
try {
|
||||||
|
final QueryResponse rsp = this.server.query(AbstractSolrConnector.catchallQuery);
|
||||||
|
if (rsp == null) return 0;
|
||||||
|
final SolrDocumentList docs = rsp.getResults();
|
||||||
|
if (docs == null) return 0;
|
||||||
|
return docs.getNumFound();
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
Log.logException(e);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* delete everything in the solr index
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void clear() throws IOException {
|
||||||
|
try {
|
||||||
|
this.server.deleteByQuery("*:*");
|
||||||
|
this.server.commit();
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void delete(final String id) throws IOException {
|
||||||
|
try {
|
||||||
|
this.server.deleteById(id, this.commitWithinMs);
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void delete(final List<String> ids) throws IOException {
|
||||||
|
try {
|
||||||
|
this.server.deleteById(ids, this.commitWithinMs);
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* delete entries from solr according the given solr query string
|
||||||
|
* @param id the url hash of the entry
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void deleteByQuery(final String querystring) throws IOException {
|
||||||
|
try {
|
||||||
|
this.server.deleteByQuery(querystring, this.commitWithinMs);
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void add(final File file, final String solrId) throws IOException {
|
||||||
|
final ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update/extract");
|
||||||
|
up.addFile(file);
|
||||||
|
up.setParam("literal.id", solrId);
|
||||||
|
up.setParam("uprefix", "attr_");
|
||||||
|
up.setParam("fmap.content", "attr_content");
|
||||||
|
up.setCommitWithin(this.commitWithinMs);
|
||||||
|
//up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
|
||||||
|
try {
|
||||||
|
this.server.request(up);
|
||||||
|
this.server.commit();
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
|
||||||
|
try {
|
||||||
|
this.server.add(solrdoc, this.commitWithinMs);
|
||||||
|
//this.server.commit();
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
Log.logWarning("SolrConnector", e.getMessage() + " DOC=" + solrdoc.toString());
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
|
||||||
|
ArrayList<SolrInputDocument> l = new ArrayList<SolrInputDocument>();
|
||||||
|
for (SolrInputDocument d: solrdocs) l.add(d);
|
||||||
|
try {
|
||||||
|
this.server.add(l, this.commitWithinMs);
|
||||||
|
//this.server.commit();
|
||||||
|
} catch (SolrServerException e) {
|
||||||
|
Log.logWarning("SolrConnector", e.getMessage() + " DOC=" + solrdocs.toString());
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get a query result from solr
|
||||||
|
* to get all results set the query String to "*:*"
|
||||||
|
* @param querystring
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public SolrDocumentList query(final String querystring, final int offset, final int count) throws IOException {
|
||||||
|
// construct query
|
||||||
|
final SolrQuery query = new SolrQuery();
|
||||||
|
query.setQuery(querystring);
|
||||||
|
query.setRows(count);
|
||||||
|
query.setStart(offset);
|
||||||
|
//query.addSortField( "price", SolrQuery.ORDER.asc );
|
||||||
|
|
||||||
|
// query the server
|
||||||
|
//SearchResult result = new SearchResult(count);
|
||||||
|
Throwable error = null; // well this is a bad hack; see https://issues.apache.org/jira/browse/LUCENE-2239
|
||||||
|
// also: https://issues.apache.org/jira/browse/SOLR-2247
|
||||||
|
// we might try also: $JAVA_OPTS -Dsolr.directoryFactory=solr.MMapDirectoryFactory
|
||||||
|
for (int retry = 30; retry > 0; retry--) {
|
||||||
|
try {
|
||||||
|
final QueryResponse rsp = this.server.query(query);
|
||||||
|
final SolrDocumentList docs = rsp.getResults();
|
||||||
|
if (error != null) Log.logWarning("AbstractSolrConnector", "produced search result by silently ignoring an error before, message = " + error.getMessage());
|
||||||
|
return docs;
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
Log.logWarning("AbstractSolrConnection", "problem with query=" + querystring, e);
|
||||||
|
error = e;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new IOException(error.getMessage(), error);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get a document from solr by given id
|
||||||
|
* @param id
|
||||||
|
* @return one result or null if no result exists
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public SolrDocument get(final String id) throws IOException {
|
||||||
|
// construct query
|
||||||
|
StringBuffer sb = new StringBuffer(id.length() + 5);
|
||||||
|
sb.append(YaCySchema.id.getSolrFieldName()).append(':').append('"').append(id).append('"');
|
||||||
|
final SolrQuery query = new SolrQuery();
|
||||||
|
query.setQuery(sb.toString());
|
||||||
|
query.setRows(1);
|
||||||
|
query.setStart(0);
|
||||||
|
|
||||||
|
// query the server
|
||||||
|
Throwable error = null; // well this is a bad hack; see https://issues.apache.org/jira/browse/LUCENE-2239
|
||||||
|
// also: https://issues.apache.org/jira/browse/SOLR-2247
|
||||||
|
// we might try also: $JAVA_OPTS -Dsolr.directoryFactory=solr.MMapDirectoryFactory
|
||||||
|
for (int retry = 30; retry > 0; retry--) {
|
||||||
|
try {
|
||||||
|
final QueryResponse rsp = this.server.query(query);
|
||||||
|
final SolrDocumentList docs = rsp.getResults();
|
||||||
|
if (docs.isEmpty()) return null;
|
||||||
|
if (error != null) Log.logWarning("AbstractSolrConnector", "produced search result by silently ignoring an error before, message = " + error.getMessage());
|
||||||
|
return docs.get(0);
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
Log.logWarning("AbstractSolrConnection", "problem with id=" + id, e);
|
||||||
|
error = e;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new IOException(error.getMessage(), error);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in new issue