From d39463a85ccc6647bff55b5363b9ebd6abad9eb6 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 17 Aug 2012 17:05:46 +0200 Subject: [PATCH] added deleteByQuery to solr connectors --- .classpath | 4 ++-- .../federated/solr/AbstractSolrConnector.java | 14 ++++++++++++++ .../federated/solr/MirrorSolrConnector.java | 9 +++++++++ .../federated/solr/MultipleSolrConnector.java | 5 +++++ .../federated/solr/RetrySolrConnector.java | 15 +++++++++++++++ .../federated/solr/ShardSolrConnector.java | 5 +++++ .../services/federated/solr/SolrConnector.java | 7 +++++++ 7 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.classpath b/.classpath index bdf89be5c..bef2c2b61 100644 --- a/.classpath +++ b/.classpath @@ -60,8 +60,8 @@ - - + + diff --git a/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java b/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java index a7de26fcd..7ba32d6ba 100644 --- a/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java @@ -141,6 +141,20 @@ public class AbstractSolrConnector implements SolrConnector { } } + /** + * delete entries from solr according the given solr query string + * @param id the url hash of the entry + * @throws IOException + */ + @Override + public void deleteByQuery(final String querystring) throws IOException { + try { + this.server.deleteByQuery(querystring, this.commitWithinMs); + } catch (final Throwable e) { + throw new IOException(e); + } + } + @Override public boolean exists(final String id) throws IOException { try { diff --git a/source/net/yacy/cora/services/federated/solr/MirrorSolrConnector.java b/source/net/yacy/cora/services/federated/solr/MirrorSolrConnector.java index bf306a193..4830d3bdd 100644 --- a/source/net/yacy/cora/services/federated/solr/MirrorSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/MirrorSolrConnector.java @@ -170,6 +170,15 @@ public class MirrorSolrConnector implements SolrConnector { if (this.solr1 != null) this.solr1.delete(ids); } + @Override + public void deleteByQuery(final String querystring) throws IOException { + if (this.solr0 != null) this.solr0.deleteByQuery(querystring); + if (this.solr1 != null) this.solr1.deleteByQuery(querystring); + this.hitCache.clear(); + this.missCache.clear(); + this.documentCache.clear(); + } + /** * check if a given id exists in solr * @param id diff --git a/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java b/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java index 983a89cc8..144a1287b 100644 --- a/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java @@ -128,6 +128,11 @@ public class MultipleSolrConnector implements SolrConnector { this.solr.delete(ids); } + @Override + public void deleteByQuery(final String querystring) throws IOException { + this.solr.deleteByQuery(querystring); + } + @Override public boolean exists(String id) throws IOException { return this.solr.exists(id); diff --git a/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java b/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java index 0fd577676..ec7f48e3c 100644 --- a/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java @@ -103,6 +103,21 @@ public class RetrySolrConnector implements SolrConnector { if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); } + @Override + public void deleteByQuery(final String querystring) throws IOException { + final long t = System.currentTimeMillis() + this.retryMaxTime; + Throwable ee = null; + while (System.currentTimeMillis() < t) try { + this.solrConnector.deleteByQuery(querystring); + return; + } catch (final Throwable e) { + ee = e; + try {Thread.sleep(10);} catch (final InterruptedException e1) {} + continue; + } + if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage()); + } + @Override public boolean exists(final String id) throws IOException { final long t = System.currentTimeMillis() + this.retryMaxTime; diff --git a/source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java b/source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java index 52eaa6309..eddc6fff5 100644 --- a/source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java @@ -104,6 +104,11 @@ public class ShardSolrConnector implements SolrConnector { for (final SolrConnector connector: this.connectors) connector.delete(ids); } + @Override + public void deleteByQuery(final String querystring) throws IOException { + for (final SolrConnector connector: this.connectors) connector.deleteByQuery(querystring); + } + /** * check if a given id exists in solr * @param id diff --git a/source/net/yacy/cora/services/federated/solr/SolrConnector.java b/source/net/yacy/cora/services/federated/solr/SolrConnector.java index 41434c4a0..cf79f0796 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/SolrConnector.java @@ -72,6 +72,13 @@ public interface SolrConnector { */ public void delete(final List ids) throws IOException; + /** + * delete entries from solr according the given solr query string + * @param id the url hash of the entry + * @throws IOException + */ + public void deleteByQuery(final String querystring) throws IOException; + /** * check if a given id exists in solr * @param id