From a632b0d2a4ff84ab775e6d820f14d16e31a8c973 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Thu, 27 Feb 2014 12:50:40 +0100 Subject: [PATCH] added a forced commit to index deletion to enable synchronized index updates --- htroot/IndexDeletion_p.java | 12 ++++++-- .../ConcurrentUpdateSolrConnector.java | 28 ++++++++++--------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/htroot/IndexDeletion_p.java b/htroot/IndexDeletion_p.java index 89e7bba75..40ddcaeda 100644 --- a/htroot/IndexDeletion_p.java +++ b/htroot/IndexDeletion_p.java @@ -35,6 +35,7 @@ import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ScoreMap; +import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.WorkTables; import net.yacy.search.Switchboard; import net.yacy.search.query.QueryModifier; @@ -52,8 +53,7 @@ public class IndexDeletion_p { SolrConnector defaultConnector = sb.index.fulltext().getDefaultConnector(); SolrConnector webgraphConnector = sb.index.fulltext().getWebgraphConnector(); - if (post != null && post.size() > 0) defaultConnector.commit(false); // we must do a commit here because the user cannot see a proper count. - prop.put("doccount", defaultConnector.getSize()); + if (post == null || post.size() == 0) defaultConnector.commit(false); // we must do a commit here because the user cannot see a proper count. // Delete by URL Matching String urldelete = post == null ? "" : post.get("urldelete", ""); @@ -147,6 +147,7 @@ public class IndexDeletion_p { prop.put("urldelete-active", count == 0 ? 2 : 1); } else { sb.remove(ids); + defaultConnector.commit(false); sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs matching with " + urldelete); prop.put("urldelete-active", 2); } @@ -162,6 +163,7 @@ public class IndexDeletion_p { } else { try { defaultConnector.deleteByQuery(regexquery); + defaultConnector.commit(false); sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, regex match = " + urldelete); } catch (final IOException e) { } @@ -187,6 +189,7 @@ public class IndexDeletion_p { } else { try { defaultConnector.deleteByQuery(collection1Query); + defaultConnector.commit(false); if (webgraphConnector != null) webgraphConnector.deleteByQuery(webgraphQuery); sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs older than " + timedelete_number + " " + timedelete_unit); } catch (final IOException e) { @@ -209,6 +212,7 @@ public class IndexDeletion_p { } else { try { defaultConnector.deleteByQuery(query); + defaultConnector.commit(false); sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, collection " + collectiondelete); } catch (final IOException e) { } @@ -227,7 +231,10 @@ public class IndexDeletion_p { prop.put("querydelete-active", count == 0 ? 2 : 1); } else { try { + ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size before deletion = " + defaultConnector.getSize()); defaultConnector.deleteByQuery(querydelete); + defaultConnector.commit(false); + ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size after commit = " + defaultConnector.getSize()); sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, solr query, q = " + querydelete); } catch (final IOException e) { } @@ -235,6 +242,7 @@ public class IndexDeletion_p { } prop.put("querydelete-active_count", count); } + prop.put("doccount", defaultConnector.getSize()); // return rewrite properties return prop; diff --git a/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java index 1e10d3d93..1e743c47f 100644 --- a/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java @@ -301,18 +301,16 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { @Override public void commit(boolean softCommit) { - if (!softCommit) { - long timeout = System.currentTimeMillis() + 1000; - ensureAliveDeletionHandler(); - while (this.deleteQueue.size() > 0) { - try {Thread.sleep(10);} catch (final InterruptedException e) {} - if (System.currentTimeMillis() > timeout) break; - } - ensureAliveUpdateHandler(); - while (this.updateQueue.size() > 0) { - try {Thread.sleep(10);} catch (final InterruptedException e) {} - if (System.currentTimeMillis() > timeout) break; - } + long timeout = System.currentTimeMillis() + 1000; + ensureAliveDeletionHandler(); + while (this.deleteQueue.size() > 0) { + try {Thread.sleep(10);} catch (final InterruptedException e) {} + if (System.currentTimeMillis() > timeout) break; + } + ensureAliveUpdateHandler(); + while (this.updateQueue.size() > 0) { + try {Thread.sleep(10);} catch (final InterruptedException e) {} + if (System.currentTimeMillis() > timeout) break; } this.connector.commit(softCommit); } @@ -337,7 +335,9 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { ensureAliveDeletionHandler(); try {this.deleteQueue.put(POISON_ID);} catch (final InterruptedException e) {} ensureAliveUpdateHandler(); - try {this.updateQueue.put(POISON_DOCUMENT);} catch (final InterruptedException e) {} + for (int i = 0; i < this.updateHandler.length; i++) { + try {this.updateQueue.put(POISON_DOCUMENT);} catch (final InterruptedException e) {} + } try {this.deletionHandler.join();} catch (final InterruptedException e) {} for (Thread t: this.updateHandler) try {t.join();} catch (final InterruptedException e) {} this.connector.close(); @@ -361,6 +361,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { removeIdFromUpdateQueue(id); this.metadataCache.remove(id); this.missCache.add(id); + ensureAliveDeletionHandler(); if (this.deletionHandler.isAlive()) { try {this.deleteQueue.put(id);} catch (final InterruptedException e) {} } else { @@ -375,6 +376,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector { this.metadataCache.remove(id); this.missCache.add(id); } + ensureAliveDeletionHandler(); if (this.deletionHandler.isAlive()) { for (String id: ids) try {this.deleteQueue.put(id);} catch (final InterruptedException e) {} } else {