added a forced commit to index deletion to enable synchronized index

updates

htroot/IndexDeletion_p.java

@@ -35,6 +35,7 @@ import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
 import net.yacy.cora.federate.solr.connector.SolrConnector;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.cora.sorting.ScoreMap;
+import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.data.WorkTables;
 import net.yacy.search.Switchboard;
 import net.yacy.search.query.QueryModifier;
@@ -52,8 +53,7 @@ public class IndexDeletion_p {
 
         SolrConnector defaultConnector = sb.index.fulltext().getDefaultConnector();
         SolrConnector webgraphConnector = sb.index.fulltext().getWebgraphConnector();
-        if (post != null && post.size() > 0) defaultConnector.commit(false); // we must do a commit here because the user cannot see a proper count.
+        if (post == null || post.size() == 0) defaultConnector.commit(false); // we must do a commit here because the user cannot see a proper count.
-        prop.put("doccount", defaultConnector.getSize());
 
         // Delete by URL Matching
         String urldelete = post == null ? "" : post.get("urldelete", "");
@@ -147,6 +147,7 @@ public class IndexDeletion_p {
                     prop.put("urldelete-active", count == 0 ? 2 : 1);
                 } else {
                     sb.remove(ids);
+                    defaultConnector.commit(false);
                     sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs matching with " + urldelete);
                     prop.put("urldelete-active", 2);
                 }
@@ -162,6 +163,7 @@ public class IndexDeletion_p {
                 } else {
                     try {
                         defaultConnector.deleteByQuery(regexquery);
+                        defaultConnector.commit(false);
                         sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, regex match = " + urldelete);
                     } catch (final IOException e) {
                     }
@@ -187,6 +189,7 @@ public class IndexDeletion_p {
             } else {
                 try {
                     defaultConnector.deleteByQuery(collection1Query);
+                    defaultConnector.commit(false);
                     if (webgraphConnector != null) webgraphConnector.deleteByQuery(webgraphQuery);
                     sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs older than " + timedelete_number + " " + timedelete_unit);
                 } catch (final IOException e) {
@@ -209,6 +212,7 @@ public class IndexDeletion_p {
             } else {
                 try {
                     defaultConnector.deleteByQuery(query);
+                    defaultConnector.commit(false);
                     sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, collection " + collectiondelete);
                 } catch (final IOException e) {
                 }
@@ -227,7 +231,10 @@ public class IndexDeletion_p {
                 prop.put("querydelete-active", count == 0 ? 2 : 1);
             } else {
                 try {
+                    ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size before deletion = " + defaultConnector.getSize());
                     defaultConnector.deleteByQuery(querydelete);
+                    defaultConnector.commit(false);
+                    ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size after commit = " + defaultConnector.getSize());
                     sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, solr query, q = " + querydelete);
                 } catch (final IOException e) {
                 }
@@ -235,6 +242,7 @@ public class IndexDeletion_p {
             }
             prop.put("querydelete-active_count", count);
         }
+        prop.put("doccount", defaultConnector.getSize());
         
         // return rewrite properties
         return prop;

source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java

@@ -301,18 +301,16 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
 
     @Override
     public void commit(boolean softCommit) {
-        if (!softCommit) {
+        long timeout = System.currentTimeMillis() + 1000;
-            long timeout = System.currentTimeMillis() + 1000;
+        ensureAliveDeletionHandler();
-            ensureAliveDeletionHandler();
+        while (this.deleteQueue.size() > 0) {
-            while (this.deleteQueue.size() > 0) {
+            try {Thread.sleep(10);} catch (final InterruptedException e) {}
-                try {Thread.sleep(10);} catch (final InterruptedException e) {}
+            if (System.currentTimeMillis() > timeout) break;
-                if (System.currentTimeMillis() > timeout) break;
+        }
-            }
+        ensureAliveUpdateHandler();
-            ensureAliveUpdateHandler();
+        while (this.updateQueue.size() > 0) {
-            while (this.updateQueue.size() > 0) {
+            try {Thread.sleep(10);} catch (final InterruptedException e) {}
-                try {Thread.sleep(10);} catch (final InterruptedException e) {}
+            if (System.currentTimeMillis() > timeout) break;
-                if (System.currentTimeMillis() > timeout) break;
-            }
         }
         this.connector.commit(softCommit);
     }
@@ -337,7 +335,9 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
         ensureAliveDeletionHandler();
         try {this.deleteQueue.put(POISON_ID);} catch (final InterruptedException e) {}
         ensureAliveUpdateHandler();
-        try {this.updateQueue.put(POISON_DOCUMENT);} catch (final InterruptedException e) {}
+        for (int i = 0; i < this.updateHandler.length; i++) {
+            try {this.updateQueue.put(POISON_DOCUMENT);} catch (final InterruptedException e) {}
+        }
         try {this.deletionHandler.join();} catch (final InterruptedException e) {}
         for (Thread t: this.updateHandler) try {t.join();} catch (final InterruptedException e) {}
         this.connector.close();
@@ -361,6 +361,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
         removeIdFromUpdateQueue(id);
         this.metadataCache.remove(id);
         this.missCache.add(id);
+        ensureAliveDeletionHandler();
         if (this.deletionHandler.isAlive()) {
             try {this.deleteQueue.put(id);} catch (final InterruptedException e) {}
         } else {
@@ -375,6 +376,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
             this.metadataCache.remove(id);
             this.missCache.add(id);
         }
+        ensureAliveDeletionHandler();
         if (this.deletionHandler.isAlive()) {
             for (String id: ids) try {this.deleteQueue.put(id);} catch (final InterruptedException e) {}
         } else {