boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added a forced commit to index deletion to enable synchronized index

Browse Source 
updates
pull/1/head
Michael Peter Christen 11 years ago
parent 1d069c5861
commit a632b0d2a4
2 changed files with 25 additions and 15 deletions
Show Stats Download Patch File Download Diff File

12
htroot/IndexDeletion_p.java
Unescape View File

@ -35,6 +35,7 @@ import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.WorkTables;
import net.yacy.search.Switchboard;
import net.yacy.search.query.QueryModifier;
@ -52,8 +53,7 @@ public class IndexDeletion_p {
SolrConnector defaultConnector = sb.index.fulltext().getDefaultConnector();
SolrConnector webgraphConnector = sb.index.fulltext().getWebgraphConnector();
if (post != null && post.size() > 0) defaultConnector.commit(false); // we must do a commit here because the user cannot see a proper count.
prop.put("doccount", defaultConnector.getSize());
if (post == null || post.size() == 0) defaultConnector.commit(false); // we must do a commit here because the user cannot see a proper count.
// Delete by URL Matching
String urldelete = post == null ? "" : post.get("urldelete", "");
@ -147,6 +147,7 @@ public class IndexDeletion_p {
prop.put("urldelete-active", count == 0 ? 2 : 1);
} else {
sb.remove(ids);
defaultConnector.commit(false);
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs matching with " + urldelete);
prop.put("urldelete-active", 2);
}
@ -162,6 +163,7 @@ public class IndexDeletion_p {
} else {
try {
defaultConnector.deleteByQuery(regexquery);
defaultConnector.commit(false);
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, regex match = " + urldelete);
} catch (final IOException e) {
}
@ -187,6 +189,7 @@ public class IndexDeletion_p {
} else {
try {
defaultConnector.deleteByQuery(collection1Query);
defaultConnector.commit(false);
if (webgraphConnector != null) webgraphConnector.deleteByQuery(webgraphQuery);
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs older than " + timedelete_number + " " + timedelete_unit);
} catch (final IOException e) {
@ -209,6 +212,7 @@ public class IndexDeletion_p {
} else {
try {
defaultConnector.deleteByQuery(query);
defaultConnector.commit(false);
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, collection " + collectiondelete);
} catch (final IOException e) {
}
@ -227,7 +231,10 @@ public class IndexDeletion_p {
prop.put("querydelete-active", count == 0 ? 2 : 1);
} else {
try {
ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size before deletion = " + defaultConnector.getSize());
defaultConnector.deleteByQuery(querydelete);
defaultConnector.commit(false);
ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size after commit = " + defaultConnector.getSize());
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, solr query, q = " + querydelete);
} catch (final IOException e) {
}
@ -235,6 +242,7 @@ public class IndexDeletion_p {
}
prop.put("querydelete-active_count", count);
}
prop.put("doccount", defaultConnector.getSize());
// return rewrite properties
return prop;

28
source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java
Unescape View File

@ -301,18 +301,16 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
@Override
public void commit(boolean softCommit) {
if (!softCommit) {
long timeout = System.currentTimeMillis() + 1000;
ensureAliveDeletionHandler();
while (this.deleteQueue.size() > 0) {
try {Thread.sleep(10);} catch (final InterruptedException e) {}
if (System.currentTimeMillis() > timeout) break;
}
ensureAliveUpdateHandler();
while (this.updateQueue.size() > 0) {
try {Thread.sleep(10);} catch (final InterruptedException e) {}
if (System.currentTimeMillis() > timeout) break;
}
long timeout = System.currentTimeMillis() + 1000;
ensureAliveDeletionHandler();
while (this.deleteQueue.size() > 0) {
try {Thread.sleep(10);} catch (final InterruptedException e) {}
if (System.currentTimeMillis() > timeout) break;
}
ensureAliveUpdateHandler();
while (this.updateQueue.size() > 0) {
try {Thread.sleep(10);} catch (final InterruptedException e) {}
if (System.currentTimeMillis() > timeout) break;
}
this.connector.commit(softCommit);
}
@ -337,7 +335,9 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
ensureAliveDeletionHandler();
try {this.deleteQueue.put(POISON_ID);} catch (final InterruptedException e) {}
ensureAliveUpdateHandler();
try {this.updateQueue.put(POISON_DOCUMENT);} catch (final InterruptedException e) {}
for (int i = 0; i < this.updateHandler.length; i++) {
try {this.updateQueue.put(POISON_DOCUMENT);} catch (final InterruptedException e) {}
}
try {this.deletionHandler.join();} catch (final InterruptedException e) {}
for (Thread t: this.updateHandler) try {t.join();} catch (final InterruptedException e) {}
this.connector.close();
@ -361,6 +361,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
removeIdFromUpdateQueue(id);
this.metadataCache.remove(id);
this.missCache.add(id);
ensureAliveDeletionHandler();
if (this.deletionHandler.isAlive()) {
try {this.deleteQueue.put(id);} catch (final InterruptedException e) {}
} else {
@ -375,6 +376,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
this.metadataCache.remove(id);
this.missCache.add(id);
}
ensureAliveDeletionHandler();
if (this.deletionHandler.isAlive()) {
for (String id: ids) try {this.deleteQueue.put(id);} catch (final InterruptedException e) {}
} else {

Write Preview
Loading…
Cancel
Save