From 7889fc238982539779be454a87d945c486135288 Mon Sep 17 00:00:00 2001 From: reger Date: Sun, 13 Sep 2015 20:23:15 +0200 Subject: [PATCH] Hack to prevent Solr issue on partial update on a document containing multivalued date field (regardless if these fields part of update). Switch partial update option off in postprocessing if schema contains *_dts (multivalued date field). see http://mantis.tokeek.de/view.php?id=601 --- source/net/yacy/search/Switchboard.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index ad269ff2c..3bd21d71e 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2480,12 +2480,22 @@ public final class Switchboard extends serverSwitch { if (!minimum_load_fullfilled) log.info("postprocessing deactivated: too high load (" + Memory.load() + ") > " + getConfigFloat("postprocessing.maximum_load", 0) + ", to force change field postprocessing.maximum_load"); boolean postprocessing = process_key_exist && reference_index_exist && minimum_ram_fullfilled && minimum_load_fullfilled; if (!postprocessing) log.info("postprocessing deactivated: constraints violated"); - + + // Hack to prevent Solr problem on partial update if target document contains multivalued date field + // regardless if this field is part of the update it causes a org.apache.solr.common.SolrException: Invalid Date String Exception. + // 2015-09-12 Solr v5.2.1 & v5.3 + // this hack switches partial update off (if multivalued datefield _dts exists, like: dates_in_content_dts startDates_dts endDates_dts) + boolean partialUpdate = getConfigBool("postprocessing.partialUpdate", true); + for (String sf : index.fulltext().getDefaultConfiguration().keySet()) { + if (sf.endsWith("_dts")) { + partialUpdate = false; + } + } if (allCrawlsFinished) { if (postprocessing) { // run postprocessing on all profiles ReferenceReportCache rrCache = index.getReferenceReportCache(); - proccount += collection1Configuration.postprocessing(index, rrCache, null, getConfigBool("postprocessing.partialUpdate", true)); + proccount += collection1Configuration.postprocessing(index, rrCache, null, partialUpdate); this.index.fulltext().commit(true); // without a commit the success is not visible in the monitoring } this.crawler.cleanProfiles(this.crawler.getActiveProfiles()); @@ -2498,7 +2508,7 @@ public final class Switchboard extends serverSwitch { if (postprocessing) { // run postprocessing on these profiles ReferenceReportCache rrCache = index.getReferenceReportCache(); - for (String profileHash: deletionCandidates) proccount += collection1Configuration.postprocessing(index, rrCache, profileHash, getConfigBool("postprocessing.partialUpdate", true)); + for (String profileHash: deletionCandidates) proccount += collection1Configuration.postprocessing(index, rrCache, profileHash, partialUpdate); this.index.fulltext().commit(true); // without a commit the success is not visible in the monitoring } this.crawler.cleanProfiles(deletionCandidates);