diff --git a/.classpath b/.classpath index b4da0601b..283cb0699 100644 --- a/.classpath +++ b/.classpath @@ -35,8 +35,6 @@ - - @@ -64,26 +62,32 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + @@ -99,9 +103,9 @@ - - - - + + + + diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 2bae8e15b..a073ffd95 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -6,9 +6,10 @@ org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annota org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -96,4 +97,4 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.source=1.7 +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/build.properties b/build.properties index 0ee1b439d..6c6410bd6 100644 --- a/build.properties +++ b/build.properties @@ -1,9 +1,9 @@ # defining some compiler arguments -javacSource=1.7 -javacTarget=1.7 +javacSource=1.8 +javacTarget=1.8 # Release Configuration -releaseVersion=1.92 +releaseVersion=1.921 stdReleaseFile=yacy${branch}_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz sourceReleaseFile=yacy_src_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseFileParentDir=yacy diff --git a/build.xml b/build.xml index cf5a884a0..714413ac0 100644 --- a/build.xml +++ b/build.xml @@ -170,6 +170,7 @@ + @@ -183,6 +184,7 @@ + @@ -216,48 +218,50 @@ + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - + + + + - + diff --git a/defaults/solr/schema.xml b/defaults/solr/schema.xml index b8b803b07..546ac5c94 100644 --- a/defaults/solr/schema.xml +++ b/defaults/solr/schema.xml @@ -111,11 +111,7 @@ id - - text_t - - - + diff --git a/defaults/solr/solrconfig.xml b/defaults/solr/solrconfig.xml index ad04908e3..eba3c1074 100644 --- a/defaults/solr/solrconfig.xml +++ b/defaults/solr/solrconfig.xml @@ -35,7 +35,7 @@ that you fully re-index after changing this setting as it can affect both how text is indexed and queried. --> - 5.5.0 + 6.6.0 - + + - - - - - ${solr.hdfs.home:} - - ${solr.hdfs.confdir:} - - ${solr.hdfs.blockcache.enabled:true} - - ${solr.hdfs.blockcache.global:true} - - + class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/> - - - - - - 20000 - - - + - true + - 32 - 100 + + - - - + + --> ${solr.lock.type:native} - - - - - - - - - true - - - false @@ -399,7 +301,7 @@ --> ${solr.autoCommit.maxTime:15000} - true + false - - - + - + - + - + + - true - - - 2 + false @@ -829,6 +713,13 @@ not be initialized until the first request that uses it. --> + + + + solr-data-config.xml + + + - + explicit 10 - text_t + text - + explicit json true - text_t + text + - - - true - json - true - + Note that multiple instances of the same Request Handler + (SearchHandler) can be registered multiple times with different + names (and different init parameters) + --> + + + explicit + + + velocity + browse + layout + + + edismax + *:* + 10 + *,score + + + on + 1 + - - - - - {!xport} - xsort - false + + + text + - - query - - + + + + true + ignored_ + + + true + links + ignored_ + + + - - - - - - - - - - - true - ignored_ - - - true - links - ignored_ - - + To insert components before or after the 'standard' components, use: + + myFirstComponentName + - - - - - - + + + text_general + + + + + + default + text + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + - - + + + wordbreak + solr.WordBreakSolrSpellChecker + name + true + true + 10 + + + - + + - - - - - solrpingquery - + + freq + lowerfilt + solr.DirectSolrSpellChecker + freq + --> + + + + + + + + + text + + default + wordbreak + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + mySuggester + FuzzyLookupFactory + DocumentDictionaryFactory + cat + price + string + + + + + + true + 10 + + + suggest + + + + + + + + + text + true + + + tvComponent + + + + + + + + + + true + false + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + text + + + elevator + + + + + + + + + - all + 100 - - - - - - + + + + - explicit - true + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} - - - - - - - - - - - - - - - - + + + + - text_t - true - - - tvComponent - - - - - - - lingo - - - org.carrot2.clustering.lingo.LingoClusteringAlgorithm - - - clustering/carrot2 + ]]> + ]]> - - - stc - org.carrot2.clustering.stc.STCClusteringAlgorithm + + + + + + + + + + + + + + + + + - - kmeans - org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm - - - - - + --> + + + + - true - true - - name - - id - - features - - true - - - - false - - - edismax - - text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 - - *:* - 10 - *,score + ,, + ,, + ,, + ,, + ,]]> + ]]> - - clustering - - - - - - - - + + + - true - false + 10 + .,!? - - terms - - - - - - - - string - elevate.xml - - - - + + + - explicit - text_t + + WORD + + + en + US - - elevator - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - + + - - + + ${velocity.template.base.dir:} + - - - - *:* - + diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 25fdec2c3..a85dd12b6 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -22,15 +22,13 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.text.ParseException; +import java.time.Instant; import java.util.AbstractMap; import java.util.Date; import java.util.Iterator; import java.util.LinkedList; import java.util.Map; -import org.apache.solr.util.DateFormatUtil; - import net.yacy.cora.date.AbstractFormatter; import net.yacy.cora.document.analysis.Classification.ContentDomain; import net.yacy.cora.document.id.MultiProtocolURL; @@ -260,15 +258,14 @@ public class yacysearchtrailer { if (name.length() < 10) continue; count = theSearch.dateNavigator.get(name); String shortname = name.substring(0, 10); - long d; - Date dd; - try {dd = DateFormatUtil.parseDate(name); d = dd.getTime();} catch (ParseException e) {continue;} + long d = Instant.parse(name).toEpochMilli(); + Date dd = new Date(d); if (fromconstraint != null && dd.before(fromconstraint)) continue; if (toconstraint != null && dd.after(toconstraint)) break; if (dx > 0) { while (d - dx > AbstractFormatter.dayMillis) { dx += AbstractFormatter.dayMillis; - String sn = DateFormatUtil.formatExternal(new Date(dx)).substring(0, 10); + String sn = new Date(dx).toInstant().toString().substring(0, 10); prop.put("nav-dates_element_" + i + "_on", 0); prop.put(fileType, "nav-dates_element_" + i + "_name", sn); prop.put("nav-dates_element_" + i + "_count", 0); diff --git a/lib/commons-math3-3.4.1.jar b/lib/commons-math3-3.4.1.jar new file mode 100644 index 000000000..cf9f519fb Binary files /dev/null and b/lib/commons-math3-3.4.1.jar differ diff --git a/lib/lucene-analyzers-common-5.5.4.jar b/lib/lucene-analyzers-common-5.5.4.jar deleted file mode 100644 index 6c490d421..000000000 Binary files a/lib/lucene-analyzers-common-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-analyzers-common-6.6.0.jar b/lib/lucene-analyzers-common-6.6.0.jar new file mode 100644 index 000000000..3695b0780 Binary files /dev/null and b/lib/lucene-analyzers-common-6.6.0.jar differ diff --git a/lib/lucene-analyzers-phonetic-5.5.4.jar b/lib/lucene-analyzers-phonetic-5.5.4.jar deleted file mode 100644 index 9e29e004d..000000000 Binary files a/lib/lucene-analyzers-phonetic-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-analyzers-phonetic-6.6.0.jar b/lib/lucene-analyzers-phonetic-6.6.0.jar new file mode 100644 index 000000000..3c73b3a36 Binary files /dev/null and b/lib/lucene-analyzers-phonetic-6.6.0.jar differ diff --git a/lib/lucene-backward-codecs-5.5.4.jar b/lib/lucene-backward-codecs-5.5.4.jar deleted file mode 100644 index 76e02b103..000000000 Binary files a/lib/lucene-backward-codecs-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-backward-codecs-6.6.0.jar b/lib/lucene-backward-codecs-6.6.0.jar new file mode 100644 index 000000000..c2bbb6315 Binary files /dev/null and b/lib/lucene-backward-codecs-6.6.0.jar differ diff --git a/lib/lucene-classification-5.5.4.jar b/lib/lucene-classification-5.5.4.jar deleted file mode 100644 index 0a9cfb498..000000000 Binary files a/lib/lucene-classification-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-classification-6.6.0.jar b/lib/lucene-classification-6.6.0.jar new file mode 100644 index 000000000..3b21ef553 Binary files /dev/null and b/lib/lucene-classification-6.6.0.jar differ diff --git a/lib/lucene-codecs-5.5.4.jar b/lib/lucene-codecs-5.5.4.jar deleted file mode 100644 index 6653b8244..000000000 Binary files a/lib/lucene-codecs-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-codecs-6.6.0.jar b/lib/lucene-codecs-6.6.0.jar new file mode 100644 index 000000000..643d01217 Binary files /dev/null and b/lib/lucene-codecs-6.6.0.jar differ diff --git a/lib/lucene-core-5.5.4.jar b/lib/lucene-core-5.5.4.jar deleted file mode 100644 index 311992ca0..000000000 Binary files a/lib/lucene-core-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-core-6.6.0.jar b/lib/lucene-core-6.6.0.jar new file mode 100644 index 000000000..d9d533c9f Binary files /dev/null and b/lib/lucene-core-6.6.0.jar differ diff --git a/lib/lucene-facet-5.5.4.jar b/lib/lucene-facet-5.5.4.jar deleted file mode 100644 index 967b2a609..000000000 Binary files a/lib/lucene-facet-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-facet-6.6.0.jar b/lib/lucene-facet-6.6.0.jar new file mode 100644 index 000000000..85c644772 Binary files /dev/null and b/lib/lucene-facet-6.6.0.jar differ diff --git a/lib/lucene-grouping-5.5.4.jar b/lib/lucene-grouping-5.5.4.jar deleted file mode 100644 index c7749aadf..000000000 Binary files a/lib/lucene-grouping-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-grouping-6.6.0.jar b/lib/lucene-grouping-6.6.0.jar new file mode 100644 index 000000000..9b7f074cb Binary files /dev/null and b/lib/lucene-grouping-6.6.0.jar differ diff --git a/lib/lucene-highlighter-5.5.4.jar b/lib/lucene-highlighter-5.5.4.jar deleted file mode 100644 index b31415e2a..000000000 Binary files a/lib/lucene-highlighter-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-highlighter-6.6.0.jar b/lib/lucene-highlighter-6.6.0.jar new file mode 100644 index 000000000..dd108980c Binary files /dev/null and b/lib/lucene-highlighter-6.6.0.jar differ diff --git a/lib/lucene-join-5.5.4.jar b/lib/lucene-join-5.5.4.jar deleted file mode 100644 index 103b682b2..000000000 Binary files a/lib/lucene-join-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-join-6.6.0.jar b/lib/lucene-join-6.6.0.jar new file mode 100644 index 000000000..6ab797b50 Binary files /dev/null and b/lib/lucene-join-6.6.0.jar differ diff --git a/lib/lucene-memory-5.5.4.jar b/lib/lucene-memory-5.5.4.jar deleted file mode 100644 index e3ff559b8..000000000 Binary files a/lib/lucene-memory-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-memory-6.6.0.jar b/lib/lucene-memory-6.6.0.jar new file mode 100644 index 000000000..afde47d02 Binary files /dev/null and b/lib/lucene-memory-6.6.0.jar differ diff --git a/lib/lucene-misc-5.5.4.jar b/lib/lucene-misc-5.5.4.jar deleted file mode 100644 index 390b80574..000000000 Binary files a/lib/lucene-misc-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-misc-6.6.0.jar b/lib/lucene-misc-6.6.0.jar new file mode 100644 index 000000000..c1b5012fa Binary files /dev/null and b/lib/lucene-misc-6.6.0.jar differ diff --git a/lib/lucene-queries-5.5.4.jar b/lib/lucene-queries-5.5.4.jar deleted file mode 100644 index c8fecce21..000000000 Binary files a/lib/lucene-queries-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-queries-6.6.0.jar b/lib/lucene-queries-6.6.0.jar new file mode 100644 index 000000000..85fb0640f Binary files /dev/null and b/lib/lucene-queries-6.6.0.jar differ diff --git a/lib/lucene-queryparser-5.5.4.jar b/lib/lucene-queryparser-5.5.4.jar deleted file mode 100644 index 22e280345..000000000 Binary files a/lib/lucene-queryparser-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-queryparser-6.6.0.jar b/lib/lucene-queryparser-6.6.0.jar new file mode 100644 index 000000000..394df7eaa Binary files /dev/null and b/lib/lucene-queryparser-6.6.0.jar differ diff --git a/lib/lucene-spatial-5.5.4.jar b/lib/lucene-spatial-5.5.4.jar deleted file mode 100644 index e8ea7c930..000000000 Binary files a/lib/lucene-spatial-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-spatial-6.6.0.jar b/lib/lucene-spatial-6.6.0.jar new file mode 100644 index 000000000..2b31f7212 Binary files /dev/null and b/lib/lucene-spatial-6.6.0.jar differ diff --git a/lib/lucene-suggest-5.5.4.jar b/lib/lucene-suggest-5.5.4.jar deleted file mode 100644 index 1ad6b5a48..000000000 Binary files a/lib/lucene-suggest-5.5.4.jar and /dev/null differ diff --git a/lib/lucene-suggest-6.6.0.jar b/lib/lucene-suggest-6.6.0.jar new file mode 100644 index 000000000..05f4dad3b Binary files /dev/null and b/lib/lucene-suggest-6.6.0.jar differ diff --git a/lib/metrics-core-3.2.2.jar b/lib/metrics-core-3.2.2.jar new file mode 100644 index 000000000..a62be9671 Binary files /dev/null and b/lib/metrics-core-3.2.2.jar differ diff --git a/lib/solr-core-5.5.4.jar b/lib/solr-core-5.5.4.jar deleted file mode 100644 index 4a560a3ae..000000000 Binary files a/lib/solr-core-5.5.4.jar and /dev/null differ diff --git a/lib/solr-core-6.6.0.jar b/lib/solr-core-6.6.0.jar new file mode 100644 index 000000000..eebf3dcd7 Binary files /dev/null and b/lib/solr-core-6.6.0.jar differ diff --git a/lib/solr-dataimporthandler-6.6.0.jar b/lib/solr-dataimporthandler-6.6.0.jar new file mode 100644 index 000000000..c211ef82c Binary files /dev/null and b/lib/solr-dataimporthandler-6.6.0.jar differ diff --git a/lib/solr-solrj-5.5.4.jar b/lib/solr-solrj-5.5.4.jar deleted file mode 100644 index 17e495f90..000000000 Binary files a/lib/solr-solrj-5.5.4.jar and /dev/null differ diff --git a/lib/solr-solrj-6.6.0.jar b/lib/solr-solrj-6.6.0.jar new file mode 100644 index 000000000..32ac48148 Binary files /dev/null and b/lib/solr-solrj-6.6.0.jar differ diff --git a/lib/spatial4j-0.5.jar b/lib/spatial4j-0.5.jar deleted file mode 100644 index 02fa52ea9..000000000 Binary files a/lib/spatial4j-0.5.jar and /dev/null differ diff --git a/lib/spatial4j-0.6.jar b/lib/spatial4j-0.6.jar new file mode 100644 index 000000000..1afa5074b Binary files /dev/null and b/lib/spatial4j-0.6.jar differ diff --git a/lib/zookeeper-3.4.10.jar b/lib/zookeeper-3.4.10.jar new file mode 100644 index 000000000..0d8b0d312 Binary files /dev/null and b/lib/zookeeper-3.4.10.jar differ diff --git a/lib/zookeeper-3.4.6.jar b/lib/zookeeper-3.4.6.jar deleted file mode 100644 index 7c340be9f..000000000 Binary files a/lib/zookeeper-3.4.6.jar and /dev/null differ diff --git a/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java deleted file mode 100644 index c5695f794..000000000 --- a/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java +++ /dev/null @@ -1,400 +0,0 @@ -/** - * ConcurrentUpdateSolrConnector - * Copyright 2013 by Michael Peter Christen - * First released 28.04.2013 at http://yacy.net - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package net.yacy.cora.federate.solr.connector; - -import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.concurrent.BlockingQueue; - -import net.yacy.cora.sorting.ReversibleScoreMap; -import net.yacy.cora.storage.ARC; -import net.yacy.cora.storage.ConcurrentARC; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.word.Word; -import net.yacy.kelondro.util.MemoryControl; -import net.yacy.search.schema.CollectionSchema; - -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.ModifiableSolrParams; - -/** - * The purpose of this connector is to provide a non-blocking interface to solr. - * All time-consuming tasks like updates and deletions are done within a concurrent process - * which is started for this class in the background. - * To implement this, we introduce an id exist cache, a deletion id queue and a update document queue. - */ -public class ConcurrentUpdateSolrConnector implements SolrConnector { - - private final static long AUTOCOMMIT = 3000; // milliseconds - - private class CommitHandler extends Thread { - @Override - public void run() { - try { - while (ConcurrentUpdateSolrConnector.this.commitProcessRunning) { - commitDocBuffer(); - try {Thread.sleep(AUTOCOMMIT);} catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - } - } finally { - commitDocBuffer(); - } - } - } - - private SolrConnector connector; - private ARC metadataCache; - //private final ARH missCache; - private final LinkedHashMap docBuffer; - private CommitHandler processHandler; - private final int updateCapacity; - private boolean commitProcessRunning; - - public ConcurrentUpdateSolrConnector(final SolrConnector connector, final int updateCapacity, final int idCacheCapacity, final int concurrency) { - this.connector = connector; - this.updateCapacity = updateCapacity; - this.metadataCache = new ConcurrentARC<>(idCacheCapacity, concurrency); - //this.missCache = new ConcurrentARH<>(idCacheCapacity, concurrency); - this.docBuffer = new LinkedHashMap<>(); - this.processHandler = null; - this.commitProcessRunning = true; - ensureAliveProcessHandler(); - } - - @Override - public int hashCode() { - return this.connector.hashCode(); - } - - @Override - public boolean equals(Object o) { - return o instanceof ConcurrentUpdateSolrConnector && this.connector.equals(((ConcurrentUpdateSolrConnector) o).connector); - } - - private void commitDocBuffer() { - synchronized (this.docBuffer) { - //System.out.println("*** commit of " + this.docBuffer.size() + " documents"); - //Thread.dumpStack(); - if (this.docBuffer.size() > 0) try { - this.connector.add(this.docBuffer.values()); - } catch (final OutOfMemoryError e) { - // clear and try again... - clearCaches(); - try { - this.connector.add(this.docBuffer.values()); - } catch (final IOException ee) { - ConcurrentLog.logException(e); - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - this.docBuffer.clear(); - } - } - - @Override - public int bufferSize() { - return this.docBuffer.size(); - } - - @Override - public void clearCaches() { - this.connector.clearCaches(); - this.metadataCache.clear(); - //this.missCache.clear(); - } - - private void updateCache(final String id, final LoadTimeURL md) { - if (id == null) return; - if (MemoryControl.shortStatus()) { - this.metadataCache.clear(); - //this.missCache.clear(); - } - this.metadataCache.put(id, md); - //this.missCache.delete(id); - } - - public void ensureAliveProcessHandler() { - if (this.processHandler == null || !this.processHandler.isAlive()) { - this.processHandler = new CommitHandler(); - this.processHandler.setName(this.getClass().getName() + "_ProcessHandler"); - this.processHandler.start(); - } - } - - @Override - public Iterator iterator() { - return this.connector.iterator(); - } - - @Override - public long getSize() { - return Math.max(this.metadataCache.size(), this.connector.getSize()); - } - - @Override - public void commit(boolean softCommit) { - ensureAliveProcessHandler(); - commitDocBuffer(); - this.connector.commit(softCommit); - } - - @Override - public void optimize(int maxSegments) { - commitDocBuffer(); - this.connector.optimize(maxSegments); - } - - @Override - public int getSegmentCount() { - return this.connector.getSegmentCount(); - } - - @Override - public boolean isClosed() { - return this.connector == null || this.connector.isClosed(); - } - - @Override - public void close() { - ensureAliveProcessHandler(); - this.commitProcessRunning = false; - try {this.processHandler.join();} catch (final InterruptedException e) {} - this.connector.close(); - this.metadataCache.clear(); - this.connector = null; - this.metadataCache = null; - } - - @Override - public void clear() throws IOException { - this.docBuffer.clear(); - this.connector.clear(); - this.metadataCache.clear(); - //this.missCache.clear(); - } - - @Override - public synchronized void deleteById(String id) throws IOException { - this.metadataCache.remove(id); - //this.missCache.add(id); - synchronized (this.docBuffer) { - this.docBuffer.remove(id); - } - this.connector.deleteById(id); - } - - @Override - public synchronized void deleteByIds(Collection ids) throws IOException { - for (String id: ids) { - this.metadataCache.remove(id); - //this.missCache.add(id); - } - synchronized (this.docBuffer) { - for (String id: ids) { - this.docBuffer.remove(id); - } - } - this.connector.deleteByIds(ids); - } - - @Override - public void deleteByQuery(final String querystring) throws IOException { - commitDocBuffer(); - try { - this.connector.deleteByQuery(querystring); - this.metadataCache.clear(); - } catch (final IOException e) { - ConcurrentLog.severe("ConcurrentUpdateSolrConnector", e.getMessage(), e); - } - } - - @Override - public LoadTimeURL getLoadTimeURL(String id) throws IOException { - //if (this.missCache.contains(id)) return null; - LoadTimeURL md = this.metadataCache.get(id); - if (md != null) { - //System.out.println("*** metadata cache hit; metadataCache.size() = " + metadataCache.size()); - //Thread.dumpStack(); - return md; - } - SolrInputDocument doc = this.docBuffer.get(id); - if (doc != null) { - //System.out.println("*** docBuffer cache hit; docBuffer.size() = " + docBuffer.size()); - //Thread.dumpStack(); - return AbstractSolrConnector.getLoadTimeURL(doc); - } - md = this.connector.getLoadTimeURL(id); - if (md == null) {/*this.missCache.add(id);*/ return null;} - updateCache(id, md); - return md; - } - - @Override - public void add(SolrInputDocument solrdoc) throws IOException, SolrException { - String id = (String) solrdoc.getFieldValue(CollectionSchema.id.getSolrFieldName()); - updateCache(id, AbstractSolrConnector.getLoadTimeURL(solrdoc)); - ensureAliveProcessHandler(); - if (this.processHandler.isAlive()) { - synchronized (this.docBuffer) {this.docBuffer.put(id, solrdoc);} - } else { - this.connector.add(solrdoc); - } - if (MemoryControl.shortStatus() || this.docBuffer.size() > this.updateCapacity) { - commitDocBuffer(); - } - } - - @Override - public void add(Collection solrdocs) throws IOException, SolrException { - ensureAliveProcessHandler(); - synchronized (this.docBuffer) { - for (SolrInputDocument solrdoc: solrdocs) { - String id = (String) solrdoc.getFieldValue(CollectionSchema.id.getSolrFieldName()); - updateCache(id, AbstractSolrConnector.getLoadTimeURL(solrdoc)); - if (this.processHandler.isAlive()) { - this.docBuffer.put(id, solrdoc); - } else { - this.connector.add(solrdoc); - } - } - } - if (MemoryControl.shortStatus() || this.docBuffer.size() > this.updateCapacity) { - commitDocBuffer(); - } - } - - @Override - public SolrDocument getDocumentById(final String id, String... fields) throws IOException { - assert id.length() == Word.commonHashLength : "wrong id: " + id; - //if (this.missCache.contains(id)) return null; - SolrInputDocument idoc = this.docBuffer.get(id); - if (idoc != null) { - //System.out.println("*** docBuffer cache hit; docBuffer.size() = " + docBuffer.size()); - //Thread.dumpStack(); - return ClientUtils.toSolrDocument(idoc); - } - SolrDocument solrdoc = this.connector.getDocumentById(id, AbstractSolrConnector.ensureEssentialFieldsIncluded(fields)); - if (solrdoc == null) { - //this.missCache.add(id); - this.metadataCache.remove(id); - } else { - updateCache(id, AbstractSolrConnector.getLoadTimeURL(solrdoc)); - } - return solrdoc; - } - - @Override - public QueryResponse getResponseByParams(ModifiableSolrParams query) throws IOException, SolrException { - commitDocBuffer(); - return this.connector.getResponseByParams(query); - } - - @Override - public SolrDocumentList getDocumentListByParams(ModifiableSolrParams params) throws IOException, SolrException { - commitDocBuffer(); - SolrDocumentList sdl = this.connector.getDocumentListByParams(params); - for (SolrDocument doc: sdl) { - String id = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()); - updateCache(id, AbstractSolrConnector.getLoadTimeURL(doc)); - } - return sdl; - } - - @Override - public SolrDocumentList getDocumentListByQuery(String querystring, String sort, int offset, int count, String... fields) throws IOException, SolrException { - commitDocBuffer(); - if (offset == 0 && count == 1 && querystring.startsWith("id:") && - ((querystring.length() == 17 && querystring.charAt(3) == '"' && querystring.charAt(16) == '"') || - querystring.length() == 15)) { - final SolrDocumentList list = new SolrDocumentList(); - SolrDocument doc = getDocumentById(querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields); - list.add(doc); - return list; - } - - SolrDocumentList sdl = this.connector.getDocumentListByQuery(querystring, sort, offset, count, AbstractSolrConnector.ensureEssentialFieldsIncluded(fields)); - return sdl; - } - - @Override - public long getCountByQuery(String querystring) throws IOException { - commitDocBuffer(); - return this.connector.getCountByQuery(querystring); - } - - @Override - public LinkedHashMap> getFacets(String query, int maxresults, String... fields) throws IOException { - commitDocBuffer(); - return this.connector.getFacets(query, maxresults, fields); - } - - @Override - public BlockingQueue concurrentDocumentsByQuery(String querystring, String sort, int offset, int maxcount, long maxtime, int buffersize, final int concurrency, final boolean prefetchIDs, String... fields) { - commitDocBuffer(); - return this.connector.concurrentDocumentsByQuery(querystring, sort, offset, maxcount, maxtime, buffersize, concurrency, prefetchIDs, fields); - } - - @Override - public BlockingQueue concurrentDocumentsByQueries( - List querystrings, String sort, int offset, int maxcount, - long maxtime, int buffersize, int concurrency, boolean prefetchIDs, - String... fields) { - commitDocBuffer(); - return this.connector.concurrentDocumentsByQueries(querystrings, sort, offset, maxcount, maxtime, buffersize, concurrency, prefetchIDs, fields); - } - - @Override - public BlockingQueue concurrentIDsByQuery(String querystring, String sort, int offset, int maxcount, long maxtime, int buffersize, final int concurrency) { - commitDocBuffer(); - return this.connector.concurrentIDsByQuery(querystring, sort, offset, maxcount, maxtime, buffersize, concurrency); - } - - @Override - public BlockingQueue concurrentIDsByQueries( - List querystrings, String sort, int offset, int maxcount, - long maxtime, int buffersize, int concurrency) { - commitDocBuffer(); - return this.connector.concurrentIDsByQueries(querystrings, sort, offset, maxcount, maxtime, buffersize, concurrency); - } - - @Override - public void update(final SolrInputDocument solrdoc) throws IOException, SolrException { - commitDocBuffer(); - this.connector.update(solrdoc); - } - - @Override - public void update(final Collection solrdoc) throws IOException, SolrException { - commitDocBuffer(); - this.connector.update(solrdoc); - } - -} diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java index 403302f4b..3f2bc2d77 100644 --- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java @@ -241,7 +241,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo SolrDocumentList sdl = new SolrDocumentList(); NamedList nl = rsp.getValues(); ResultContext resultContext = (ResultContext) nl.get("response"); - DocList response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.docs; + DocList response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.getDocList(); sdl.setNumFound(response == null ? 0 : response.matches()); sdl.setStart(response == null ? 0 : response.offset()); String originalName = Thread.currentThread().getName(); @@ -391,7 +391,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo NamedList nl = rsp.getValues(); ResultContext resultContext = (ResultContext) nl.get("response"); if (resultContext == null) log.warn("DocListSearcher: no response for query '" + querystring + "'"); - this.response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.docs; + this.response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.getDocList(); } @Override diff --git a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java index 13f292920..1d14e99f7 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java @@ -30,7 +30,6 @@ import net.yacy.cora.federate.solr.instance.ServerShard; import net.yacy.cora.util.ConcurrentLog; import net.yacy.search.schema.CollectionSchema; -import org.apache.lucene.analysis.NumericTokenStream; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; @@ -51,12 +50,7 @@ import org.apache.solr.common.params.CommonParams; public abstract class SolrServerConnector extends AbstractSolrConnector implements SolrConnector { protected final static ConcurrentLog log = new ConcurrentLog(SolrServerConnector.class.getName()); - public final static NumericTokenStream classLoaderSynchro = new NumericTokenStream(); - // pre-instantiate this object to prevent sun.misc.Launcher$AppClassLoader deadlocks - // this is a very nasty problem; solr instantiates objects dynamically which can cause deadlocks - static { - assert classLoaderSynchro != null; - } + protected SolrClient server; protected SolrServerConnector() { diff --git a/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java b/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java index c8b211ad9..a5f3e0bd9 100644 --- a/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java @@ -177,7 +177,7 @@ public class EmbeddedInstance implements SolrInstance { source = new File(solr_config, "solrcore.properties"); } } - // solr alwasy reads the solrcore.properties file if exists in core/conf directory + // solr always reads the solrcore.properties file if exists in core/conf directory target = new File(conf, "solrcore.properties"); if (source.exists()) { @@ -206,7 +206,7 @@ public class EmbeddedInstance implements SolrInstance { @Override public Collection getCoreNames() { - return this.coreContainer.getCoreNames(); + return this.coreContainer.getAllCoreNames(); } @Override diff --git a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java index 317382e96..bb48a4d5a 100644 --- a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java +++ b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java @@ -27,12 +27,10 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.federate.solr.connector.ConcurrentUpdateSolrConnector; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; import net.yacy.cora.federate.solr.connector.MirrorSolrConnector; import net.yacy.cora.federate.solr.connector.RemoteSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; -import net.yacy.kelondro.util.MemoryControl; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -181,9 +179,7 @@ public class InstanceMirror { if (msc != null) return msc; EmbeddedSolrConnector esc = getEmbeddedConnector(corename); RemoteSolrConnector rsc = getRemoteConnector(corename); - int cacheSize = (int) (MemoryControl.available() / 30000); // will return about 10000 for standard ram size - msc = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), RemoteInstance.queueSizeByMemory(), cacheSize, Runtime.getRuntime().availableProcessors()); - //msc = new MirrorSolrConnector(esc, rsc); + msc = new MirrorSolrConnector(esc, rsc); this.mirrorConnectorCache.put(corename, msc); return msc; } diff --git a/source/net/yacy/cora/federate/solr/instance/ServerMirror.java b/source/net/yacy/cora/federate/solr/instance/ServerMirror.java index ca465030c..523cd5a58 100644 --- a/source/net/yacy/cora/federate/solr/instance/ServerMirror.java +++ b/source/net/yacy/cora/federate/solr/instance/ServerMirror.java @@ -429,9 +429,9 @@ public class ServerMirror extends SolrClient { if (this.solr1 != null) return this.solr1.getBinder(); return null; } - + @Override - public void shutdown() { + public void close() throws IOException { if (this.solr0 != null) try { this.solr0.close(); diff --git a/source/net/yacy/cora/federate/solr/instance/ServerShard.java b/source/net/yacy/cora/federate/solr/instance/ServerShard.java index e734f7d55..373f483f0 100644 --- a/source/net/yacy/cora/federate/solr/instance/ServerShard.java +++ b/source/net/yacy/cora/federate/solr/instance/ServerShard.java @@ -485,9 +485,9 @@ public class ServerShard extends SolrClient { } return null; } - + @Override - public void shutdown() { + public void close() throws IOException { for (SolrClient s: this.shards) { try { s.close(); diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java index dd5f586db..f444688c3 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java @@ -50,7 +50,6 @@ import org.apache.solr.schema.TextField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.util.DateFormatUtil; public class EnhancedXMLResponseWriter implements QueryResponseWriter { @@ -80,7 +79,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { assert values.get("response") != null; SimpleOrderedMap responseHeader = (SimpleOrderedMap) rsp.getResponseHeader(); - DocList response = ((ResultContext) values.get("response")).docs; + DocList response = ((ResultContext) values.get("response")).getDocList(); @SuppressWarnings("unchecked") SimpleOrderedMap highlighting = (SimpleOrderedMap) values.get("highlighting"); writeProps(writer, "responseHeader", responseHeader); // this.writeVal("responseHeader", responseHeader); @@ -268,7 +267,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { } else if (typeName.equals(SolrType.num_long.printName())) { writeTag(writer, "long", name, value, true); } else if (typeName.equals(SolrType.date.printName())) { - writeTag(writer, "date", name, DateFormatUtil.formatExternal(new Date(Long.parseLong(value))), true); + writeTag(writer, "date", name, new Date(Long.parseLong(value)).toInstant().toString(), true); } else if (typeName.equals(SolrType.num_float.printName())) { writeTag(writer, "float", name, value, true); } else if (typeName.equals(SolrType.num_double.printName())) { @@ -286,7 +285,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { } else if (value instanceof Long) { writeTag(writer, "long", name, ((Long) value).toString(), true); } else if (value instanceof Date) { - writeTag(writer, "date", name, DateFormatUtil.formatExternal((Date) value), true); + writeTag(writer, "date", name, ((Date) value).toInstant().toString(), true); } else if (value instanceof Float) { writeTag(writer, "float", name, ((Float) value).toString(), true); } else if (value instanceof Double) { diff --git a/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java index e30a566a8..9d05e00de 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java @@ -42,7 +42,6 @@ import org.apache.solr.schema.TextField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.util.DateFormatUtil; import org.json.simple.JSONArray; import net.yacy.cora.federate.solr.SolrType; @@ -70,7 +69,7 @@ public class FlatJSONResponseWriter implements QueryResponseWriter { @Override public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { NamedList values = rsp.getValues(); - DocList response = ((ResultContext) values.get("response")).docs; + DocList response = ((ResultContext) values.get("response")).getDocList(); writeDocs(writer, request, response); } @@ -144,7 +143,7 @@ public class FlatJSONResponseWriter implements QueryResponseWriter { } else if (typeName.equals(SolrType.num_long.printName())) { json.put(name, Long.parseLong(value)); } else if (typeName.equals(SolrType.date.printName())) { - json.put(name, DateFormatUtil.formatExternal(new Date(Long.parseLong(value)))); + json.put(name, new Date(Long.parseLong(value)).toInstant().toString()); } else if (typeName.equals(SolrType.num_float.printName())) { json.put(name, Double.parseDouble(value)); } else if (typeName.equals(SolrType.num_double.printName())) { diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java index fe1da6266..d5c7cc6da 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java @@ -148,7 +148,7 @@ public class GSAResponseWriter implements QueryResponseWriter { long start = System.currentTimeMillis(); SimpleOrderedMap responseHeader = (SimpleOrderedMap) rsp.getResponseHeader(); - DocList response = ((ResultContext) rsp.getValues().get("response")).docs; + DocList response = ((ResultContext) rsp.getValues().get("response")).getDocList(); @SuppressWarnings("unchecked") SimpleOrderedMap highlighting = (SimpleOrderedMap) rsp.getValues().get("highlighting"); Map> snippets = OpensearchResponseWriter.highlighting(highlighting); diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java index d022d2c9e..cdea36dad 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java @@ -109,7 +109,7 @@ public class GrepHTMLResponseWriter implements QueryResponseWriter { paramsList.remove("wt"); String xmlquery = dqp.matcher("select?" + SolrParams.toSolrParams(paramsList).toString()).replaceAll("%22"); - DocList response = ((ResultContext) values.get("response")).docs; + DocList response = ((ResultContext) values.get("response")).getDocList(); final int sz = response.size(); if (sz > 0) { SolrIndexSearcher searcher = request.getSearcher(); diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java index 8b05ba56d..679ca5cf5 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java @@ -51,7 +51,6 @@ import org.apache.solr.schema.TextField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.util.DateFormatUtil; public class HTMLResponseWriter implements QueryResponseWriter { @@ -157,7 +156,7 @@ public class HTMLResponseWriter implements QueryResponseWriter { String xmlquery = dqp.matcher("../solr/select?" + SolrParams.toSolrParams(paramsList).toString() + "&core=" + coreName).replaceAll("%22"); - DocList response = ((ResultContext) values.get("response")).docs; + DocList response = ((ResultContext) values.get("response")).getDocList(); final int sz = response.size(); if (sz > 0) { SolrIndexSearcher searcher = request.getSearcher(); @@ -283,7 +282,7 @@ public class HTMLResponseWriter implements QueryResponseWriter { if (typeName.equals(SolrType.bool.printName())) { return "F".equals(value) ? "false" : "true"; } else if (typeName.equals(SolrType.date.printName())) { - return DateFormatUtil.formatExternal(new Date(Long.parseLong(value))); + return new Date(Long.parseLong(value)).toInstant().toString(); } return value; } diff --git a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java index 043244de6..1027f48f6 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java @@ -110,7 +110,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter { assert values.get("response") != null; SimpleOrderedMap responseHeader = (SimpleOrderedMap) rsp.getResponseHeader(); - DocList response = ((ResultContext) values.get("response")).docs; + DocList response = ((ResultContext) values.get("response")).getDocList(); @SuppressWarnings("unchecked") SimpleOrderedMap facetCounts = (SimpleOrderedMap) values.get("facet_counts"); @SuppressWarnings("unchecked") diff --git a/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java index f2b4c5c26..0ee342571 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java @@ -53,7 +53,7 @@ public class SnapshotImagesReponseWriter implements QueryResponseWriter { NamedList paramsList = request.getOriginalParams().toNamedList(); paramsList.remove("wt"); - DocList response = ((ResultContext) values.get("response")).docs; + DocList response = ((ResultContext) values.get("response")).getDocList(); final int sz = response.size(); if (sz > 0) { SolrIndexSearcher searcher = request.getSearcher(); diff --git a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java index 25beff4ca..b56637078 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java @@ -94,7 +94,7 @@ public class YJsonResponseWriter implements QueryResponseWriter { assert values.get("response") != null; SimpleOrderedMap responseHeader = (SimpleOrderedMap) rsp.getResponseHeader(); - DocList response = ((ResultContext) values.get("response")).docs; + DocList response = ((ResultContext) values.get("response")).getDocList(); @SuppressWarnings("unchecked") SimpleOrderedMap facetCounts = (SimpleOrderedMap) values.get("facet_counts"); @SuppressWarnings("unchecked") diff --git a/source/net/yacy/http/servlets/GSAsearchServlet.java b/source/net/yacy/http/servlets/GSAsearchServlet.java index 7f7571860..9995b5c08 100644 --- a/source/net/yacy/http/servlets/GSAsearchServlet.java +++ b/source/net/yacy/http/servlets/GSAsearchServlet.java @@ -257,7 +257,7 @@ public class GSAsearchServlet extends HttpServlet { Object rv = response.getValues().get("response"); int matches = 0; if (rv != null && rv instanceof ResultContext) { - matches = ((ResultContext) rv).docs.matches(); + matches = ((ResultContext) rv).getDocList().matches(); } else if (rv != null && rv instanceof SolrDocumentList) { matches = (int) ((SolrDocumentList) rv).getNumFound(); } diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index 8cceb88d7..84434367e 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -263,7 +263,7 @@ public class SolrSelectServlet extends HttpServlet { NamedList values = rsp.getValues(); - DocList r = ((ResultContext) values.get("response")).docs; + DocList r = ((ResultContext) values.get("response")).getDocList(); int numFound = r.matches(); AccessTracker.addToDump(querystring, numFound, new Date(), "sq"); diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 4ef4ee611..1957f7a59 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -83,13 +83,12 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.core.SolrInfoMBean; -import org.apache.solr.util.DateFormatUtil; import org.apache.lucene.util.Version; public final class Fulltext { - private static final String SOLR_PATH = "solr_5_5"; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml - private static final String SOLR_OLD_PATH[] = new String[]{"solr_36", "solr_40", "solr_44", "solr_45", "solr_46", "solr_47", "solr_4_9", "solr_4_10", "solr_5_2"}; + private static final String SOLR_PATH = "solr_6_6"; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml + private static final String SOLR_OLD_PATH[] = new String[]{"solr_36", "solr_40", "solr_44", "solr_45", "solr_46", "solr_47", "solr_4_9", "solr_4_10", "solr_5_2", "solr_5_5"}; // class objects private final File segmentPath; @@ -666,8 +665,8 @@ public final class Fulltext { long now = System.currentTimeMillis(); if (maxseconds > 0) { long from = now - maxseconds * 1000L; - String nowstr = DateFormatUtil.formatExternal(new Date(now)); - String fromstr = DateFormatUtil.formatExternal(new Date(from)); + String nowstr = new Date(now).toInstant().toString(); + String fromstr = new Date(from).toInstant().toString(); String dateq = CollectionSchema.load_date_dt.getSolrFieldName() + ":[" + fromstr + " TO " + nowstr + "]"; query = query == null || AbstractSolrConnector.CATCHALL_QUERY.equals(query) ? dateq : query + " AND " + dateq; } else { diff --git a/source/net/yacy/search/query/QueryModifier.java b/source/net/yacy/search/query/QueryModifier.java index 4e3dffdd6..54c9fc131 100644 --- a/source/net/yacy/search/query/QueryModifier.java +++ b/source/net/yacy/search/query/QueryModifier.java @@ -26,7 +26,6 @@ import java.util.Date; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MultiMapSolrParams; -import org.apache.solr.util.DateFormatUtil; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.CommonPattern; @@ -382,7 +381,7 @@ public class QueryModifier { Date onDate = DateDetection.parseLine(onDescription, timezoneOffset); StringBuilder filterQuery = new StringBuilder(20); if (onDate != null) { - String dstr = DateFormatUtil.formatExternal(onDate); + String dstr = onDate.toInstant().toString(); filterQuery.append(CollectionSchema.dates_in_content_dts.getSolrFieldName()).append(":[").append(dstr).append(" TO ").append(dstr).append(']'); } return filterQuery.toString(); @@ -393,8 +392,8 @@ public class QueryModifier { Date toDate = to == null || to.equals("*") ? null : DateDetection.parseLine(to, timezoneOffset); StringBuilder filterQuery = new StringBuilder(20); if (fromDate != null && toDate != null) { - String dstrFrom = DateFormatUtil.formatExternal(fromDate); - String dstrTo = DateFormatUtil.formatExternal(toDate); + String dstrFrom = fromDate.toInstant().toString(); + String dstrTo = toDate.toInstant().toString(); filterQuery.append(CollectionSchema.dates_in_content_dts.getSolrFieldName()).append(":[").append(dstrFrom).append(" TO ").append(dstrTo).append(']'); } return filterQuery.toString(); diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 378d284b8..98025a2e0 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -76,7 +76,6 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.common.params.DisMaxParams; import org.apache.solr.common.params.FacetParams; -import org.apache.solr.util.DateFormatUtil; public final class QueryParams { @@ -468,8 +467,8 @@ public final class QueryParams { for (String field: this.facetfields) params.addFacetField("{!ex=" + field + "}" + field); // params.addFacetField("{!ex=" + field + "}" + field); if (this.facetfields.contains(CollectionSchema.dates_in_content_dts.name())) { params.setParam(FacetParams.FACET_RANGE, CollectionSchema.dates_in_content_dts.name()); - String start = DateFormatUtil.formatExternal(new Date(System.currentTimeMillis() - 1000L * 60L * 60L * 24L * 3)); - String end = DateFormatUtil.formatExternal(new Date(System.currentTimeMillis() + 1000L * 60L * 60L * 24L * 3)); + String start = new Date(System.currentTimeMillis() - 1000L * 60L * 60L * 24L * 3).toInstant().toString(); + String end = new Date(System.currentTimeMillis() + 1000L * 60L * 60L * 24L * 3).toInstant().toString(); params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.start", start); params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.end", end); params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.gap", "+1DAY"); diff --git a/test/java/net/yacy/document/DateDetectionTest.java b/test/java/net/yacy/document/DateDetectionTest.java index ec36dc20f..d2c908105 100644 --- a/test/java/net/yacy/document/DateDetectionTest.java +++ b/test/java/net/yacy/document/DateDetectionTest.java @@ -5,7 +5,6 @@ import java.util.Date; import java.util.LinkedHashSet; import java.util.Set; import java.util.TimeZone; -import org.apache.solr.util.DateFormatUtil; import org.junit.Test; import static org.junit.Assert.*; @@ -35,8 +34,8 @@ public class DateDetectionTest { Date d = DateDetection.parseLine(text, 0); // this formatter is used to create Solr search queries, use it to compare equality - String cs = DateFormatUtil.formatExternal(cal.getTime()); - String ds = DateFormatUtil.formatExternal(d); + String cs = cal.getTime().toInstant().toString(); + String ds = d.toInstant().toString(); assertEquals(text, cs, ds); } @@ -55,8 +54,8 @@ public class DateDetectionTest { Date d = DateDetection.parseLine(text, 0); // this formatter is used to create Solr search queries, use it to compare equality - String cs = DateFormatUtil.formatExternal(cal.getTime()); - String ds = DateFormatUtil.formatExternal(d); + String cs = cal.getTime().toInstant().toString(); + String ds = d.toInstant().toString(); assertEquals(text, cs, ds); } @@ -79,8 +78,8 @@ public class DateDetectionTest { Date d = DateDetection.parseLine(text, 0); // this formatter is used to create Solr search queries, use it to compare equality - String cs = DateFormatUtil.formatExternal(cal.getTime()); - String ds = DateFormatUtil.formatExternal(d); + String cs = cal.getTime().toInstant().toString(); + String ds = d.toInstant().toString(); assertEquals(text, cs, ds); }