boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixes https://github.com/yacy/yacy_search_server/issues/424

Browse Source
pull/436/head
Michael Peter Christen 4 years ago
parent 63ad8ce6b2
commit be0aebad84
3 changed files with 551 additions and 563 deletions
Show Stats Download Patch File Download Diff File

5
defaults/yacy.init
Unescape View File

@ -526,8 +526,9 @@ seedScpPassword=
seedScpPath=
# every peer periodically scans for other peers. you can set the time
# of the period here (minutes)
peerCycle=2
# of the period here (minutes). You might switch it on in kubernetes clusters
scan.enabled=false
scan.peerCycle=2
# debug flags
debug.search.local.dht.off=false

143
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -71,7 +71,7 @@ import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.schema.CollectionSchema;
public class EmbeddedSolrConnector extends SolrServerConnector implements SolrConnector {
public static final String SELECT = "/select";
public static final String CONTEXT = "/solr";
@ -85,20 +85,20 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
this.instance = instance;
this.core = this.instance.getDefaultCore();
this.requestHandler = new SearchHandler();
this.requestHandler.init(new NamedList<Object>());
this.requestHandler.init(new NamedList<>());
this.requestHandler.inform(this.core);
//this.suggestHandler = new SuggestComponent();
//this.suggestHandler.init(new NamedList<Object>());
//this.suggestHandler.inform(this.core);
super.init(this.instance.getDefaultServer());
}
public EmbeddedSolrConnector(EmbeddedInstance instance, String coreName) {
super();
this.instance = instance;
this.core = this.instance.getCore(coreName);
this.requestHandler = new SearchHandler();
this.requestHandler.init(new NamedList<Object>());
this.requestHandler.init(new NamedList<>());
this.requestHandler.inform(this.core);
//this.suggestHandler = new SuggestComponent();
//this.suggestHandler.init(new NamedList<Object>());
@ -110,12 +110,12 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
public int hashCode() {
return this.instance.hashCode();
}
@Override
public boolean equals(Object o) {
return o instanceof EmbeddedSolrConnector && this.instance.equals(((EmbeddedSolrConnector) o).instance);
}
@Override
public int bufferSize() {
return 0;
@ -123,25 +123,29 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
@Override
public void clearCaches() {
SolrConfig solrConfig = this.core.getSolrConfig();
final SolrConfig solrConfig = this.core.getSolrConfig();
@SuppressWarnings("unchecked")
final
SolrCache<String, ?> fieldValueCache = solrConfig.fieldValueCacheConfig == null ? null : solrConfig.fieldValueCacheConfig.newInstance();
if (fieldValueCache != null) fieldValueCache.clear();
@SuppressWarnings("unchecked")
final
SolrCache<Query, DocSet> filterCache= solrConfig.filterCacheConfig == null ? null : solrConfig.filterCacheConfig.newInstance();
if (filterCache != null) filterCache.clear();
@SuppressWarnings("unchecked")
final
SolrCache<QueryResultKey, DocList> queryResultCache = solrConfig.queryResultCacheConfig == null ? null : solrConfig.queryResultCacheConfig.newInstance();
if (queryResultCache != null) queryResultCache.clear();
@SuppressWarnings("unchecked")
final
SolrCache<Integer, Document> documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
if (documentCache != null) documentCache.clear();
}
public SolrInstance getInstance() {
return this.instance;
}
public SolrCore getCore() {
return this.core;
}
@ -154,7 +158,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
public boolean isClosed() {
return this.core == null || this.core.isClosed();
}
@Override
protected void finalize() throws Throwable {
this.close();
@ -171,10 +175,10 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
@Override
public long getSize() {
RefCounted<SolrIndexSearcher> refCountedIndexSearcher = this.core.getSearcher();
SolrIndexSearcher searcher = refCountedIndexSearcher.get();
DirectoryReader reader = searcher.getIndexReader();
long numDocs = reader.numDocs();
final RefCounted<SolrIndexSearcher> refCountedIndexSearcher = this.core.getSearcher();
final SolrIndexSearcher searcher = refCountedIndexSearcher.get();
final DirectoryReader reader = searcher.getIndexReader();
final long numDocs = reader.numDocs();
refCountedIndexSearcher.decref();
return numDocs;
}
@ -185,23 +189,23 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
* @return
*/
public SolrQueryRequest request(final SolrParams params) {
SolrQueryRequest req = new SolrQueryRequestBase(this.core, params){};
final SolrQueryRequest req = new SolrQueryRequestBase(this.core, params){};
req.getContext().put("path", SELECT);
req.getContext().put("webapp", CONTEXT);
return req;
}
public SolrQueryResponse query(SolrQueryRequest req) throws SolrException {
final long startTime = System.currentTimeMillis();
// during the solr query we set the thread name to the query string to get more debugging info in thread dumps
String threadname = Thread.currentThread().getName();
String ql = ""; try {ql = URLDecoder.decode(req.getParams().toString(), StandardCharsets.UTF_8.name());} catch (UnsupportedEncodingException e) {}
final String threadname = Thread.currentThread().getName();
String ql = ""; try {ql = URLDecoder.decode(req.getParams().toString(), StandardCharsets.UTF_8.name());} catch (final UnsupportedEncodingException e) {}
Thread.currentThread().setName("Embedded solr query: " + ql); // for debugging in Threaddump
ConcurrentLog.fine("EmbeddedSolrConnector.query", "QUERY: " + ql);
SolrQueryResponse rsp = new SolrQueryResponse();
NamedList<Object> responseHeader = new SimpleOrderedMap<Object>();
final SolrQueryResponse rsp = new SolrQueryResponse();
final NamedList<Object> responseHeader = new SimpleOrderedMap<>();
responseHeader.add("params", req.getOriginalParams().toNamedList());
rsp.add("responseHeader", responseHeader);
//SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
@ -210,8 +214,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
this.requestHandler.handleRequest(req, rsp);
// get statistics and add a header with that
Exception exception = rsp.getException();
int status = exception == null ? 0 : exception instanceof SolrException ? ((SolrException) exception).code() : 500;
final Exception exception = rsp.getException();
final int status = exception == null ? 0 : exception instanceof SolrException ? ((SolrException) exception).code() : 500;
responseHeader.add("status", status);
responseHeader.add("QTime",(int) (System.currentTimeMillis() - startTime));
@ -219,43 +223,43 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
// return result
return rsp;
}
/**
* conversion from a SolrQueryResponse (which is a solr-internal data format) to SolrDocumentList (which is a solrj-format)
* The conversion is done inside the solrj api using the BinaryResponseWriter and a very complex unfolding process
* via org.apache.solr.common.util.JavaBinCodec.marshal.
* via org.apache.solr.common.util.JavaBinCodec.marshal.
* @param request
* @param sqr
* @return
*/
public SolrDocumentList SolrQueryResponse2SolrDocumentList(final SolrQueryRequest req, final SolrQueryResponse rsp) {
SolrDocumentList sdl = new SolrDocumentList();
NamedList<?> nl = rsp.getValues();
ResultContext resultContext = (ResultContext) nl.get("response");
DocList response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f, TotalHits.Relation.EQUAL_TO) : resultContext.getDocList();
final SolrDocumentList sdl = new SolrDocumentList();
final NamedList<?> nl = rsp.getValues();
final ResultContext resultContext = (ResultContext) nl.get("response");
final DocList response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f, TotalHits.Relation.EQUAL_TO) : resultContext.getDocList();
sdl.setNumFound(response == null ? 0 : response.matches());
sdl.setStart(response == null ? 0 : response.offset());
String originalName = Thread.currentThread().getName();
final String originalName = Thread.currentThread().getName();
if (response != null) {
try {
SolrIndexSearcher searcher = req.getSearcher();
final SolrIndexSearcher searcher = req.getSearcher();
final int responseCount = response.size();
DocIterator iterator = response.iterator();
final DocIterator iterator = response.iterator();
for (int i = 0; i < responseCount; i++) {
int docid = iterator.nextDoc();
final int docid = iterator.nextDoc();
Thread.currentThread().setName("EmbeddedSolrConnector.SolrQueryResponse2SolrDocumentList: " + docid);
Document responsedoc = searcher.doc(docid, (Set<String>) null);
SolrDocument sordoc = doc2SolrDoc(responsedoc);
final Document responsedoc = searcher.doc(docid, (Set<String>) null);
final SolrDocument sordoc = this.doc2SolrDoc(responsedoc);
sdl.add(sordoc);
}
} catch (IOException e) {
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
}
Thread.currentThread().setName(originalName);
return sdl;
}
/**
* The following schemaFieldCache is a hack-patch of a Solr internal request which is really slow.
* The Solr-internal method is flexible because it may respond on a real-time schema change, but that
@ -263,25 +267,25 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
*/
private final Map<String, SchemaField> schemaFieldCache = new ConcurrentHashMap<>();
private final SchemaField getSchemaField(final String fieldName) {
SchemaField sf = schemaFieldCache.get(fieldName);
SchemaField sf = this.schemaFieldCache.get(fieldName);
if (sf == null) {
sf = this.core.getLatestSchema().getFieldOrNull(fieldName);
schemaFieldCache.put(fieldName, sf);
this.schemaFieldCache.put(fieldName, sf);
}
return sf;
}
public SolrDocument doc2SolrDoc(Document doc) {
SolrDocument solrDoc = new SolrDocument();
for (IndexableField field : doc) {
String fieldName = field.name();
SchemaField sf = getSchemaField(fieldName); // hack-patch of this.core.getLatestSchema().getFieldOrNull(fieldName); makes it a lot faster!!
final SolrDocument solrDoc = new SolrDocument();
for (final IndexableField field : doc) {
final String fieldName = field.name();
final SchemaField sf = this.getSchemaField(fieldName); // hack-patch of this.core.getLatestSchema().getFieldOrNull(fieldName); makes it a lot faster!!
Object val = null;
try {
FieldType ft = null;
if (sf != null) ft = sf.getType();
if (ft == null) {
BytesRef bytesRef = field.binaryValue();
final BytesRef bytesRef = field.binaryValue();
if (bytesRef != null) {
if (bytesRef.offset == 0 && bytesRef.length == bytesRef.bytes.length) {
val = bytesRef.bytes;
@ -296,12 +300,12 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
} else {
val = ft.toObject(field);
}
} catch (Throwable e) {
} catch (final Throwable e) {
continue;
}
if (sf != null && sf.multiValued() && !solrDoc.containsKey(fieldName)) {
ArrayList<Object> l = new ArrayList<Object>();
final ArrayList<Object> l = new ArrayList<>();
l.add(val);
solrDoc.addField(fieldName, l);
} else {
@ -311,7 +315,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
return solrDoc;
}
/**
* the usage of getResponseByParams is disencouraged for the embedded Solr connector. Please use request(SolrParams) instead.
* Reason: Solr makes a very complex folding/unfolding including data compression for SolrQueryResponses.
@ -320,7 +324,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException {
if (this.server == null) throw new IOException("server disconnected");
// during the solr query we set the thread name to the query string to get more debugging info in thread dumps
String threadname = Thread.currentThread().getName();
final String threadname = Thread.currentThread().getName();
String ql = "";
try {ql = URLDecoder.decode(params.toString(), StandardCharsets.UTF_8.name());} catch (UnsupportedEncodingException|IllegalStateException e) {e.printStackTrace();}
Thread.currentThread().setName("Embedded.getResponseByParams solr query: q=" + ql);
@ -328,6 +332,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
//System.out.println("EmbeddedSolrConnector.getResponseByParams * QUERY: " + ql); System.out.println("STACKTRACE: " + ConcurrentLog.stackTrace());
QueryResponse rsp;
try {
System.out.println("*** PARAMS: " + params);
rsp = this.server.query(params);
Thread.currentThread().setName(threadname);
if (rsp != null) if (log.isFine()) log.fine(rsp.getResults().getNumFound() + " results for " + ql);
@ -350,24 +355,24 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
*/
@Override
public SolrDocumentList getDocumentListByParams(ModifiableSolrParams params) throws IOException, SolrException {
SolrQueryRequest req = this.request(params);
final SolrQueryRequest req = this.request(params);
SolrQueryResponse response = null;
String q = params.get(CommonParams.Q);
String fq = params.get(CommonParams.FQ);
String sort = params.get(CommonParams.SORT);
String threadname = Thread.currentThread().getName();
final String q = params.get(CommonParams.Q);
final String fq = params.get(CommonParams.FQ);
final String sort = params.get(CommonParams.SORT);
final String threadname = Thread.currentThread().getName();
try {
if (q != null) Thread.currentThread().setName("Embedded.getDocumentListByParams solr query: q = " + q + (fq == null ? "" : ", fq = " + fq) + (sort == null ? "" : ", sort = " + sort)); // for debugging in Threaddump
response = this.query(req);
if (q != null) Thread.currentThread().setName(threadname);
if (response == null) throw new IOException("response == null");
return SolrQueryResponse2SolrDocumentList(req, response);
return this.SolrQueryResponse2SolrDocumentList(req, response);
} finally {
req.close();
}
}
private class DocListSearcher implements AutoCloseable {
private SolrQueryRequest request;
private DocList response;
@ -375,16 +380,16 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
public DocListSearcher(final String querystring, String sort, final int offset, final int count, final String ... fields) {
// construct query
final SolrQuery params = AbstractSolrConnector.getSolrQuery(querystring, sort, offset, count, fields);
// query the server
this.request = EmbeddedSolrConnector.this.request(params);
SolrQueryResponse rsp = query(request);
NamedList<?> nl = rsp.getValues();
ResultContext resultContext = (ResultContext) nl.get("response");
final SolrQueryResponse rsp = EmbeddedSolrConnector.this.query(this.request);
final NamedList<?> nl = rsp.getValues();
final ResultContext resultContext = (ResultContext) nl.get("response");
if (resultContext == null) log.warn("DocListSearcher: no response for query '" + querystring + "'");
this.response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f, TotalHits.Relation.EQUAL_TO) : resultContext.getDocList();
}
@Override
public void close() {
if (this.request != null) this.request.close();
@ -392,7 +397,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
this.response = null;
}
}
@Override
public long getCountByQuery(String querystring) {
long numFound = 0;
@ -400,12 +405,12 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
try {
docListSearcher = new DocListSearcher(querystring, null, 0, 0, CollectionSchema.id.getSolrFieldName());
numFound = docListSearcher.response.matches();
} finally {
} finally {
if (docListSearcher != null) docListSearcher.close();
}
return numFound;
}
/**
* check if a given document, identified by url hash as document id exists
* @param id the url hash and document id
@ -416,7 +421,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
final String query = "{!cache=false raw f=" + CollectionSchema.id.getSolrFieldName() + "}" + id;
try (DocListSearcher docListSearcher = new DocListSearcher(query, null, 0, 0, CollectionSchema.id.getSolrFieldName())) {
return docListSearcher.response.matches() > 0l;
} catch (Throwable e) {
} catch (final Throwable e) {
ConcurrentLog.logException(e);
return false;
}
@ -436,14 +441,14 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
docListSearcher = new DocListSearcher("{!cache=false raw f=" + CollectionSchema.id.getSolrFieldName() + "}" + id, null, 0, 1, CollectionSchema.id.getSolrFieldName(), CollectionSchema.load_date_dt.getSolrFieldName());
responseCount = docListSearcher.response.size();
if (responseCount == 0) return null;
SolrIndexSearcher searcher = docListSearcher.request.getSearcher();
DocIterator iterator = docListSearcher.response.iterator();
final SolrIndexSearcher searcher = docListSearcher.request.getSearcher();
final DocIterator iterator = docListSearcher.response.iterator();
//for (int i = 0; i < responseCount; i++) {
Document doc = searcher.doc(iterator.nextDoc(), AbstractSolrConnector.SOLR_ID_and_LOAD_DATE_FIELDS);
final Document doc = searcher.doc(iterator.nextDoc(), AbstractSolrConnector.SOLR_ID_and_LOAD_DATE_FIELDS);
if (doc == null) return null;
return AbstractSolrConnector.getLoadTimeURL(doc);
//}
} catch (Throwable e) {
} catch (final Throwable e) {
ConcurrentLog.logException(e);
throw new IOException(e.getMessage());
} finally {

966
source/net/yacy/search/Switchboard.java
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save