boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

better abstraction for solr objects

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 8864141872
commit adeb33bb36
7 changed files with 45 additions and 34 deletions
Show Stats Download Patch File Download Diff File

3
source/net/yacy/cora/services/federated/solr/SolrConnector.java
Unescape View File

@ -31,7 +31,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
public interface SolrConnector {
@ -71,7 +70,7 @@ public interface SolrConnector {
* @throws IOException
* @throws SolrException
*/
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException;
public void add(final SolrDoc solrdoc) throws IOException, SolrException;
/**
* register an entry as error document

19
source/net/yacy/cora/services/federated/solr/SolrField.java
Unescape View File

@ -27,6 +27,23 @@ package net.yacy.cora.services.federated.solr;
public interface SolrField {
/**
* this shall be implemented as enum, thus shall have the name() method
* @return the name of the enum constant
*/
public String name();
}
public SolrType getType();
public boolean isIndexed();
public boolean isStored();
public boolean isMultiValued();
public boolean isOmitNorms();
public String getComment();
}

3
source/net/yacy/cora/services/federated/solr/SolrRetryConnector.java
Unescape View File

@ -31,7 +31,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
public class SolrRetryConnector implements SolrConnector {
@ -109,7 +108,7 @@ public class SolrRetryConnector implements SolrConnector {
}
@Override
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
public void add(final SolrDoc solrdoc) throws IOException, SolrException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {

8
source/net/yacy/cora/services/federated/solr/SolrShardingConnector.java
Unescape View File

@ -31,12 +31,10 @@ import java.util.Collection;
import java.util.List;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.services.federated.solr.SolrShardingSelection.Method;
import net.yacy.kelondro.data.meta.DigestURI;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
public class SolrShardingConnector implements SolrConnector {
@ -109,7 +107,7 @@ public class SolrShardingConnector implements SolrConnector {
* @throws IOException
*/
@Override
public void add(final SolrInputDocument solrdoc) throws IOException {
public void add(final SolrDoc solrdoc) throws IOException {
this.connectors.get(this.sharding.select(solrdoc)).add(solrdoc);
}
@ -118,8 +116,8 @@ public class SolrShardingConnector implements SolrConnector {
* @param docs
* @throws IOException
*/
protected void addSolr(final Collection<SolrInputDocument> docs) throws IOException {
for (final SolrInputDocument doc: docs) add(doc);
protected void addSolr(final Collection<SolrDoc> docs) throws IOException {
for (final SolrDoc doc: docs) add(doc);
}
/**

4
source/net/yacy/cora/services/federated/solr/SolrShardingSelection.java
Unescape View File

@ -32,8 +32,6 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.solr.common.SolrInputDocument;
public class SolrShardingSelection {
public final static Charset charsetUTF8;
@ -58,7 +56,7 @@ public class SolrShardingSelection {
return (int) (this.chardID.getAndIncrement() % this.dimension);
}
public int select(final SolrInputDocument solrdoc) throws IOException {
public int select(final SolrDoc solrdoc) throws IOException {
if (this.method == Method.MODULO_HOST_MD5) {
final String sku = (String) solrdoc.getField("sku").getValue();
return selectURL(sku);

8
source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java
Unescape View File

@ -67,7 +67,7 @@ public class SolrSingleConnector implements SolrConnector {
private final static int transmissionQueueCount = 4; // allow concurrent http sessions to solr
private final static int transmissionQueueSize = 50; // number of documents that are collected until a commit is sent
private final Worker[] transmissionWorker; // the transmission workers to solr
private final BlockingQueue<SolrInputDocument>[] transmissionQueue; // the queues quere documents are collected
private final BlockingQueue<SolrDoc>[] transmissionQueue; // the queues quere documents are collected
private int transmissionRoundRobinCounter; // a rount robin counter for the transmission queues
/**
@ -82,7 +82,7 @@ public class SolrSingleConnector implements SolrConnector {
this.transmissionRoundRobinCounter = 0;
this.transmissionQueue = new ArrayBlockingQueue[transmissionQueueCount];
for (int i = 0; i < transmissionQueueCount; i++) {
this.transmissionQueue[i] = new ArrayBlockingQueue<SolrInputDocument>(transmissionQueueSize);
this.transmissionQueue[i] = new ArrayBlockingQueue<SolrDoc>(transmissionQueueSize);
}
// connect using authentication
@ -253,7 +253,7 @@ public class SolrSingleConnector implements SolrConnector {
}
@Override
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
public void add(final SolrDoc solrdoc) throws IOException, SolrException {
int thisrrc = this.transmissionRoundRobinCounter;
int nextrrc = thisrrc++;
if (nextrrc >= transmissionQueueCount) nextrrc = 0;
@ -290,7 +290,7 @@ public class SolrSingleConnector implements SolrConnector {
@Override
public void err(final DigestURI digestURI, final String failReason, final int httpstatus) throws IOException {
final SolrInputDocument solrdoc = new SolrInputDocument();
final SolrDoc solrdoc = new SolrDoc();
solrdoc.addField("id", ASCII.String(digestURI.hash()));
solrdoc.addField("sku", digestURI.toNormalform(true, false), 3.0f);
final InetAddress address = digestURI.getInetAddress();

34
source/net/yacy/search/index/SolrConfiguration.java
Unescape View File

@ -40,6 +40,7 @@ import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.services.federated.solr.SolrDoc;
import net.yacy.cora.storage.ConfigurationSet;
import net.yacy.document.Document;
import net.yacy.document.parser.html.ContentScraper;
@ -48,7 +49,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
public class SolrConfiguration extends ConfigurationSet {
@ -84,37 +84,37 @@ public class SolrConfiguration extends ConfigurationSet {
*/
}
protected void addSolr(final SolrInputDocument solrdoc, final SolrField key, final String value) {
if (isEmpty() || contains(key.name())) solrdoc.setField(key.name(), value);
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final String value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrInputDocument solrdoc, final SolrField key, final Date value) {
if (isEmpty() || contains(key.name())) solrdoc.setField(key.name(), value);
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final Date value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrInputDocument solrdoc, final SolrField key, final int value) {
if (isEmpty() || contains(key.name())) solrdoc.setField(key.name(), value);
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final int value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrInputDocument solrdoc, final SolrField key, final String[] value) {
if (isEmpty() || contains(key.name())) solrdoc.setField(key.name(), value);
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final String[] value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrInputDocument solrdoc, final SolrField key, final float value) {
if (isEmpty() || contains(key.name())) solrdoc.setField(key.name(), value);
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final float value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrInputDocument solrdoc, final SolrField key, final boolean value) {
if (isEmpty() || contains(key.name())) solrdoc.setField(key.name(), value);
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final boolean value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrInputDocument solrdoc, final SolrField key, final String value, final float boost) {
if (isEmpty() || contains(key.name())) solrdoc.setField(key.name(), value, boost);
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final String value, final float boost) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value, boost);
}
public SolrInputDocument yacy2solr(final String id, final ResponseHeader header, final Document yacydoc) {
public SolrDoc yacy2solr(final String id, final ResponseHeader header, final Document yacydoc) {
// we user the SolrCell design as index scheme
final SolrInputDocument solrdoc = new SolrInputDocument();
final SolrDoc solrdoc = new SolrDoc();
final DigestURI digestURI = new DigestURI(yacydoc.dc_source());
addSolr(solrdoc, SolrField.failreason_t, ""); // overwrite a possible fail reason (in case that there was a fail reason before)
addSolr(solrdoc, SolrField.id, id);

Write Preview
Loading…
Cancel
Save