boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed the commitWithin attribute because that is not the way how the

Browse Source 
index is updated the right way for us. May also be be superfluous with
the solr 4.0 softcommit.
pull/1/head
Michael Peter Christen 12 years ago
parent c20fa3640d
commit 4111606654
12 changed files with 13 additions and 127 deletions
Show Stats Download Patch File Download Diff File

1
defaults/yacy.init
Unescape View File

@ -1056,7 +1056,6 @@ color_searchurlhover = #008000
# - to check whats in solr after indexing, open http://localhost:8983/solr/admin/
federated.service.solr.indexing.enabled = false
federated.service.solr.indexing.url = http://127.0.0.1:8983/solr
federated.service.solr.indexing.commitWithinMs = -1
federated.service.solr.indexing.sharding = MODULO_HOST_MD5
federated.service.solr.indexing.schemefile = solr.keys.default.list
# the lazy attribute causes that fields containing "" or 0 are not added and not written

6
htroot/IndexFederated_p.java
Unescape View File

@ -79,7 +79,6 @@ public class IndexFederated_p {
final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1));
if (previous_core_fulltext && !post_core_fulltext) {
// switch off
sb.index.fulltext().disconnectLocalSolr();
@ -88,7 +87,7 @@ public class IndexFederated_p {
if (!previous_core_fulltext && post_core_fulltext) {
// switch on
sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
try { sb.index.fulltext().connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); }
try { sb.index.fulltext().connectLocalSolr(); } catch (IOException e) { Log.logException(e); }
}
// solr
@ -114,7 +113,6 @@ public class IndexFederated_p {
}
solrurls = s.toString().trim();
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, commitWithinMs);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")));
final String schemename = post.get("solr.indexing.schemefile", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list"));
@ -136,7 +134,6 @@ public class IndexFederated_p {
try {
if (usesolr) {
SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true);
if (commitWithinMs >= 0) solr.setCommitWithinMs(commitWithinMs);
sb.index.fulltext().connectRemoteSolr(solr);
} else {
sb.index.fulltext().disconnectRemoteSolr();
@ -225,7 +222,6 @@ public class IndexFederated_p {
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
prop.put("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000));
prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"));
prop.put("solr.indexing.schemefile", schemename);

17
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -148,23 +148,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
this.documentCache.clear();
}
@Override
public int getCommitWithinMs() {
if (this.solr0 != null) this.solr0.getCommitWithinMs();
if (this.solr1 != null) this.solr1.getCommitWithinMs();
return -1;
}
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(int c) {
if (this.solr0 != null) this.solr0.setCommitWithinMs(c);
if (this.solr1 != null) this.solr1.setCommitWithinMs(c);
}
@Override
public void commit(boolean softCommit) {
if (this.solr0 != null) this.solr0.commit(softCommit);

19
source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
Unescape View File

@ -36,13 +36,11 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
private final ArrayBlockingQueue<SolrInputDocument> queue;
private final AddWorker[] worker;
private final SolrConnector solr;
private int commitWithinMs;
public MultipleSolrConnector(final String url, final int connections) throws IOException {
this.solr = new RemoteSolrConnector(url);
this.queue = new ArrayBlockingQueue<SolrInputDocument>(1000);
this.worker = new AddWorker[connections];
this.commitWithinMs = -1;
for (int i = 0; i < connections; i++) {
this.worker[i] = new AddWorker(url);
this.worker[i].start();
@ -53,7 +51,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
private final SolrConnector solr;
public AddWorker(final String url) throws IOException {
this.solr = new RemoteSolrConnector(url);
if (MultipleSolrConnector.this.commitWithinMs >= 0 ) this.solr.setCommitWithinMs(MultipleSolrConnector.this.commitWithinMs);
}
@Override
public void run() {
@ -75,22 +72,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
}
}
@Override
public int getCommitWithinMs() {
return this.commitWithinMs;
}
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(final int c) {
this.commitWithinMs = c;
this.solr.setCommitWithinMs(c);
for (AddWorker w: this.worker) w.solr.setCommitWithinMs(c);
}
@Override
public void commit(boolean softCommit) {
this.solr.commit(softCommit);

14
source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java
Unescape View File

@ -38,20 +38,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon
this.retryMaxTime = retryMaxTime;
}
@Override
public int getCommitWithinMs() {
return this.solrConnector.getCommitWithinMs();
}
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(int c) {
this.solrConnector.setCommitWithinMs(c);
}
@Override
public void commit(boolean softCommit) {
this.solrConnector.commit(softCommit);

14
source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java
Unescape View File

@ -58,20 +58,6 @@ public class ShardSolrConnector extends AbstractSolrConnector implements SolrCon
this.sharding = new ShardSelection(method, this.urls.length);
}
@Override
public int getCommitWithinMs() {
return this.connectors.get(0).getCommitWithinMs();
}
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(int c) {
for (final SolrConnector connector: this.connectors) connector.setCommitWithinMs(c);
}
@Override
public void commit(boolean softCommit) {
for (final SolrConnector connector: this.connectors) connector.commit(softCommit);

12
source/net/yacy/cora/federate/solr/connector/SolrConnector.java
Unescape View File

@ -36,18 +36,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
public interface SolrConnector extends Iterable<String> /* Iterable of document IDs */ {
/**
* get the solr autocommit delay
* @return the maximum waiting time after a solr command until it is transported to the server
*/
public int getCommitWithinMs();
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
public void setCommitWithinMs(int c);
/**
* force a commit
*/

33
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -56,11 +56,9 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
protected final static Logger log = Logger.getLogger(SolrServerConnector.class);
protected SolrServer server;
protected int commitWithinMs; // max time (in ms) before a commit will happen
protected SolrServerConnector() {
this.server = null;
this.commitWithinMs = 180000;
}
protected void init(SolrServer server) {
@ -71,25 +69,6 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
return this.server;
}
/**
* get the solr autocommit delay
* @return the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public int getCommitWithinMs() {
return this.commitWithinMs;
}
/**
* set the solr autocommit delay
* when doing continuous inserts, don't set this value because it would cause continuous commits
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(int c) {
this.commitWithinMs = c;
}
@Override
public synchronized void commit(final boolean softCommit) {
//if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).getHttpClient().getConnectionManager().closeExpiredConnections();
@ -209,7 +188,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void delete(final String id) throws IOException {
try {
synchronized (this.server) {
this.server.deleteById(id, this.commitWithinMs);
this.server.deleteById(id, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@ -220,7 +199,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void delete(final List<String> ids) throws IOException {
try {
synchronized (this.server) {
this.server.deleteById(ids, this.commitWithinMs);
this.server.deleteById(ids, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@ -236,7 +215,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void deleteByQuery(final String querystring) throws IOException {
try {
synchronized (this.server) {
this.server.deleteByQuery(querystring, this.commitWithinMs);
this.server.deleteByQuery(querystring, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@ -249,7 +228,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
up.setParam("literal.id", solrId);
up.setParam("uprefix", "attr_");
up.setParam("fmap.content", "attr_content");
up.setCommitWithin(this.commitWithinMs);
up.setCommitWithin(-1);
//up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
try {
synchronized (this.server) {
@ -266,7 +245,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try {
if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict"
synchronized (this.server) {
this.server.add(solrdoc, this.commitWithinMs);
this.server.add(solrdoc, -1);
}
} catch (Throwable e) {
// catches "version conflict for": try this again and delete the document in advance
@ -275,7 +254,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
} catch (SolrServerException e1) {}
try {
synchronized (this.server) {
this.server.add(solrdoc, this.commitWithinMs);
this.server.add(solrdoc, -1);
}
} catch (Throwable ee) {
log.warn(e.getMessage() + " DOC=" + solrdoc.toString());

8
source/net/yacy/search/Switchboard.java
Unescape View File

@ -428,12 +428,11 @@ public final class Switchboard extends serverSwitch {
ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0);
final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS");
this.index = new Segment(this.log, new File(segmentsPath, "default"), solrScheme);
final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
this.index.fulltext().connectLocalSolr(connectWithinMs);
this.index.fulltext().connectLocalSolr();
}
// set up the solr interface
@ -446,7 +445,6 @@ public final class Switchboard extends serverSwitch {
solrurls,
ShardSelection.Method.MODULO_HOST_MD5,
10000, true);
solr.setCommitWithinMs(connectWithinMs);
this.index.fulltext().connectRemoteSolr(solr);
} catch ( final IOException e ) {
Log.logException(e);
@ -1282,11 +1280,10 @@ public final class Switchboard extends serverSwitch {
this.useTailCache,
this.exceed134217727);
this.index = new Segment(this.log, new File(new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), "default"), solrScheme);
final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.fulltext().connectLocalSolr(connectWithinMs);
this.index.fulltext().connectLocalSolr();
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
}
@ -1300,7 +1297,6 @@ public final class Switchboard extends serverSwitch {
solrurls,
ShardSelection.Method.MODULO_HOST_MD5,
10000, true);
if (connectWithinMs >= 0) solr.setCommitWithinMs(connectWithinMs);
this.index.fulltext().connectRemoteSolr(solr);
} catch ( final IOException e ) {
Log.logException(e);

1
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -294,7 +294,6 @@ public final class SwitchboardConstants {
public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS = "federated.service.solr.indexing.commitWithinMs";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_SHARDING = "federated.service.solr.indexing.sharding";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE = "federated.service.solr.indexing.schemefile";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy";

2
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -70,7 +70,7 @@ public class DocumentIndex extends Segment {
false, // useTailCache
false // exceed134217727
);
super.fulltext().connectLocalSolr(-1);
super.fulltext().connectLocalSolr();
final int cores = Runtime.getRuntime().availableProcessors() + 1;
this.callback = callback;
this.queue = new LinkedBlockingQueue<DigestURI>(cores * 300);

13
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -142,7 +142,7 @@ public final class Fulltext {
return this.solr.isConnected0();
}
public void connectLocalSolr(final int commitWithin) throws IOException {
public void connectLocalSolr() throws IOException {
File baseLocation = this.location;
if (baseLocation.getName().equals("default")) baseLocation = baseLocation.getParentFile();
File solrLocation = new File(baseLocation, SOLR_PATH);
@ -152,7 +152,6 @@ public final class Fulltext {
if (oldLocation.exists()) oldLocation.renameTo(solrLocation);
}
EmbeddedSolrConnector esc = new EmbeddedSolrConnector(solrLocation, new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr"));
if (commitWithin >= 0) esc.setCommitWithinMs(commitWithin);
Version luceneVersion = esc.getConfig().getLuceneVersion("luceneMatchVersion");
String lvn = luceneVersion.name();
Log.logInfo("Fulltext", "using lucene version " + lvn);
@ -230,10 +229,6 @@ public final class Fulltext {
}
this.solr.close();
}
public int getCommitWithinMs() {
return this.solr.getCommitWithinMs();
}
public void commit(boolean softCommit) {
this.solr.commit(softCommit);
@ -571,7 +566,6 @@ public final class Fulltext {
*/
public File dumpSolr() {
EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0();
int commitWithin = esc.getCommitWithinMs();
File storagePath = esc.getStoragePath();
File zipOut = new File(storagePath.toString() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip");
synchronized (this.solr) {
@ -582,7 +576,7 @@ public final class Fulltext {
Log.logException(e);
} finally {
try {
this.connectLocalSolr(commitWithin);
this.connectLocalSolr();
} catch (IOException e) {
Log.logException(e);
}
@ -597,7 +591,6 @@ public final class Fulltext {
*/
public void restoreSolr(File solrDumpZipFile) {
EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0();
int commitWithin = esc.getCommitWithinMs();
File storagePath = esc.getStoragePath();
synchronized (this.solr) {
this.disconnectLocalSolr();
@ -607,7 +600,7 @@ public final class Fulltext {
Log.logException(e);
} finally {
try {
this.connectLocalSolr(commitWithin);
this.connectLocalSolr();
} catch (IOException e) {
Log.logException(e);
}

Write Preview
Loading…
Cancel
Save