boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

code simplification

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 54bea21c02
commit 06a78eecb7
10 changed files with 34 additions and 86 deletions
Show Stats Download Patch File Download Diff File

6
htroot/IndexControlRWIs_p.java
Unescape View File

@ -93,7 +93,7 @@ public class IndexControlRWIs_p {
prop.put("keyhash", "");
prop.put("result", "");
prop.put("cleanup", post == null || post.containsKey("maxReferencesLimit") ? 1 : 0);
prop.put("cleanup_solr", sb.index.connectedSolr() ? 1 : 0);
prop.put("cleanup_solr", sb.index.urlMetadata().connectedSolr() ? 1 : 0);
// switch off all optional forms/lists
prop.put("searchresult", 0);
@ -158,9 +158,9 @@ public class IndexControlRWIs_p {
if ( post.get("deleteIndex", "").equals("on") ) {
segment.clear();
}
if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.connectedSolr()) {
if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.urlMetadata().connectedSolr()) {
try {
sb.index.getSolr().clear();
sb.index.urlMetadata().getSolr().clear();
} catch ( final Exception e ) {
Log.logException(e);
}

28
htroot/IndexFederated_p.java
Unescape View File

@ -73,23 +73,23 @@ public class IndexFederated_p {
} catch (IOException e) { Log.logException(e); } // switch on
boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT);
final boolean previous_core_fulltext = sb.index.connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
final boolean previous_core_fulltext = sb.index.urlMetadata().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000));
if (previous_core_fulltext && !post_core_fulltext) {
// switch off
sb.index.disconnectLocalSolr();
sb.index.disconnectUrlDb();
sb.index.urlMetadata().disconnectLocalSolr();
sb.index.urlMetadata().disconnectUrlDb();
}
if (!previous_core_fulltext && post_core_fulltext) {
// switch on
sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
try { sb.index.connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); }
try { sb.index.urlMetadata().connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); }
}
// solr
final boolean solrRemoteWasOn = sb.index.connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true);
final boolean solrRemoteWasOn = sb.index.urlMetadata().connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true);
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote");
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
@ -119,7 +119,7 @@ public class IndexFederated_p {
if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {
// switch off
sb.index.disconnectRemoteSolr();
sb.index.urlMetadata().disconnectRemoteSolr();
}
if (!solrRemoteWasOn && solrRemoteIsOnAfterwards) {
@ -129,18 +129,18 @@ public class IndexFederated_p {
if (usesolr) {
SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true);
solr.setCommitWithinMs(commitWithinMs);
sb.index.connectRemoteSolr(solr);
sb.index.urlMetadata().connectRemoteSolr(solr);
} else {
sb.index.disconnectRemoteSolr();
sb.index.urlMetadata().disconnectRemoteSolr();
}
} catch (final IOException e) {
Log.logException(e);
sb.index.disconnectRemoteSolr();
sb.index.urlMetadata().disconnectRemoteSolr();
}
}
// read index scheme table flags
final Iterator<ConfigurationSet.Entry> i = sb.index.getSolrScheme().entryIterator();
final Iterator<ConfigurationSet.Entry> i = sb.index.urlMetadata().getSolrScheme().entryIterator();
ConfigurationSet.Entry entry;
boolean modified = false; // flag to remember changes
while (i.hasNext()) {
@ -163,18 +163,18 @@ public class IndexFederated_p {
}
if (modified) { // save settings to config file if modified
try {
sb.index.getSolrScheme().commit();
sb.index.urlMetadata().getSolrScheme().commit();
modified = false;
} catch (IOException ex) {}
}
}
// show solr host table
if (!sb.index.connectedRemoteSolr()) {
if (!sb.index.urlMetadata().connectedRemoteSolr()) {
prop.put("table", 0);
} else {
prop.put("table", 1);
final SolrConnector solr = sb.index.getRemoteSolr();
final SolrConnector solr = sb.index.urlMetadata().getRemoteSolr();
final long[] size = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getSizeList() : new long[]{((SingleSolrConnector) solr).getSize()};
final String[] urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaceList() : new String[]{((SingleSolrConnector) solr).getAdminInterface()};
boolean dark = false;
@ -194,7 +194,7 @@ public class IndexFederated_p {
// use enum SolrField to keep defined order
for(YaCySchema field : YaCySchema.values()) {
prop.put("scheme_" + c + "_dark", dark ? 1 : 0); dark = !dark;
prop.put("scheme_" + c + "_checked", sb.index.getSolrScheme().contains(field.name()) ? 1 : 0);
prop.put("scheme_" + c + "_checked", sb.index.urlMetadata().getSolrScheme().contains(field.name()) ? 1 : 0);
prop.putHTML("scheme_" + c + "_key", field.name());
prop.putHTML("scheme_" + c + "_solrfieldname",field.name().equalsIgnoreCase(field.getSolrFieldName()) ? "" : field.getSolrFieldName());
if (field.getComment() != null) prop.putHTML("scheme_" + c + "_comment",field.getComment());

2
htroot/api/schema_p.java
Unescape View File

@ -38,7 +38,7 @@ public class schema_p {
// write scheme
int c = 0;
SolrConfiguration solrScheme = sb.index.getSolrScheme();
SolrConfiguration solrScheme = sb.index.urlMetadata().getSolrScheme();
for (YaCySchema field : YaCySchema.values()) {
if (solrScheme.contains(field.name())) {
prop.put("fields_" + c + "_solrname", field.getSolrFieldName());

2
htroot/gsa/searchresult.java
Unescape View File

@ -96,7 +96,7 @@ public class searchresult {
post.put(CommonParams.ROWS, post.remove("num"));
// get the embedded connector
EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr();
EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.urlMetadata().getLocalSolr();
if (connector == null) return null;
// do the solr request

2
htroot/solr/select.java
Unescape View File

@ -146,7 +146,7 @@ public class select {
}
// get the embedded connector
EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr();
EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.urlMetadata().getLocalSolr();
if (connector == null) return null;
// do the solr request

8
source/de/anomic/crawler/CrawlQueues.java
Unescape View File

@ -82,8 +82,8 @@ public class CrawlQueues {
this.log.logConfig("Starting Crawling Management");
this.noticeURL = new NoticedURL(queuePath, sb.peers.myBotIDs(), sb.useTailCache, sb.exceed134217727);
FileUtils.deletedelete(new File(queuePath, ERROR_DB_FILENAME));
this.errorURL = new ZURL(sb.index.getSolr(), sb.index.getSolrScheme(), queuePath, ERROR_DB_FILENAME, false, sb.useTailCache, sb.exceed134217727);
this.delegatedURL = new ZURL(sb.index.getSolr(), sb.index.getSolrScheme(), queuePath, DELEGATED_DB_FILENAME, true, sb.useTailCache, sb.exceed134217727);
this.errorURL = new ZURL(sb.index.urlMetadata().getSolr(), sb.index.urlMetadata().getSolrScheme(), queuePath, ERROR_DB_FILENAME, false, sb.useTailCache, sb.exceed134217727);
this.delegatedURL = new ZURL(sb.index.urlMetadata().getSolr(), sb.index.urlMetadata().getSolrScheme(), queuePath, DELEGATED_DB_FILENAME, true, sb.useTailCache, sb.exceed134217727);
}
public void relocate(final File newQueuePath) {
@ -94,8 +94,8 @@ public class CrawlQueues {
this.noticeURL = new NoticedURL(newQueuePath, this.sb.peers.myBotIDs(), this.sb.useTailCache, this.sb.exceed134217727);
FileUtils.deletedelete(new File(newQueuePath, ERROR_DB_FILENAME));
this.errorURL = new ZURL(this.sb.index.getSolr(), this.sb.index.getSolrScheme(), newQueuePath, ERROR_DB_FILENAME, false, this.sb.useTailCache, this.sb.exceed134217727);
this.delegatedURL = new ZURL(this.sb.index.getSolr(), this.sb.index.getSolrScheme(), newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727);
this.errorURL = new ZURL(this.sb.index.urlMetadata().getSolr(), this.sb.index.urlMetadata().getSolrScheme(), newQueuePath, ERROR_DB_FILENAME, false, this.sb.useTailCache, this.sb.exceed134217727);
this.delegatedURL = new ZURL(this.sb.index.urlMetadata().getSolr(), this.sb.index.urlMetadata().getSolrScheme(), newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727);
}
public synchronized void close() {

10
source/net/yacy/search/Switchboard.java
Unescape View File

@ -397,7 +397,7 @@ public final class Switchboard extends serverSwitch
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
this.index.connectLocalSolr(connectWithinMs);
this.index.urlMetadata().connectLocalSolr(connectWithinMs);
}
// set up the solr interface
@ -411,7 +411,7 @@ public final class Switchboard extends serverSwitch
ShardSelection.Method.MODULO_HOST_MD5,
10000, true);
solr.setCommitWithinMs(connectWithinMs);
this.index.connectRemoteSolr(solr);
this.index.urlMetadata().connectRemoteSolr(solr);
} catch ( final IOException e ) {
Log.logException(e);
}
@ -1133,7 +1133,7 @@ public final class Switchboard extends serverSwitch
synchronized ( this ) {
// remember the solr scheme
SolrConfiguration solrScheme = this.index.getSolrScheme();
SolrConfiguration solrScheme = this.index.urlMetadata().getSolrScheme();
// shut down
this.crawler.close();
@ -1186,7 +1186,7 @@ public final class Switchboard extends serverSwitch
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.connectLocalSolr(connectWithinMs);
this.index.urlMetadata().connectLocalSolr(connectWithinMs);
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
}
@ -1201,7 +1201,7 @@ public final class Switchboard extends serverSwitch
ShardSelection.Method.MODULO_HOST_MD5,
10000, true);
solr.setCommitWithinMs(connectWithinMs);
this.index.connectRemoteSolr(solr);
this.index.urlMetadata().connectRemoteSolr(solr);
} catch ( final IOException e ) {
Log.logException(e);
}

2
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -80,7 +80,7 @@ public class DocumentIndex extends Segment {
false, // useTailCache
false // exceed134217727
);
super.connectLocalSolr(1000);
super.urlMetadata().connectLocalSolr(1000);
final int cores = Runtime.getRuntime().availableProcessors() + 1;
this.callback = callback;
this.queue = new LinkedBlockingQueue<DigestURI>(cores * 300);

58
source/net/yacy/search/index/Segment.java
Unescape View File

@ -39,7 +39,6 @@ import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.order.ByteOrder;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.services.federated.solr.SolrConnector;
import net.yacy.cora.services.federated.solr.SolrDoc;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.cora.storage.HandleSet;
@ -162,61 +161,10 @@ public class Segment {
this.urlCitationIndex = null;
}
public boolean connectedUrlDb() {
return this.urlMetadata.connectedUrlDb();
}
public void connectUrlDb(final boolean useTailCache, final boolean exceed134217727) {
this.urlMetadata.connectUrlDb(UrlDbName, useTailCache, exceed134217727);
}
public void disconnectUrlDb() {
this.urlMetadata.disconnectUrlDb();
}
public boolean connectedSolr() {
return this.urlMetadata.connectedSolr();
}
public boolean connectedRemoteSolr() {
return this.urlMetadata.connectedRemoteSolr();
}
public void connectRemoteSolr(final SolrConnector solr) {
this.urlMetadata.connectRemoteSolr(solr);
}
public void disconnectRemoteSolr() {
this.urlMetadata.disconnectRemoteSolr();
}
public boolean connectedLocalSolr() {
return this.urlMetadata.connectedLocalSolr();
}
public void connectLocalSolr(final int connectWithin) throws IOException {
this.urlMetadata.connectLocalSolr(connectWithin);
}
public void disconnectLocalSolr() {
this.urlMetadata.disconnectLocalSolr();
}
public SolrConnector getSolr() {
return this.urlMetadata.getSolr();
}
public SolrConfiguration getSolrScheme() {
return this.urlMetadata.getSolrScheme();
}
public SolrConnector getRemoteSolr() {
return this.urlMetadata.getRemoteSolr();
}
public SolrConnector getLocalSolr() {
return this.urlMetadata.getLocalSolr();
}
public MetadataRepository urlMetadata() {
return this.urlMetadata;
}
@ -452,13 +400,13 @@ public class Segment {
document.getVideolinks().size(), // lvideo
document.getApplinks().size() // lapp
);
// STORE TO SOLR
// we do not store the data in metadatadb any more if a solr is connected
if (this.connectedSolr()) {
if (this.urlMetadata.connectedSolr()) {
try {
SolrDoc solrDoc = this.urlMetadata.getSolrScheme().yacy2solr(id, responseHeader, document, metadata);
this.getSolr().add(solrDoc);
this.urlMetadata.getSolr().add(solrDoc);
} catch ( final IOException e ) {
Log.logWarning("SOLR", "failed to send " + urlNormalform + " to solr: " + e.getMessage());
}

2
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -208,7 +208,7 @@ public final class RWIProcess extends Thread
String solrQuery = RWIProcess.this.query.solrQuery();
try {
ReferenceContainer<WordReference> wr = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, null);
SolrDocumentList sdl = RWIProcess.this.query.getSegment().getSolr().query(solrQuery, 0, 20);
SolrDocumentList sdl = RWIProcess.this.query.getSegment().urlMetadata().getSolr().query(solrQuery, 0, 20);
for (SolrDocument d : sdl) {
try {wr.add(new WordReferenceVars(d));} catch (SpaceExceededException e) {}
}

Write Preview
Loading…
Cancel
Save