boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

code simplification

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 54bea21c02
commit 06a78eecb7
10 changed files with 34 additions and 86 deletions
Show Stats Download Patch File Download Diff File

6
htroot/IndexControlRWIs_p.java
Unescape View File

@ -93,7 +93,7 @@ public class IndexControlRWIs_p {
prop.put("keyhash", ""); prop.put("keyhash", "");
prop.put("result", ""); prop.put("result", "");
prop.put("cleanup", post == null || post.containsKey("maxReferencesLimit") ? 1 : 0); prop.put("cleanup", post == null || post.containsKey("maxReferencesLimit") ? 1 : 0);
prop.put("cleanup_solr", sb.index.connectedSolr() ? 1 : 0); prop.put("cleanup_solr", sb.index.urlMetadata().connectedSolr() ? 1 : 0);
// switch off all optional forms/lists // switch off all optional forms/lists
prop.put("searchresult", 0); prop.put("searchresult", 0);
@ -158,9 +158,9 @@ public class IndexControlRWIs_p {
if ( post.get("deleteIndex", "").equals("on") ) { if ( post.get("deleteIndex", "").equals("on") ) {
segment.clear(); segment.clear();
} }
if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.connectedSolr()) { if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.urlMetadata().connectedSolr()) {
try { try {
sb.index.getSolr().clear(); sb.index.urlMetadata().getSolr().clear();
} catch ( final Exception e ) { } catch ( final Exception e ) {
Log.logException(e); Log.logException(e);
} }

28
htroot/IndexFederated_p.java
Unescape View File

@ -73,23 +73,23 @@ public class IndexFederated_p {
} catch (IOException e) { Log.logException(e); } // switch on } catch (IOException e) { Log.logException(e); } // switch on
boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT); boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT);
final boolean previous_core_fulltext = sb.index.connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false); final boolean previous_core_fulltext = sb.index.urlMetadata().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext); env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000));
if (previous_core_fulltext && !post_core_fulltext) { if (previous_core_fulltext && !post_core_fulltext) {
// switch off // switch off
sb.index.disconnectLocalSolr(); sb.index.urlMetadata().disconnectLocalSolr();
sb.index.disconnectUrlDb(); sb.index.urlMetadata().disconnectUrlDb();
} }
if (!previous_core_fulltext && post_core_fulltext) { if (!previous_core_fulltext && post_core_fulltext) {
// switch on // switch on
sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727); sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
try { sb.index.connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); } try { sb.index.urlMetadata().connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); }
} }
// solr // solr
final boolean solrRemoteWasOn = sb.index.connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true); final boolean solrRemoteWasOn = sb.index.urlMetadata().connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true);
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote"); final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote");
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr")); String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
@ -119,7 +119,7 @@ public class IndexFederated_p {
if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) { if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {
// switch off // switch off
sb.index.disconnectRemoteSolr(); sb.index.urlMetadata().disconnectRemoteSolr();
} }
if (!solrRemoteWasOn && solrRemoteIsOnAfterwards) { if (!solrRemoteWasOn && solrRemoteIsOnAfterwards) {
@ -129,18 +129,18 @@ public class IndexFederated_p {
if (usesolr) { if (usesolr) {
SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true); SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true);
solr.setCommitWithinMs(commitWithinMs); solr.setCommitWithinMs(commitWithinMs);
sb.index.connectRemoteSolr(solr); sb.index.urlMetadata().connectRemoteSolr(solr);
} else { } else {
sb.index.disconnectRemoteSolr(); sb.index.urlMetadata().disconnectRemoteSolr();
} }
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
sb.index.disconnectRemoteSolr(); sb.index.urlMetadata().disconnectRemoteSolr();
} }
} }
// read index scheme table flags // read index scheme table flags
final Iterator<ConfigurationSet.Entry> i = sb.index.getSolrScheme().entryIterator(); final Iterator<ConfigurationSet.Entry> i = sb.index.urlMetadata().getSolrScheme().entryIterator();
ConfigurationSet.Entry entry; ConfigurationSet.Entry entry;
boolean modified = false; // flag to remember changes boolean modified = false; // flag to remember changes
while (i.hasNext()) { while (i.hasNext()) {
@ -163,18 +163,18 @@ public class IndexFederated_p {
} }
if (modified) { // save settings to config file if modified if (modified) { // save settings to config file if modified
try { try {
sb.index.getSolrScheme().commit(); sb.index.urlMetadata().getSolrScheme().commit();
modified = false; modified = false;
} catch (IOException ex) {} } catch (IOException ex) {}
} }
} }
// show solr host table // show solr host table
if (!sb.index.connectedRemoteSolr()) { if (!sb.index.urlMetadata().connectedRemoteSolr()) {
prop.put("table", 0); prop.put("table", 0);
} else { } else {
prop.put("table", 1); prop.put("table", 1);
final SolrConnector solr = sb.index.getRemoteSolr(); final SolrConnector solr = sb.index.urlMetadata().getRemoteSolr();
final long[] size = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getSizeList() : new long[]{((SingleSolrConnector) solr).getSize()}; final long[] size = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getSizeList() : new long[]{((SingleSolrConnector) solr).getSize()};
final String[] urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaceList() : new String[]{((SingleSolrConnector) solr).getAdminInterface()}; final String[] urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaceList() : new String[]{((SingleSolrConnector) solr).getAdminInterface()};
boolean dark = false; boolean dark = false;
@ -194,7 +194,7 @@ public class IndexFederated_p {
// use enum SolrField to keep defined order // use enum SolrField to keep defined order
for(YaCySchema field : YaCySchema.values()) { for(YaCySchema field : YaCySchema.values()) {
prop.put("scheme_" + c + "_dark", dark ? 1 : 0); dark = !dark; prop.put("scheme_" + c + "_dark", dark ? 1 : 0); dark = !dark;
prop.put("scheme_" + c + "_checked", sb.index.getSolrScheme().contains(field.name()) ? 1 : 0); prop.put("scheme_" + c + "_checked", sb.index.urlMetadata().getSolrScheme().contains(field.name()) ? 1 : 0);
prop.putHTML("scheme_" + c + "_key", field.name()); prop.putHTML("scheme_" + c + "_key", field.name());
prop.putHTML("scheme_" + c + "_solrfieldname",field.name().equalsIgnoreCase(field.getSolrFieldName()) ? "" : field.getSolrFieldName()); prop.putHTML("scheme_" + c + "_solrfieldname",field.name().equalsIgnoreCase(field.getSolrFieldName()) ? "" : field.getSolrFieldName());
if (field.getComment() != null) prop.putHTML("scheme_" + c + "_comment",field.getComment()); if (field.getComment() != null) prop.putHTML("scheme_" + c + "_comment",field.getComment());

2
htroot/api/schema_p.java
Unescape View File

@ -38,7 +38,7 @@ public class schema_p {
// write scheme // write scheme
int c = 0; int c = 0;
SolrConfiguration solrScheme = sb.index.getSolrScheme(); SolrConfiguration solrScheme = sb.index.urlMetadata().getSolrScheme();
for (YaCySchema field : YaCySchema.values()) { for (YaCySchema field : YaCySchema.values()) {
if (solrScheme.contains(field.name())) { if (solrScheme.contains(field.name())) {
prop.put("fields_" + c + "_solrname", field.getSolrFieldName()); prop.put("fields_" + c + "_solrname", field.getSolrFieldName());

2
htroot/gsa/searchresult.java
Unescape View File

@ -96,7 +96,7 @@ public class searchresult {
post.put(CommonParams.ROWS, post.remove("num")); post.put(CommonParams.ROWS, post.remove("num"));
// get the embedded connector // get the embedded connector
EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr(); EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.urlMetadata().getLocalSolr();
if (connector == null) return null; if (connector == null) return null;
// do the solr request // do the solr request

2
htroot/solr/select.java
Unescape View File

@ -146,7 +146,7 @@ public class select {
} }
// get the embedded connector // get the embedded connector
EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr(); EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.urlMetadata().getLocalSolr();
if (connector == null) return null; if (connector == null) return null;
// do the solr request // do the solr request

8
source/de/anomic/crawler/CrawlQueues.java
Unescape View File

@ -82,8 +82,8 @@ public class CrawlQueues {
this.log.logConfig("Starting Crawling Management"); this.log.logConfig("Starting Crawling Management");
this.noticeURL = new NoticedURL(queuePath, sb.peers.myBotIDs(), sb.useTailCache, sb.exceed134217727); this.noticeURL = new NoticedURL(queuePath, sb.peers.myBotIDs(), sb.useTailCache, sb.exceed134217727);
FileUtils.deletedelete(new File(queuePath, ERROR_DB_FILENAME)); FileUtils.deletedelete(new File(queuePath, ERROR_DB_FILENAME));
this.errorURL = new ZURL(sb.index.getSolr(), sb.index.getSolrScheme(), queuePath, ERROR_DB_FILENAME, false, sb.useTailCache, sb.exceed134217727); this.errorURL = new ZURL(sb.index.urlMetadata().getSolr(), sb.index.urlMetadata().getSolrScheme(), queuePath, ERROR_DB_FILENAME, false, sb.useTailCache, sb.exceed134217727);
this.delegatedURL = new ZURL(sb.index.getSolr(), sb.index.getSolrScheme(), queuePath, DELEGATED_DB_FILENAME, true, sb.useTailCache, sb.exceed134217727); this.delegatedURL = new ZURL(sb.index.urlMetadata().getSolr(), sb.index.urlMetadata().getSolrScheme(), queuePath, DELEGATED_DB_FILENAME, true, sb.useTailCache, sb.exceed134217727);
} }
public void relocate(final File newQueuePath) { public void relocate(final File newQueuePath) {
@ -94,8 +94,8 @@ public class CrawlQueues {
this.noticeURL = new NoticedURL(newQueuePath, this.sb.peers.myBotIDs(), this.sb.useTailCache, this.sb.exceed134217727); this.noticeURL = new NoticedURL(newQueuePath, this.sb.peers.myBotIDs(), this.sb.useTailCache, this.sb.exceed134217727);
FileUtils.deletedelete(new File(newQueuePath, ERROR_DB_FILENAME)); FileUtils.deletedelete(new File(newQueuePath, ERROR_DB_FILENAME));
this.errorURL = new ZURL(this.sb.index.getSolr(), this.sb.index.getSolrScheme(), newQueuePath, ERROR_DB_FILENAME, false, this.sb.useTailCache, this.sb.exceed134217727); this.errorURL = new ZURL(this.sb.index.urlMetadata().getSolr(), this.sb.index.urlMetadata().getSolrScheme(), newQueuePath, ERROR_DB_FILENAME, false, this.sb.useTailCache, this.sb.exceed134217727);
this.delegatedURL = new ZURL(this.sb.index.getSolr(), this.sb.index.getSolrScheme(), newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727); this.delegatedURL = new ZURL(this.sb.index.urlMetadata().getSolr(), this.sb.index.urlMetadata().getSolrScheme(), newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727);
} }
public synchronized void close() { public synchronized void close() {

10
source/net/yacy/search/Switchboard.java
Unescape View File

@ -397,7 +397,7 @@ public final class Switchboard extends serverSwitch
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) { if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.connectUrlDb(this.useTailCache, this.exceed134217727); this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
this.index.connectLocalSolr(connectWithinMs); this.index.urlMetadata().connectLocalSolr(connectWithinMs);
} }
// set up the solr interface // set up the solr interface
@ -411,7 +411,7 @@ public final class Switchboard extends serverSwitch
ShardSelection.Method.MODULO_HOST_MD5, ShardSelection.Method.MODULO_HOST_MD5,
10000, true); 10000, true);
solr.setCommitWithinMs(connectWithinMs); solr.setCommitWithinMs(connectWithinMs);
this.index.connectRemoteSolr(solr); this.index.urlMetadata().connectRemoteSolr(solr);
} catch ( final IOException e ) { } catch ( final IOException e ) {
Log.logException(e); Log.logException(e);
} }
@ -1133,7 +1133,7 @@ public final class Switchboard extends serverSwitch
synchronized ( this ) { synchronized ( this ) {
// remember the solr scheme // remember the solr scheme
SolrConfiguration solrScheme = this.index.getSolrScheme(); SolrConfiguration solrScheme = this.index.urlMetadata().getSolrScheme();
// shut down // shut down
this.crawler.close(); this.crawler.close();
@ -1186,7 +1186,7 @@ public final class Switchboard extends serverSwitch
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) { if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.connectLocalSolr(connectWithinMs); this.index.urlMetadata().connectLocalSolr(connectWithinMs);
this.index.connectUrlDb(this.useTailCache, this.exceed134217727); this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
} }
@ -1201,7 +1201,7 @@ public final class Switchboard extends serverSwitch
ShardSelection.Method.MODULO_HOST_MD5, ShardSelection.Method.MODULO_HOST_MD5,
10000, true); 10000, true);
solr.setCommitWithinMs(connectWithinMs); solr.setCommitWithinMs(connectWithinMs);
this.index.connectRemoteSolr(solr); this.index.urlMetadata().connectRemoteSolr(solr);
} catch ( final IOException e ) { } catch ( final IOException e ) {
Log.logException(e); Log.logException(e);
} }

2
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -80,7 +80,7 @@ public class DocumentIndex extends Segment {
false, // useTailCache false, // useTailCache
false // exceed134217727 false // exceed134217727
); );
super.connectLocalSolr(1000); super.urlMetadata().connectLocalSolr(1000);
final int cores = Runtime.getRuntime().availableProcessors() + 1; final int cores = Runtime.getRuntime().availableProcessors() + 1;
this.callback = callback; this.callback = callback;
this.queue = new LinkedBlockingQueue<DigestURI>(cores * 300); this.queue = new LinkedBlockingQueue<DigestURI>(cores * 300);

56
source/net/yacy/search/index/Segment.java
Unescape View File

@ -39,7 +39,6 @@ import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.ByteOrder;
import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.services.federated.solr.SolrConnector;
import net.yacy.cora.services.federated.solr.SolrDoc; import net.yacy.cora.services.federated.solr.SolrDoc;
import net.yacy.cora.services.federated.yacy.CacheStrategy; import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.cora.storage.HandleSet; import net.yacy.cora.storage.HandleSet;
@ -162,61 +161,10 @@ public class Segment {
this.urlCitationIndex = null; this.urlCitationIndex = null;
} }
public boolean connectedUrlDb() {
return this.urlMetadata.connectedUrlDb();
}
public void connectUrlDb(final boolean useTailCache, final boolean exceed134217727) { public void connectUrlDb(final boolean useTailCache, final boolean exceed134217727) {
this.urlMetadata.connectUrlDb(UrlDbName, useTailCache, exceed134217727); this.urlMetadata.connectUrlDb(UrlDbName, useTailCache, exceed134217727);
} }
public void disconnectUrlDb() {
this.urlMetadata.disconnectUrlDb();
}
public boolean connectedSolr() {
return this.urlMetadata.connectedSolr();
}
public boolean connectedRemoteSolr() {
return this.urlMetadata.connectedRemoteSolr();
}
public void connectRemoteSolr(final SolrConnector solr) {
this.urlMetadata.connectRemoteSolr(solr);
}
public void disconnectRemoteSolr() {
this.urlMetadata.disconnectRemoteSolr();
}
public boolean connectedLocalSolr() {
return this.urlMetadata.connectedLocalSolr();
}
public void connectLocalSolr(final int connectWithin) throws IOException {
this.urlMetadata.connectLocalSolr(connectWithin);
}
public void disconnectLocalSolr() {
this.urlMetadata.disconnectLocalSolr();
}
public SolrConnector getSolr() {
return this.urlMetadata.getSolr();
}
public SolrConfiguration getSolrScheme() {
return this.urlMetadata.getSolrScheme();
}
public SolrConnector getRemoteSolr() {
return this.urlMetadata.getRemoteSolr();
}
public SolrConnector getLocalSolr() {
return this.urlMetadata.getLocalSolr();
}
public MetadataRepository urlMetadata() { public MetadataRepository urlMetadata() {
return this.urlMetadata; return this.urlMetadata;
} }
@ -455,10 +403,10 @@ public class Segment {
// STORE TO SOLR // STORE TO SOLR
// we do not store the data in metadatadb any more if a solr is connected // we do not store the data in metadatadb any more if a solr is connected
if (this.connectedSolr()) { if (this.urlMetadata.connectedSolr()) {
try { try {
SolrDoc solrDoc = this.urlMetadata.getSolrScheme().yacy2solr(id, responseHeader, document, metadata); SolrDoc solrDoc = this.urlMetadata.getSolrScheme().yacy2solr(id, responseHeader, document, metadata);
this.getSolr().add(solrDoc); this.urlMetadata.getSolr().add(solrDoc);
} catch ( final IOException e ) { } catch ( final IOException e ) {
Log.logWarning("SOLR", "failed to send " + urlNormalform + " to solr: " + e.getMessage()); Log.logWarning("SOLR", "failed to send " + urlNormalform + " to solr: " + e.getMessage());
} }

2
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -208,7 +208,7 @@ public final class RWIProcess extends Thread
String solrQuery = RWIProcess.this.query.solrQuery(); String solrQuery = RWIProcess.this.query.solrQuery();
try { try {
ReferenceContainer<WordReference> wr = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, null); ReferenceContainer<WordReference> wr = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, null);
SolrDocumentList sdl = RWIProcess.this.query.getSegment().getSolr().query(solrQuery, 0, 20); SolrDocumentList sdl = RWIProcess.this.query.getSegment().urlMetadata().getSolr().query(solrQuery, 0, 20);
for (SolrDocument d : sdl) { for (SolrDocument d : sdl) {
try {wr.add(new WordReferenceVars(d));} catch (SpaceExceededException e) {} try {wr.add(new WordReferenceVars(d));} catch (SpaceExceededException e) {}
} }

Write Preview
Loading…
Cancel
Save