boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixes for problems with remote solr and non-activated webgraph index

Browse Source
pull/1/head
orbiter 12 years ago
parent 336f86394c
commit 6fb2811e68
8 changed files with 35 additions and 33 deletions
Show Stats Download Patch File Download Diff File

2
htroot/IndexDeletion_p.java
Unescape View File

@ -187,7 +187,7 @@ public class IndexDeletion_p {
} else {
try {
defaultConnector.deleteByQuery(collection1Query);
webgraphConnector.deleteByQuery(webgraphQuery);
if (webgraphConnector != null) webgraphConnector.deleteByQuery(webgraphQuery);
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs older than " + timedelete_number + " " + timedelete_unit);
} catch (final IOException e) {
}

2
htroot/IndexFederated_p.java
Unescape View File

@ -84,7 +84,7 @@ public class IndexFederated_p {
}
boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH);
sb.index.writeWebgraph(webgraph);
sb.index.fulltext().writeWebgraph(webgraph);
env.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, webgraph);
boolean jena = post.getBoolean(SwitchboardConstants.CORE_SERVICE_JENA);

2
source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java
Unescape View File

@ -181,7 +181,7 @@ public class OpenSearchConnector {
if (sb == null) {
return false;
}
final SolrConnector connector = sb.index.fulltext().getWebgraphConnector();
final SolrConnector connector = sb.index.fulltext().writeToWebgraph() ? null : sb.index.fulltext().getWebgraphConnector();
// check if needed Solr fields are available (selected)
if (connector == null) {
ConcurrentLog.severe("OpenSearchConnector.Discover", "Error on connecting to embedded Solr webgraph index");

4
source/net/yacy/search/Switchboard.java
Unescape View File

@ -489,7 +489,7 @@ public final class Switchboard extends serverSwitch {
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
try {this.index.fulltext().connectLocalSolr();} catch (final IOException e) {ConcurrentLog.logException(e);}
}
this.index.writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false));
this.index.fulltext().writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false));
// set up the solr interface
final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr");
@ -1322,7 +1322,7 @@ public final class Switchboard extends serverSwitch {
this.index.fulltext().connectLocalSolr();
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
}
this.index.writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false));
this.index.fulltext().writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false));
// set up the solr interface
final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr");

2
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -77,7 +77,7 @@ public class DocumentIndex extends Segment {
false // exceed134217727
);
super.fulltext().connectLocalSolr();
super.writeWebgraph(true);
super.fulltext().writeWebgraph(true);
this.callback = callback;
this.queue = new LinkedBlockingQueue<DigestURI>(WorkflowProcessor.availableCPU * 300);
this.worker = new Worker[WorkflowProcessor.availableCPU];

38
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -89,14 +89,15 @@ public final class Fulltext {
private static final String SOLR_OLD_PATH[] = new String[]{"solr_36"};
// class objects
private final File segmentPath;
private Index urlIndexFile;
private Export exportthread; // will have a export thread assigned if exporter is running
private String tablename;
private ArrayList<HostStat> statsDump;
private InstanceMirror solrInstances;
private final File segmentPath;
private Index urlIndexFile;
private Export exportthread; // will have a export thread assigned if exporter is running
private String tablename;
private ArrayList<HostStat> statsDump;
private InstanceMirror solrInstances;
private final CollectionConfiguration collectionConfiguration;
private final WebgraphConfiguration webgraphConfiguration;
private boolean writeWebgraph;
protected Fulltext(final File segmentPath, final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) {
this.segmentPath = segmentPath;
@ -107,6 +108,15 @@ public final class Fulltext {
this.solrInstances = new InstanceMirror();
this.collectionConfiguration = collectionConfiguration;
this.webgraphConfiguration = webgraphConfiguration;
this.writeWebgraph = false;
}
public void writeWebgraph(boolean check) {
this.writeWebgraph = check;
}
public boolean writeToWebgraph() {
return this.writeWebgraph;
}
/**
@ -211,6 +221,7 @@ public final class Fulltext {
}
public SolrConnector getWebgraphConnector() {
if (!this.writeWebgraph) return null;
synchronized (this.solrInstances) {
return this.solrInstances.getMirrorConnector(WebgraphSchema.CORE_NAME);
}
@ -276,7 +287,7 @@ public final class Fulltext {
* @return
*/
public long webgraphSize() {
return this.getWebgraphConnector().getSize();
return this.writeWebgraph ? this.getWebgraphConnector().getSize() : 0;
}
public void close() {
@ -294,7 +305,7 @@ public final class Fulltext {
if (lastCommit + 10000 > t) return;
lastCommit = t;
getDefaultConnector().commit(softCommit);
getWebgraphConnector().commit(softCommit);
if (this.writeWebgraph) getWebgraphConnector().commit(softCommit);
}
public Date getLoadDate(final String urlHash) {
@ -393,6 +404,7 @@ public final class Fulltext {
}
public void putEdges(final Collection<SolrInputDocument> edges) throws IOException {
if (!this.writeToWebgraph()) return;
if (edges == null || edges.size() == 0) return;
try {
this.getWebgraphConnector().add(edges);
@ -443,7 +455,7 @@ public final class Fulltext {
// delete in solr
try {Fulltext.this.getDefaultConnector().deleteByQuery(collection1Query);} catch (final IOException e) {}
try {Fulltext.this.getWebgraphConnector().deleteByQuery(webgraphQuery);} catch (final IOException e) {}
if (this.writeWebgraph) try {Fulltext.this.getWebgraphConnector().deleteByQuery(webgraphQuery);} catch (final IOException e) {}
// delete in old metadata structure
if (Fulltext.this.urlIndexFile != null) {
@ -493,7 +505,7 @@ public final class Fulltext {
// delete in solr
try {Fulltext.this.getDefaultConnector().deleteByQuery(collectionQuery);} catch (final IOException e) {}
try {Fulltext.this.getWebgraphConnector().deleteByQuery(webgraphQuery);} catch (final IOException e) {}
if (this.writeWebgraph) try {Fulltext.this.getWebgraphConnector().deleteByQuery(webgraphQuery);} catch (final IOException e) {}
// finally remove the line with statistics
if (Fulltext.this.statsDump != null) {
@ -546,7 +558,7 @@ public final class Fulltext {
if (deleteIDs == null || deleteIDs.size() == 0) return;
try {
this.getDefaultConnector().deleteByIds(deleteIDs);
this.getWebgraphConnector().deleteByIds(deleteIDs);
if (this.writeWebgraph) this.getWebgraphConnector().deleteByIds(deleteIDs);
} catch (final Throwable e) {
ConcurrentLog.logException(e);
}
@ -563,7 +575,7 @@ public final class Fulltext {
try {
String id = ASCII.String(urlHash);
this.getDefaultConnector().deleteById(id);
this.getWebgraphConnector().deleteById(id);
if (this.writeWebgraph) this.getWebgraphConnector().deleteById(id);
} catch (final Throwable e) {
ConcurrentLog.logException(e);
}
@ -709,7 +721,7 @@ public final class Fulltext {
public void optimize(final int size) {
if (size < 1) return;
getDefaultConnector().optimize(size);
getWebgraphConnector().optimize(size);
if (this.writeWebgraph) getWebgraphConnector().optimize(size);
}
/**

17
source/net/yacy/search/index/Segment.java
Unescape View File

@ -74,7 +74,6 @@ import net.yacy.kelondro.rwi.ReferenceFactory;
import net.yacy.kelondro.util.Bitfield;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.ISO639;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.workflow.WorkflowProcessor;
import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.StorageQueueEntry;
@ -111,12 +110,11 @@ public class Segment {
public static final ReferenceFactory<CitationReference> citationReferenceFactory = new CitationReferenceFactory();
public static final ByteOrder wordOrder = Base64Order.enhancedCoder;
private final ConcurrentLog log;
private final ConcurrentLog log;
private final File segmentPath;
protected final Fulltext fulltext;
protected IndexCell<WordReference> termIndex;
protected IndexCell<CitationReference> urlCitationIndex;
protected boolean writeWebgraph;
private WorkflowProcessor<StorageQueueEntry> indexingPutDocumentProcessor;
/**
@ -135,7 +133,6 @@ public class Segment {
this.fulltext = new Fulltext(segmentPath, collectionConfiguration, webgraphConfiguration);
this.termIndex = null;
this.urlCitationIndex = null;
this.writeWebgraph = false;
this.indexingPutDocumentProcessor = new WorkflowProcessor<StorageQueueEntry>(
"putDocument",
@ -148,14 +145,6 @@ public class Segment {
1);
}
public void writeWebgraph(boolean check) {
this.writeWebgraph = check;
}
public boolean writeToWebgraph() {
return this.writeWebgraph;
}
public boolean connectedRWI() {
return this.termIndex != null;
}
@ -317,7 +306,7 @@ public class Segment {
this.externalHosts = new RowHandleSet(6, Base64Order.enhancedCoder, 0);
this.internalIDs = new RowHandleSet(12, Base64Order.enhancedCoder, 0);
this.externalIDs = new RowHandleSet(12, Base64Order.enhancedCoder, 0);
if (writeToWebgraph()) {
if (Segment.this.fulltext.writeToWebgraph()) {
// reqd the references from the webgraph
SolrConnector webgraph = Segment.this.fulltext.getWebgraphConnector();
webgraph.commit(true);
@ -670,7 +659,7 @@ public class Segment {
// STORE TO SOLR
String error = null;
this.putDocumentInQueue(vector);
if (this.writeWebgraph) {
if (this.fulltext.writeToWebgraph()) {
tryloop: for (int i = 0; i < 20; i++) {
try {
error = null;

1
source/net/yacy/search/schema/WebgraphConfiguration.java
Unescape View File

@ -279,6 +279,7 @@ public class WebgraphConfiguration extends SchemaConfiguration implements Serial
public int postprocessing(Segment segment) {
if (!this.contains(WebgraphSchema.process_sxt)) return 0;
if (!segment.connectedCitation()) return 0;
if (!segment.fulltext().writeToWebgraph()) return 0;
SolrConnector connector = segment.fulltext().getWebgraphConnector();
// that means we must search for those entries.
connector.commit(true); // make sure that we have latest information that can be found

Write Preview
Loading…
Cancel
Save