boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added flags in IndexFederated_p.html to switch on or off the webgraph

Browse Source 
index (new solr core webgraph) .. this is now off by default
- completely redesigned this servlet
- added description how to attach a remote solr
- adjusted naming of servlet and menues
- moved 'lazy initialization' attribut from IndexSchema to
IndexFederated (this is a general option) back again.
pull/1/head
Michael Peter Christen 12 years ago
parent 461d46101d
commit 56d5946a59
13 changed files with 163 additions and 78 deletions
Show Stats Download Patch File Download Diff File

2
defaults/yacy.init
Unescape View File

@ -1069,6 +1069,8 @@ federated.service.solr.indexing.lazy = true
core.service.fulltext = true
core.service.rwi.tmp = true
core.service.citation.tmp = true
core.service.webgraph.tmp = false
core.service.jena.tmp = false
# RDF triplestore settings
triplestore.persistent = true

4
htroot/IndexControlURLs_p.html
Unescape View File

@ -3,7 +3,7 @@
<html xmlns="http://www.w3.org/1999/xhtml">
#(reload)#::<meta http-equiv="REFRESH" content="5; url=/IndexControlURLs_p.html">#(/reload)#
<head>
<title>YaCy '#[clientname]#': URL References Administration</title>
<title>YaCy '#[clientname]#': URL Database Administration</title>
#%env/templates/metas.template%#
<script type="text/javascript">
//<![CDATA[
@ -62,7 +62,7 @@ function updatepage(str) {
<body id="IndexControl">
#%env/templates/header.template%#
#%env/templates/submenuIndexControl.template%#
<h2>URL References Administration</h2>
<h2>URL Database Administration</h2>
<p>The local index currently contains #[ucount]# URL references</p>
<form action="IndexControlURLs_p.html" id="searchform" method="post" enctype="multipart/form-data" accept-charset="UTF-8" onkeyup="xmlhttpPost(); return false;">
<fieldset><legend>URL Retrieval</legend>

154
htroot/IndexFederated_p.html
Unescape View File

@ -1,50 +1,49 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Remote Solr Configuration</title>
<title>YaCy '#[clientname]#': Index Sources &amp; Targets</title>
#%env/templates/metas.template%#
</head>
<body id="IndexFederated_p">
<div id="api">
<a href="/api/schema.xml">
<img src="env/grafics/api.png" width="60" height="40" alt="API" /></a>
<span>The solr schema can also be retrieved as xml here. Click the API icon to see the xml. Just copy this xml to solr/conf/schema.xml to configure solr.</span>
</div>
#%env/templates/header.template%#
#%env/templates/submenuIndexControl.template%#
<h2>Remote Solr Configuration</h2>
<h2>Index Sources &amp; Targets</h2>
<p>
YaCy supports multiple index storage locations. At this time only the YaCy-internal search index can be used for the YaCy search interface
A Solr index storage location is optional. The local index storage location can be disabled.
YaCy supports multiple index storage locations.
As an internal indexing database a deep-embedded multi-core Solr is used and it is possible to attach also a remote Solr.
</p>
<form action="IndexFederated_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<form id="config" action="IndexFederated_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<fieldset>
<legend>
Local Search Index
<label for="p2p">Solr Search Index</label>
</legend>
This is a switchboard for the usage of embedded metadata to embedded solr.
The rwi index is necessary for index transmission and shall be switched off in future portalmode configurations.
<dl>
<dt><input type="checkbox" name="core.service.fulltext" id="core.service.fulltext" #(core.service.fulltext.checked)#:: checked="checked"#(/core.service.fulltext.checked)# /></dt><dd>url metadata and embedded solr fulltext search index, interface at: <a href="/solr/select?q=*:*&start=0&rows=10" target="_blank">/solr/select?q=*:*&amp;start=0&amp;rows=10</a>
#(migrateUrlDbtoSolr)#:: <input type="button" class="submitready" onclick="window.location = 'migrateurldb_p.html';" value="migrate old index" />#(/migrateUrlDbtoSolr)# </dd>
<dt><input type="checkbox" name="core.service.rwi.tmp" id="core.service.rwi" #(core.service.rwi.tmp.checked)#:: checked="checked"#(/core.service.rwi.tmp.checked)# /></dt><dd>embedded 'classic' rwi index</dd>
<dt><input type="checkbox" name="core.service.citation.tmp" id="core.service.citation" #(core.service.citation.tmp.checked)#:: checked="checked"#(/core.service.citation.tmp.checked)# /></dt><dd>embedded citation reference index (link structure, used for ranking)</dd>
<dt></dt><dd><input type="submit" name="set" value="Set" /></dd>
</dl>
</fieldset>
<fieldset>
<legend>
<input type="checkbox" name="solr.indexing.solrremote" id="solr.indexing.solrremote" #(solr.indexing.solrremote.checked)#:: checked="checked"#(/solr.indexing.solrremote.checked)# />
<label for="p2p">Remote Solr Search Index</label>
</legend>
<div align="left">
Solr stores the main search index. It is the home of two cores, the default 'collection1' core for documents and the 'webgraph' core for a web structure graph. Detailed information about the used Solr fields can be edited in the <a href="/IndexSchema_p.html">Schema Editor</a>.
<dl>
<dt class="TableCellDark"><input type="checkbox" name="solr.indexing.lazy" id="solr.indexing.lazy" #(solr.indexing.lazy.checked)#:: checked="checked"#(/solr.indexing.lazy.checked)# /></dt>
<dd>Lazy Value Initialization. If checked, only non-zero values and non-empty strings are written to Solr fields.</dd>
<dt>&nbsp;</dt><dd>&nbsp;</dd>
<dt>Use deep-embedded local Solr</dt>
<dd><input type="checkbox" name="core.service.fulltext" id="core_service_fulltext" #(core.service.fulltext.checked)#:: checked="checked"#(/core.service.fulltext.checked)# onclick="if(!document.getElementById('config').core_service_fulltext.checked) {document.getElementById('config').solr_indexing_solrremote.checked = true;}"/><br/>
This will write the YaCy-embedded Solr index which stored within the YaCy DATA directory.<br/>
The Solr native search interface is accessible at<br/>
<a href="/solr/select?q=*:*&start=0&rows=3&core=collection1">/solr/select?q=*:*&start=0&rows=3&core=collection1</a>
for the default search index (core: collection1) and at<br/>
<a href="/solr/select?q=*:*&start=0&rows=3&core=webgraph">/solr/select?q=*:*&start=0&rows=3&core=webgraph</a> for the webgraph core.<br/>
If you switch off this index, a remote Solr must be activated.</dd>
<dt><input type="submit" name="set" value="Set" /></dt><dd></dd>
<dt>Use remote Solr server(s)</dt>
<dd><input type="checkbox" name="solr.indexing.solrremote" id="solr_indexing_solrremote" #(solr.indexing.solrremote.checked)#:: checked="checked"#(/solr.indexing.solrremote.checked)# onclick="if(!document.getElementById('config').solr_indexing_solrremote.checked) {document.getElementById('config').core_service_fulltext.checked = true;}"/><br/>
Here you can define a single or a set of remote solr servers. If both, an internal and an external Solr is used, then both are mirrored.
That means, every write request goes to internal and external Solr, but a read request goes only to the internal index.
Only if the internal index does not give any result on a search request, also the remote is requested.</dd>
#(table)#::
<dt class="TableCellDark">Solr Hosts</dt>
<dd>
<dt class="TableCellDark">&nbsp;</dt>
<dd>Solr Hosts<br/>
<div>
<table class="sortable" border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
@ -61,18 +60,89 @@
</div>
</dd>
#(/table)#
<dt class="TableCellDark">Solr URL(s)</dt>
<dd><textarea rows="2" cols="80" name="solr.indexing.url" id="solr.indexing.url"/>#[solr.indexing.url]#</textarea><br/>
You can set one or more Solr targets here which are accessed as a shard. For several targets, list them using a ',' (comma) as separator.</dd>
<!-- will be moved here again...
<dt class="TableCellDark">Lazy Value Initialization</dt>
<dd><input type="checkbox" name="solr.indexing.lazy" id="solr.indexing.lazy" #(solr.indexing.lazy.checked)#:: checked="checked"#(/solr.indexing.lazy.checked)# /> (if checked, only non-zero values and non-empty strings are written)</dd>
-->
<dt class="TableCellDark">Sharding Method</dt>
<dd><input type="text" size="50" maxlength="50" value="#[solr.indexing.sharding]#" name="solr.indexing.sharding" id="solr.indexing.sharding" disabled="disabled"/></dd>
<dt></dt><dd><input type="submit" name="set" value="Set" /></dd>
<dt class="TableCellDark"></dt>
<dd>Solr URL(s)<br/><textarea rows="2" cols="80" name="solr.indexing.url" id="solr.indexing.url"/>#[solr.indexing.url]#</textarea><br/>
You can set one or more Solr targets here which are accessed as a shard. For several targets, list them using a ',' (comma) as separator.
The set of remote targets are used as shard of a complete index. The host part of the url is used as key for a hash function which selects one of the shards (one of your remote servers).
When a search request is made, all servers are accessed synchronously and the result is combined.</dd>
<dt class="TableCellDark"></dt>
<dd>Sharding Method<br/><input type="text" size="50" maxlength="50" value="#[solr.indexing.sharding]#" name="solr.indexing.sharding" id="solr.indexing.sharding" disabled="disabled"/></dd>
<dt><input type="submit" name="set" value="Set" /></dt><dd></dd>
<dt></dt><dd>
An <strong>external Solr installation</strong> is easily done following these steps (by example for Solr 4.1.0):
<ul>
<li>Download solr-4.1.0.tgz from http://lucene.apache.org/solr/</li>
<li>Decompress solr-4.1.0.tgz (with 'tar xfz solr-4.1.0.tgz') and put solr-4.1.0 into ~/</li>
<li>Consider that YaCy is already running and stored in ~/yacy/</li>
<li>To configure the multi-core configuration of YaCy, execute:<br>
<pre>mkdir ~/solr-4.1.0/example/solr/webgraph
cp -R ~/solr-4.1.0/example/solr/collection1/conf ~/solr-4.1.0/example/solr/webgraph/conf
~/yacy/bin/apicat.sh /api/schema.xml?core=collection1 > ~/solr-4.1.0/example/solr/collection1/conf/schema.xml
~/yacy/bin/apicat.sh /api/schema.xml?core=webgraph > ~/solr-4.1.0/example/solr/webgraph/conf/schema.xml</pre></li>
<li>edit ~/solr-4.1.0/example/solr/solr.xml and put in the following content:<br/>
<pre>&lt;?xml version="1.0" encoding="UTF-8" ?&gt;
&lt;solr persistent="true"&gt;
&lt;cores adminPath="/admin/cores" defaultCoreName="collection1"&gt;
&lt;core name="collection1" instanceDir="collection1" /&gt;
&lt;core name="webgraph" instanceDir="webgraph" /&gt;
&lt;/cores&gt;
&lt;/solr&gt;</pre></li>
<li>Finally, start the external Solr with:<br>
<pre>cd ~/solr-4.1.0/example/ && java -jar start.jar</pre></li>
<li>open <a href="http://localhost:8983/solr/">http://localhost:8983/solr/</a> to visit Solr's administration console.</li>
</ul>
</dd>
</dl>
</fieldset>
<fieldset>
<legend>
Web Structure Index
</legend>
The web structure index is used for host browsing (to discover the internal file/folder structure), ranking (counting the number of references) and file search (there are about fourty times more links from loaded pages as in documents of the main search index).
<dl>
<dt><input type="checkbox" name="core.service.citation.tmp" id="core_service_citation" #(core.service.citation.tmp.checked)#:: checked="checked"#(/core.service.citation.tmp.checked)# /></dt>
<dd>use citation reference index (lightweight and fast)</dd>
<dt><input type="checkbox" name="core.service.webgraph.tmp" id="core_service_webgraph" #(core.service.webgraph.tmp.checked)#:: checked="checked"#(/core.service.webgraph.tmp.checked)# /></dt>
<dd>use webgraph search index (rich information in second Solr core)</dd>
<dt><input type="submit" name="set" value="Set" /></dt><dd></dd>
</dl>
</fieldset>
<!--
<fieldset>
<legend>
Content Semantics
</legend>
YaCy uses a Apache Jena instance to host metadata about web pages.
<dl>
<dt><input type="checkbox" name="core.service.jena.tmp" id="core_service_jena" #(core.service.jena.tmp.checked)#:: checked="checked"#(/core.service.jena.tmp.checked)# /></dt>
<dd>write document metadata to the Jena index.</dd>
<dt><input type="submit" name="set" value="Set" /></dt><dd></dd>
</dl>
</fieldset>
-->
<fieldset>
<legend>
Peer-to-Peer Operation
</legend>
The 'RWI' (Reverse Word Index) is necessary for index transmission in distributed mode. For portal or intranet mode this must be switched off.
<dl>
<dt><input type="checkbox" name="core.service.rwi.tmp" id="core_service_rwi" #(core.service.rwi.tmp.checked)#:: checked="checked"#(/core.service.rwi.tmp.checked)# /></dt>
<dd>support peer-to-peer index transmission (DHT RWI index)</dd>
<dt><input type="submit" name="set" value="Set" /></dt><dd></dd>
</dl>
</div>
</fieldset>
</form>

21
htroot/IndexFederated_p.java
Unescape View File

@ -42,14 +42,13 @@ import net.yacy.server.serverSwitch;
public class IndexFederated_p {
@SuppressWarnings("deprecation")
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
if (post != null && post.containsKey("set")) {
// yacy
//yacy
boolean post_core_rwi = post.getBoolean(SwitchboardConstants.CORE_SERVICE_RWI);
final boolean previous_core_rwi = sb.index.connectedRWI() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, post_core_rwi);
@ -85,6 +84,13 @@ public class IndexFederated_p {
try { sb.index.fulltext().connectLocalSolr(); } catch (IOException e) { Log.logException(e); }
}
boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH);
sb.index.writeWebgraph(webgraph);
env.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, webgraph);
boolean jena = post.getBoolean(SwitchboardConstants.CORE_SERVICE_JENA);
env.setConfig(SwitchboardConstants.CORE_SERVICE_JENA, jena);
// solr
final boolean solrRemoteWasOn = sb.index.fulltext().connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true);
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
@ -140,6 +146,9 @@ public class IndexFederated_p {
} catch (SolrException e) {
Log.logSevere("IndexFederated_p", "change of solr connection failed", e);
}
boolean lazy = post.getBoolean("solr.indexing.lazy");
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
}
// show solr host table
@ -162,14 +171,14 @@ public class IndexFederated_p {
prop.put(SwitchboardConstants.CORE_SERVICE_FULLTEXT + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_RWI + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_JENA + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_JENA, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"));
if ((sb.index.fulltext().connectedURLDb())) {
prop.put("migrateUrlDbtoSolr", 1);
} else prop.put("migrateUrlDbtoSolr", 0);
prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
// return rewrite properties
return prop;
}

4
htroot/IndexSchema_p.html
Unescape View File

@ -45,10 +45,6 @@
</tr>
#{/schema}#
</table>
<dl>
<dt class="TableCellDark">Lazy Value Initialization</dt>
<dd><input type="checkbox" name="lazy" id="lazy" #(lazy.checked)#:: checked="checked"#(/lazy.checked)# /> (if checked, only non-zero values and non-empty strings are written)</dd>
</dl>
</div>
</fieldset>
<input type="submit" name="set" value="Set" />

9
htroot/IndexSchema_p.java
Unescape View File

@ -25,7 +25,6 @@ import net.yacy.cora.federate.solr.SchemaConfiguration;
import net.yacy.cora.federate.solr.SchemaDeclaration;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
import net.yacy.server.serverObjects;
@ -72,11 +71,7 @@ public class IndexSchema_p {
sb.index.fulltext().getWebgraphConfiguration().commit();
modified = false;
} catch (IOException ex) {}
}
boolean lazy = post.getBoolean("lazy");
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
}
}
int c = 0;
@ -100,8 +95,6 @@ public class IndexSchema_p {
prop.put("cores", 2);
prop.put("core", schemaName);
prop.put("lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
// return rewrite properties
return prop;
}

4
htroot/env/base.css vendored
Unescape View File

@ -351,7 +351,7 @@ div.SubMenugroup{
div.content {
position:absolute;
top:3em;
left:12em;
left:13em;
right:10px;
margin:0;
}
@ -871,7 +871,7 @@ ul#menu {
position:absolute;
top:3em;
left:0;
width:11em;
width:12em;
overflow:hidden;
}
ul#menu a.MenuItemLink {

2
htroot/env/templates/header.template vendored
Unescape View File

@ -13,7 +13,7 @@
<!--<li><a href="/yacysearch_location.html" class="MenuItemLink">Location Search</a></li>-->
<li><a href="/solr/select?q=*:*&start=0&rows=3&core=collection1" class="MenuItemLink">Solr Default Core</a></li>
<li><a href="/solr/select?q=*:*&start=0&rows=3&core=webgraph" class="MenuItemLink">Solr Webgraph Core</a></li>
<li><a href="/gsa/search?q=www&size=3" class="MenuItemLink">Google Search API</a></li>
<li><a href="/gsa/search?q=www&size=3" class="MenuItemLink">Google Appliance API</a></li>
<!--<li><a href="/yacy/ui/" accesskey="s" class="MenuItemLink">Rich Client Search</a></li>-->
<li><a href="/compare_yacy.html?display=1" class="MenuItemLink">Compare Search</a></li>
<li><a href="/ViewFile.html" class="MenuItemLink">URL Viewer</a></li>

4
htroot/env/templates/submenuIndexControl.template vendored
Unescape View File

@ -1,9 +1,9 @@
<div class="SubMenu">
<h3>Index Administration</h3>
<ul class="SubMenu">
<li><a href="/IndexControlURLs_p.html" class="MenuItemLink lock">Database Administration</a></li>
<li><a href="/IndexControlURLs_p.html" class="MenuItemLink lock">URL Database Administration</a></li>
<li><a href="/IndexFederated_p.html" class="MenuItemLink lock">Index Sources &amp; Targets</a></li>
<li><a href="/IndexSchema_p.html" class="MenuItemLink lock">Solr Schema Editor</a></li>
<li><a href="/IndexFederated_p.html" class="MenuItemLink lock">Remote Solr Configuration</a></li>
#(p2p)#::<li><a href="/IndexControlRWIs_p.html" class="MenuItemLink lock">Reverse Word Index Administration</a></li>#(/p2p)#
<!--<li><a href="/IndexControlCleaner_p.html" class="MenuItemLink lock">Index Cleaner</a></li>-->
<li><a href="/ConfigHTCache_p.html" class="MenuItemLink lock">Web Cache</a></li>

2
source/net/yacy/search/Switchboard.java
Unescape View File

@ -455,6 +455,7 @@ public final class Switchboard extends serverSwitch {
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
this.index.fulltext().connectLocalSolr();
}
this.index.writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false));
// set up the solr interface
final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr");
@ -1305,6 +1306,7 @@ public final class Switchboard extends serverSwitch {
this.index.fulltext().connectLocalSolr();
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
}
this.index.writeWebgraph(this.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false));
// set up the solr interface
final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr");

2
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -302,6 +302,8 @@ public final class SwitchboardConstants {
public static final String CORE_SERVICE_FULLTEXT = "core.service.fulltext";
public static final String CORE_SERVICE_RWI = "core.service.rwi.tmp";
public static final String CORE_SERVICE_CITATION = "core.service.citation.tmp";
public static final String CORE_SERVICE_WEBGRAPH = "core.service.webgraph.tmp";
public static final String CORE_SERVICE_JENA = "core.service.jena.tmp";
/**
* <p><code>public static final String <strong>CRAWLER_THREADS_ACTIVE_MAX</strong> = "crawler.MaxActiveThreads"</code></p>

1
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -76,6 +76,7 @@ public class DocumentIndex extends Segment {
false // exceed134217727
);
super.fulltext().connectLocalSolr();
super.writeWebgraph(true);
final int cores = Runtime.getRuntime().availableProcessors() + 1;
this.callback = callback;
this.queue = new LinkedBlockingQueue<DigestURI>(cores * 300);

32
source/net/yacy/search/index/Segment.java
Unescape View File

@ -113,6 +113,7 @@ public class Segment {
protected final Fulltext fulltext;
protected IndexCell<WordReference> termIndex;
protected IndexCell<CitationReference> urlCitationIndex;
protected boolean writeWebgraph;
/**
* create a new Segment
@ -128,8 +129,15 @@ public class Segment {
// create LURL-db
this.fulltext = new Fulltext(segmentPath, collectionConfiguration, webgraphConfiguration);
this.termIndex = null;
this.urlCitationIndex = null;
this.writeWebgraph = false;
}
public void writeWebgraph(boolean check) {
this.writeWebgraph = check;
}
public boolean connectedRWI() {
return this.termIndex != null;
}
@ -477,17 +485,19 @@ public class Segment {
continue tryloop;
}
}
tryloop: for (int i = 0; i < 20; i++) {
try {
error = null;
this.fulltext.putEdges(vector.getWebgraphDocuments());
break tryloop;
} catch ( final IOException e ) {
error = "failed to send " + urlNormalform + " to solr";
Log.logWarning("SOLR", error + e.getMessage());
if (i == 10) this.fulltext.commit(false);
try {Thread.sleep(1000);} catch (InterruptedException e1) {}
continue tryloop;
if (this.writeWebgraph) {
tryloop: for (int i = 0; i < 20; i++) {
try {
error = null;
this.fulltext.putEdges(vector.getWebgraphDocuments());
break tryloop;
} catch ( final IOException e ) {
error = "failed to send " + urlNormalform + " to solr";
Log.logWarning("SOLR", error + e.getMessage());
if (i == 10) this.fulltext.commit(false);
try {Thread.sleep(1000);} catch (InterruptedException e1) {}
continue tryloop;
}
}
}
if (error != null) {

Write Preview
Loading…
Cancel
Save