- migrates all entries in old urldb Metadata coordinate (lat / lon) NumberFormatException still relative often (see excerpt below), - added try/catch for URIMetadataRow (seems not to be needed in URIMetaDataNode, as Solr internally checks for number format) - removed possible typ conversion for lat() / lon() comparison with 0.0f, changed to 0.0 (leaving it to the compiler/optimizer to choose number format) current log excerpt for NumberFormatException: W 2013/01/14 00:10:07 StackTrace For input string: "-" java.lang.NumberFormatException: For input string: "-" at sun.misc.FloatingDecimal.readJavaFormatString(Unknown Source) at java.lang.Double.parseDouble(Unknown Source) at net.yacy.kelondro.data.meta.URIMetadataRow$Components.lon(URIMetadataRow.java:525) at net.yacy.kelondro.data.meta.URIMetadataRow.lon(URIMetadataRow.java:279) at net.yacy.search.index.SolrConfiguration.metadata2solr(SolrConfiguration.java:277) at net.yacy.search.index.Fulltext.putMetadata(Fulltext.java:329) at transferURL.respond(transferURL.java:152) ... Caused by: java.lang.NumberFormatException: For input string: "-" at sun.misc.FloatingDecimal.readJavaFormatString(Unknown Source) at java.lang.Double.parseDouble(Unknown Source) at net.yacy.kelondro.data.meta.URIMetadataRow$Components.lon(URIMetadataRow.java:525) at net.yacy.kelondro.data.meta.URIMetadataRow.lon(URIMetadataRow.java:279) at net.yacy.search.index.SolrConfiguration.metadata2solr(SolrConfiguration.java:277) at net.yacy.search.index.Fulltext.putMetadata(Fulltext.java:329) at transferURL.respond(transferURL.java:152)pull/1/head
reger
12 years ago
parent
3b6e08b49f
commit
3897bb4409
@ -0,0 +1,36 @@
|
|||||||
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<title>Migrate URLdb</title>
|
||||||
|
#%env/templates/metas.template%#
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
#%env/templates/header.template%#
|
||||||
|
#%env/templates/simpleheader.template%#
|
||||||
|
|
||||||
|
<h2>Migrate URLdb to embedded Solr Index</h2>
|
||||||
|
|
||||||
|
<p>Convert old meta data (urldb) index to embedded Solr fulltext index.</p>
|
||||||
|
|
||||||
|
<dl>
|
||||||
|
<dd>
|
||||||
|
<p>A low priority background job has been started which reads the old index, adds it to Solr and deletes the entry from the old index.</p>
|
||||||
|
<p>The default "slow migration" updates any entry in the old urldb index upon access (e.g. during search events).<br />
|
||||||
|
If you feel that the not accessed entries are still relevant, with this migration all entries from the old urldb index will be migrated.</p>
|
||||||
|
<p>You may refresh this page to see how many entries in the old index are left for migration</p>
|
||||||
|
<p>Hint: this background task runs until all entries are migrated or YaCy is shutdown. The migration is not automatically restarted.</p>
|
||||||
|
</dd>
|
||||||
|
</dl>
|
||||||
|
<form action="migrateurldb_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
|
||||||
|
<fieldset>
|
||||||
|
<input type="hidden" name="lastcount" value="#[lastcount]#" />
|
||||||
|
<input type="hidden" name="lasttime" value="#[lasttime]#" />
|
||||||
|
|
||||||
|
<p><b>#[count]# entries</b> in old index left to migrate.</p>
|
||||||
|
<p>For large indexes this may run for a long time (migration speed: #[speed]# entries per minute) <input type="submit" name="dorefresh" value="refresh" /></p>
|
||||||
|
</fieldset>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
#%env/templates/footer.template%#
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@ -0,0 +1,44 @@
|
|||||||
|
// migrateurldb_p.java
|
||||||
|
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.migration;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
|
||||||
|
public class migrateurldb_p {
|
||||||
|
|
||||||
|
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
final Switchboard sb = (Switchboard) env;
|
||||||
|
|
||||||
|
int cnt;
|
||||||
|
|
||||||
|
if ((cnt = migration.migrateUrldbtoSolr(sb)) > 0) {
|
||||||
|
prop.put("count", cnt);
|
||||||
|
|
||||||
|
if (post != null && post.containsKey("dorefresh")) {
|
||||||
|
int lastcount = post.getInt("lastcount", 0);
|
||||||
|
Long t = post.getLong("lasttime", 1);
|
||||||
|
|
||||||
|
Double difft = (System.currentTimeMillis() - t) / 60000.0d;
|
||||||
|
int diff = (int)((lastcount - cnt) / difft) ;
|
||||||
|
prop.put("speed", diff);
|
||||||
|
prop.put("lasttime", t);
|
||||||
|
prop.put("lastcount", lastcount);
|
||||||
|
|
||||||
|
} else {
|
||||||
|
prop.put("speed", "?");
|
||||||
|
prop.put("lastcount",cnt);
|
||||||
|
prop.put("lasttime", System.currentTimeMillis());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
prop.put("speed", "");
|
||||||
|
prop.put("count", "no urldb index available");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// return rewrite properties
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in new issue