Michael Peter Christen
2 years ago
parent
efa0425f00
commit
62d177bf59
@ -0,0 +1,56 @@
|
|||||||
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<title>YaCy '#[clientname]#': JsonList Import</title>
|
||||||
|
#%env/templates/metas.template%#
|
||||||
|
#(import)#::<meta http-equiv="REFRESH" content="10;url=IndexImportJsonList_p.html" />
|
||||||
|
<!-- the url= removes http get parameters on refresh, preventing restart of import -->
|
||||||
|
#(/import)#
|
||||||
|
</head>
|
||||||
|
<body id="IndexImportJsonList">
|
||||||
|
#%env/templates/header.template%#
|
||||||
|
#%env/templates/submenuIndexImport.template%#
|
||||||
|
<h2>JSON List Index Dump File Import</h2>
|
||||||
|
|
||||||
|
#(import)#
|
||||||
|
<p>No import thread is running, you can start a new thread here</p>
|
||||||
|
<form action="IndexImportJsonList_p.html" method="get" accept-charset="UTF-8">
|
||||||
|
<!-- no post method here, we don't want to transmit the whole file, only the path-->
|
||||||
|
<fieldset>
|
||||||
|
<legend>JsonList File Selection: select an jsonlist file (which may be gz compressed)</legend>
|
||||||
|
<p>
|
||||||
|
You can download jsonlist archives from the <a href="https://searchlab.eu" target="_blank">YaCy Searchlab</a> portal.
|
||||||
|
</p>
|
||||||
|
<dl>
|
||||||
|
<dt class="TableCellDark"><label for="file">File:</label></dt>
|
||||||
|
<dd><input name="file" id="file" type="file" value="" size="75" /></dd>
|
||||||
|
<dt></dt>
|
||||||
|
<dd>or</dd>
|
||||||
|
<dt class="TableCellDark"><label for="url">Url:</label></dt>
|
||||||
|
<dd><input name="url" id="url" value="" size="75"/></dd>
|
||||||
|
<dt></dt>
|
||||||
|
<dd><input name="submit" class="btn btn-primary" type="submit" value="Import JsonList File" /></dd>
|
||||||
|
</dl>
|
||||||
|
</fieldset>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<br />
|
||||||
|
::
|
||||||
|
<form>
|
||||||
|
<fieldset><legend>Import Process</legend>
|
||||||
|
<dl>
|
||||||
|
<dt>Thread:</dt><dd>#[thread]#</dd>
|
||||||
|
<dt>JsonList File:</dt><dd>#[jsonlistfile]#</dd>
|
||||||
|
<dt>Processed:</dt><dd>#[count]# Entries</dd>
|
||||||
|
<dt>Speed:</dt><dd>#[speed]# pages per second</dd>
|
||||||
|
<dt>Running Time:</dt><dd>#[runningHours]# hours, #[runningMinutes]# minutes</dd>
|
||||||
|
<dt>Remaining Time:</dt><dd>#[remainingHours]# hours, #[remainingMinutes]# minutes</dd>
|
||||||
|
</dl>
|
||||||
|
</fieldset>
|
||||||
|
<input name="abort" type="submit" class="btn btn-danger" value="Stop"/>
|
||||||
|
</form>
|
||||||
|
#(/import)#
|
||||||
|
|
||||||
|
#%env/templates/footer.template%#
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@ -0,0 +1,104 @@
|
|||||||
|
/**
|
||||||
|
* IndexImportJsonList_p
|
||||||
|
* Copyright 23.10.2022 by Michael Peter Christen, @orbiterlab
|
||||||
|
*
|
||||||
|
* This library is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU Lesser General Public
|
||||||
|
* License as published by the Free Software Foundation; either
|
||||||
|
* version 2.1 of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This library is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* Lesser General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Lesser General Public License
|
||||||
|
* along with this program in the file lgpl21.txt
|
||||||
|
* If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package net.yacy.htroot;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.nio.channels.Channels;
|
||||||
|
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.document.importer.JsonListImporter;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
|
||||||
|
public class IndexImportJsonList_p {
|
||||||
|
|
||||||
|
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
|
||||||
|
if (JsonListImporter.job != null && JsonListImporter.job.isAlive()) {
|
||||||
|
// one import is running, no option to insert anything
|
||||||
|
prop.put("import", 1);
|
||||||
|
prop.put("import_thread", "running");
|
||||||
|
prop.put("import_jsonlistfile", JsonListImporter.job.source());
|
||||||
|
prop.put("import_count", JsonListImporter.job.count());
|
||||||
|
prop.put("import_speed", JsonListImporter.job.speed());
|
||||||
|
prop.put("import_runningHours", (JsonListImporter.job.runningTime() / 60) / 60);
|
||||||
|
prop.put("import_runningMinutes", (JsonListImporter.job.runningTime() / 60) % 60);
|
||||||
|
prop.put("import_remainingHours", (JsonListImporter.job.remainingTime() / 60) / 60);
|
||||||
|
prop.put("import_remainingMinutes", (JsonListImporter.job.remainingTime() / 60) % 60);
|
||||||
|
if (post != null && post.containsKey("abort")) {
|
||||||
|
JsonListImporter.job.quit();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
prop.put("import", 0);
|
||||||
|
if (post != null) {
|
||||||
|
if (post.containsKey("file") || post.containsKey("url")) {
|
||||||
|
final String filename = post.get("file");
|
||||||
|
if (filename != null && filename.length() > 0) {
|
||||||
|
final File sourcefile = new File(filename);
|
||||||
|
if (sourcefile.exists()) {
|
||||||
|
try {
|
||||||
|
final JsonListImporter wi = new JsonListImporter(sourcefile, false);
|
||||||
|
wi.start();
|
||||||
|
prop.put("import_thread", "started");
|
||||||
|
} catch (final IOException ex) {
|
||||||
|
prop.put("import_thread", "Error: file not found [" + filename + "]");
|
||||||
|
}
|
||||||
|
prop.put("import", 1);
|
||||||
|
prop.put("import_jsonlistfile", filename);
|
||||||
|
} else {
|
||||||
|
prop.put("import_jsonlistfile", "");
|
||||||
|
prop.put("import_thread", "Error: file not found [" + filename + "]");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
final String urlstr = post.get("url");
|
||||||
|
if (urlstr != null && urlstr.length() > 0) {
|
||||||
|
try {
|
||||||
|
final URL url = new URL(urlstr);
|
||||||
|
final File tempfile = File.createTempFile("jsonlistimporter", "");
|
||||||
|
final FileOutputStream fos = new FileOutputStream(tempfile);
|
||||||
|
fos.getChannel().transferFrom(Channels.newChannel(url.openStream()), 0, Long.MAX_VALUE);
|
||||||
|
fos.close();
|
||||||
|
final JsonListImporter wi = new JsonListImporter(tempfile, true);
|
||||||
|
wi.start();
|
||||||
|
prop.put("import_thread", "started");
|
||||||
|
} catch (final IOException ex) {
|
||||||
|
prop.put("import_thread", ex.getMessage());
|
||||||
|
}
|
||||||
|
prop.put("import", 1);
|
||||||
|
prop.put("import_jsonlistfile", urlstr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
prop.put("import_count", 0);
|
||||||
|
prop.put("import_speed", 0);
|
||||||
|
prop.put("import_runningHours", 0);
|
||||||
|
prop.put("import_runningMinutes", 0);
|
||||||
|
prop.put("import_remainingHours", 0);
|
||||||
|
prop.put("import_remainingMinutes", 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in new issue