boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Improved MediaWiki dump import monitoring.

Browse Source 
When import thread is terminated :
 - now stop refreshing and stay on the monitoring page to give user a
feedback after a long running import
 - added link to the next monitoring step : results from surrogates
reader
 - added link to new import
 
On the new import page, added a link on the eventual last import report.
pull/122/head
luccioman 8 years ago
parent edd7ccac40
commit 10c03c6c64
2 changed files with 17 additions and 20 deletions
Show Stats Download Patch File Download Diff File

11
htroot/IndexImportMediawiki_p.html
Unescape View File

@ -3,9 +3,9 @@
<head>
<title>YaCy '#[clientname]#': MediaWiki Dump Import</title>
#%env/templates/metas.template%#
#(import)#::<meta http-equiv="REFRESH" content="10;url=IndexImportMediawiki_p.html" />
#(refresh)#::<meta http-equiv="REFRESH" content="10;url=IndexImportMediawiki_p.html?report=" />
<!-- the url= removes http get parameters on refresh, preventing restart of import -->
#(/import)#
#(/refresh)#
</head>
<body id="IndexImportMediawiki">
#%env/templates/header.template%#
@ -13,9 +13,7 @@
<h2>MediaWiki Dump Import</h2>
#(import)#
<p>#(prevStatus)#
::<div class="alert alert-danger" role="alert">Error on last import : #[message]#</div>
#(/prevStatus)#</p>
<p>#(prevReport)#::<a href="IndexImportMediawiki_p.html?report=">Check</a> the last import report.#(/prevReport)#</p>
<p>#(status)#<div class="alert alert-info" role="alert">No import thread is running, you can start a new thread here</div>
::<div class="alert alert-danger" role="alert">Error : dump <abbr title="Uniform Resource Locator">URL</abbr> is malformed.</div>
::<div class="alert alert-danger" role="alert">Error : file not found "#[sourceFile]#"</div>
@ -77,7 +75,8 @@
#(/status)#</p>
<form><fieldset><legend>Import Process</legend>
<dl>
<dt>Thread:</dt><dd>#[thread]#</dd>
<dt>Thread:</dt><dd>#(thread)#terminated. You can check the next step in the <a href="CrawlResults.html?process=7">Crawl Results page</a>
or start a <a href="IndexImportMediawiki_p.html">new import</a>.::started::running#(/thread)#</dd>
<dt>Dump:</dt><dd>#[dump]#</dd>
<dt>Processed:</dt><dd>#[count]# Wiki Entries</dd>
<dt>Speed:</dt><dd>#[speed]# articles per second</dd>

26
htroot/IndexImportMediawiki_p.java
Unescape View File

@ -45,7 +45,9 @@ public class IndexImportMediawiki_p {
* @param header servlet request header
* @param post request parameters. Supported keys :
* <ul>
* <li>file : a dump file path on this YaCy server local file system</li>
* <li>file : a dump URL or file path on this YaCy server local file system</li>
* <li>report : when set, display the currently running thread monitoring info, or the last import report when no one is running.
* Ignored when no import thread is known.</li>
* </ul>
* @param env server environment
* @return the servlet answer object
@ -54,15 +56,17 @@ public class IndexImportMediawiki_p {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
if (MediawikiImporter.job != null && MediawikiImporter.job.isAlive()) {
// one import is running, no option to insert anything
if (MediawikiImporter.job != null && (MediawikiImporter.job.isAlive() || (post != null && post.containsKey("report")))) {
/* one import is running, or report was explicitly requested : no option to insert anything */
prop.put("import", 1);
/* Only refresh automatically when the job is running */
prop.put("refresh", MediawikiImporter.job.isAlive() ? 1 : 0);
final String jobErrorMessage = MediawikiImporter.job.status();
if( jobErrorMessage != null && !jobErrorMessage.isEmpty()) {
prop.put("import_status", 1);
prop.put("import_status_message", jobErrorMessage);
}
prop.put("import_thread", "running");
prop.put("import_thread", MediawikiImporter.job.isAlive() ? 2 : 0);
prop.put("import_dump", MediawikiImporter.job.source());
prop.put("import_count", MediawikiImporter.job.count());
prop.put("import_speed", MediawikiImporter.job.speed());
@ -72,15 +76,8 @@ public class IndexImportMediawiki_p {
prop.put("import_remainingMinutes", (MediawikiImporter.job.remainingTime() / 60) % 60);
} else {
prop.put("import", 0);
if(MediawikiImporter.job != null) {
/* Report eventual fail report from the last terminated import (for example an HTTP 404 status)
* that else could be missed by the user because of page refresh */
final String jobErrorMessage = MediawikiImporter.job.status();
if( jobErrorMessage != null && !jobErrorMessage.isEmpty()) {
prop.put("import_prevStatus", 1);
prop.put("import_prevStatus_message", jobErrorMessage);
}
}
prop.put("refresh", 0);
prop.put("import_prevReport", MediawikiImporter.job != null ? 1 : 0);
if (post == null) {
prop.put("import_status", 0);
@ -118,8 +115,9 @@ public class IndexImportMediawiki_p {
MediawikiImporter.job = new MediawikiImporter(sourceURL, sb.surrogatesInPath);
MediawikiImporter.job.start();
prop.put("import_dump", MediawikiImporter.job.source());
prop.put("import_thread", "started");
prop.put("import_thread", 1);
prop.put("import", 1);
prop.put("refresh", 1);
} else {
prop.put("import_status", status);
prop.put("import_status_sourceFile", sourceFilePath);

Write Preview
Loading…
Cancel
Save