From 10c03c6c641df3c1c915d582560fec520de55f77 Mon Sep 17 00:00:00 2001 From: luccioman Date: Tue, 2 May 2017 09:38:45 +0200 Subject: [PATCH] Improved MediaWiki dump import monitoring. When import thread is terminated : - now stop refreshing and stay on the monitoring page to give user a feedback after a long running import - added link to the next monitoring step : results from surrogates reader - added link to new import On the new import page, added a link on the eventual last import report. --- htroot/IndexImportMediawiki_p.html | 11 +++++------ htroot/IndexImportMediawiki_p.java | 26 ++++++++++++-------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/htroot/IndexImportMediawiki_p.html b/htroot/IndexImportMediawiki_p.html index d3379e858..907b597ff 100644 --- a/htroot/IndexImportMediawiki_p.html +++ b/htroot/IndexImportMediawiki_p.html @@ -3,9 +3,9 @@ YaCy '#[clientname]#': MediaWiki Dump Import #%env/templates/metas.template%# - #(import)#:: + #(refresh)#:: - #(/import)# + #(/refresh)# #%env/templates/header.template%# @@ -13,9 +13,7 @@

MediaWiki Dump Import

#(import)# -

#(prevStatus)# - ::

- #(/prevStatus)#

+

#(prevReport)#::Check the last import report.#(/prevReport)#

#(status)#

:: :: @@ -77,7 +75,8 @@ #(/status)#

Import Process
-
Thread:
#[thread]#
+
Thread:
#(thread)#terminated. You can check the next step in the Crawl Results page + or start a new import.::started::running#(/thread)#
Dump:
#[dump]#
Processed:
#[count]# Wiki Entries
Speed:
#[speed]# articles per second
diff --git a/htroot/IndexImportMediawiki_p.java b/htroot/IndexImportMediawiki_p.java index 574892d46..8e0967b85 100644 --- a/htroot/IndexImportMediawiki_p.java +++ b/htroot/IndexImportMediawiki_p.java @@ -45,7 +45,9 @@ public class IndexImportMediawiki_p { * @param header servlet request header * @param post request parameters. Supported keys : *
    - *
  • file : a dump file path on this YaCy server local file system
    • + *
  • file : a dump URL or file path on this YaCy server local file system
    • + *
  • report : when set, display the currently running thread monitoring info, or the last import report when no one is running. + * Ignored when no import thread is known.
    • *
* @param env server environment * @return the servlet answer object @@ -54,15 +56,17 @@ public class IndexImportMediawiki_p { final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; - if (MediawikiImporter.job != null && MediawikiImporter.job.isAlive()) { - // one import is running, no option to insert anything + if (MediawikiImporter.job != null && (MediawikiImporter.job.isAlive() || (post != null && post.containsKey("report")))) { + /* one import is running, or report was explicitly requested : no option to insert anything */ prop.put("import", 1); + /* Only refresh automatically when the job is running */ + prop.put("refresh", MediawikiImporter.job.isAlive() ? 1 : 0); final String jobErrorMessage = MediawikiImporter.job.status(); if( jobErrorMessage != null && !jobErrorMessage.isEmpty()) { prop.put("import_status", 1); prop.put("import_status_message", jobErrorMessage); } - prop.put("import_thread", "running"); + prop.put("import_thread", MediawikiImporter.job.isAlive() ? 2 : 0); prop.put("import_dump", MediawikiImporter.job.source()); prop.put("import_count", MediawikiImporter.job.count()); prop.put("import_speed", MediawikiImporter.job.speed()); @@ -72,15 +76,8 @@ public class IndexImportMediawiki_p { prop.put("import_remainingMinutes", (MediawikiImporter.job.remainingTime() / 60) % 60); } else { prop.put("import", 0); - if(MediawikiImporter.job != null) { - /* Report eventual fail report from the last terminated import (for example an HTTP 404 status) - * that else could be missed by the user because of page refresh */ - final String jobErrorMessage = MediawikiImporter.job.status(); - if( jobErrorMessage != null && !jobErrorMessage.isEmpty()) { - prop.put("import_prevStatus", 1); - prop.put("import_prevStatus_message", jobErrorMessage); - } - } + prop.put("refresh", 0); + prop.put("import_prevReport", MediawikiImporter.job != null ? 1 : 0); if (post == null) { prop.put("import_status", 0); @@ -118,8 +115,9 @@ public class IndexImportMediawiki_p { MediawikiImporter.job = new MediawikiImporter(sourceURL, sb.surrogatesInPath); MediawikiImporter.job.start(); prop.put("import_dump", MediawikiImporter.job.source()); - prop.put("import_thread", "started"); + prop.put("import_thread", 1); prop.put("import", 1); + prop.put("refresh", 1); } else { prop.put("import_status", status); prop.put("import_status_sourceFile", sourceFilePath);