boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Improved MediaWiki dump import monitoring.

Browse Source 
When import thread is terminated :
 - now stop refreshing and stay on the monitoring page to give user a
feedback after a long running import
 - added link to the next monitoring step : results from surrogates
reader
 - added link to new import
 
On the new import page, added a link on the eventual last import report.
pull/122/head
luccioman 8 years ago
parent edd7ccac40
commit 10c03c6c64
2 changed files with 17 additions and 20 deletions
Show Stats Download Patch File Download Diff File

11
htroot/IndexImportMediawiki_p.html
Unescape View File

@ -3,9 +3,9 @@
<head> <head>
<title>YaCy '#[clientname]#': MediaWiki Dump Import</title> <title>YaCy '#[clientname]#': MediaWiki Dump Import</title>
#%env/templates/metas.template%# #%env/templates/metas.template%#
#(import)#::<meta http-equiv="REFRESH" content="10;url=IndexImportMediawiki_p.html" /> #(refresh)#::<meta http-equiv="REFRESH" content="10;url=IndexImportMediawiki_p.html?report=" />
<!-- the url= removes http get parameters on refresh, preventing restart of import --> <!-- the url= removes http get parameters on refresh, preventing restart of import -->
#(/import)# #(/refresh)#
</head> </head>
<body id="IndexImportMediawiki"> <body id="IndexImportMediawiki">
#%env/templates/header.template%# #%env/templates/header.template%#
@ -13,9 +13,7 @@
<h2>MediaWiki Dump Import</h2> <h2>MediaWiki Dump Import</h2>
#(import)# #(import)#
<p>#(prevStatus)# <p>#(prevReport)#::<a href="IndexImportMediawiki_p.html?report=">Check</a> the last import report.#(/prevReport)#</p>
::<div class="alert alert-danger" role="alert">Error on last import : #[message]#</div>
#(/prevStatus)#</p>
<p>#(status)#<div class="alert alert-info" role="alert">No import thread is running, you can start a new thread here</div> <p>#(status)#<div class="alert alert-info" role="alert">No import thread is running, you can start a new thread here</div>
::<div class="alert alert-danger" role="alert">Error : dump <abbr title="Uniform Resource Locator">URL</abbr> is malformed.</div> ::<div class="alert alert-danger" role="alert">Error : dump <abbr title="Uniform Resource Locator">URL</abbr> is malformed.</div>
::<div class="alert alert-danger" role="alert">Error : file not found "#[sourceFile]#"</div> ::<div class="alert alert-danger" role="alert">Error : file not found "#[sourceFile]#"</div>
@ -77,7 +75,8 @@
#(/status)#</p> #(/status)#</p>
<form><fieldset><legend>Import Process</legend> <form><fieldset><legend>Import Process</legend>
<dl> <dl>
<dt>Thread:</dt><dd>#[thread]#</dd> <dt>Thread:</dt><dd>#(thread)#terminated. You can check the next step in the <a href="CrawlResults.html?process=7">Crawl Results page</a>
or start a <a href="IndexImportMediawiki_p.html">new import</a>.::started::running#(/thread)#</dd>
<dt>Dump:</dt><dd>#[dump]#</dd> <dt>Dump:</dt><dd>#[dump]#</dd>
<dt>Processed:</dt><dd>#[count]# Wiki Entries</dd> <dt>Processed:</dt><dd>#[count]# Wiki Entries</dd>
<dt>Speed:</dt><dd>#[speed]# articles per second</dd> <dt>Speed:</dt><dd>#[speed]# articles per second</dd>

26
htroot/IndexImportMediawiki_p.java
Unescape View File

@ -45,7 +45,9 @@ public class IndexImportMediawiki_p {
* @param header servlet request header * @param header servlet request header
* @param post request parameters. Supported keys : * @param post request parameters. Supported keys :
* <ul> * <ul>
* <li>file : a dump file path on this YaCy server local file system</li> * <li>file : a dump URL or file path on this YaCy server local file system</li>
* <li>report : when set, display the currently running thread monitoring info, or the last import report when no one is running.
* Ignored when no import thread is known.</li>
* </ul> * </ul>
* @param env server environment * @param env server environment
* @return the servlet answer object * @return the servlet answer object
@ -54,15 +56,17 @@ public class IndexImportMediawiki_p {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
if (MediawikiImporter.job != null && MediawikiImporter.job.isAlive()) { if (MediawikiImporter.job != null && (MediawikiImporter.job.isAlive() || (post != null && post.containsKey("report")))) {
// one import is running, no option to insert anything /* one import is running, or report was explicitly requested : no option to insert anything */
prop.put("import", 1); prop.put("import", 1);
/* Only refresh automatically when the job is running */
prop.put("refresh", MediawikiImporter.job.isAlive() ? 1 : 0);
final String jobErrorMessage = MediawikiImporter.job.status(); final String jobErrorMessage = MediawikiImporter.job.status();
if( jobErrorMessage != null && !jobErrorMessage.isEmpty()) { if( jobErrorMessage != null && !jobErrorMessage.isEmpty()) {
prop.put("import_status", 1); prop.put("import_status", 1);
prop.put("import_status_message", jobErrorMessage); prop.put("import_status_message", jobErrorMessage);
} }
prop.put("import_thread", "running"); prop.put("import_thread", MediawikiImporter.job.isAlive() ? 2 : 0);
prop.put("import_dump", MediawikiImporter.job.source()); prop.put("import_dump", MediawikiImporter.job.source());
prop.put("import_count", MediawikiImporter.job.count()); prop.put("import_count", MediawikiImporter.job.count());
prop.put("import_speed", MediawikiImporter.job.speed()); prop.put("import_speed", MediawikiImporter.job.speed());
@ -72,15 +76,8 @@ public class IndexImportMediawiki_p {
prop.put("import_remainingMinutes", (MediawikiImporter.job.remainingTime() / 60) % 60); prop.put("import_remainingMinutes", (MediawikiImporter.job.remainingTime() / 60) % 60);
} else { } else {
prop.put("import", 0); prop.put("import", 0);
if(MediawikiImporter.job != null) { prop.put("refresh", 0);
/* Report eventual fail report from the last terminated import (for example an HTTP 404 status) prop.put("import_prevReport", MediawikiImporter.job != null ? 1 : 0);
* that else could be missed by the user because of page refresh */
final String jobErrorMessage = MediawikiImporter.job.status();
if( jobErrorMessage != null && !jobErrorMessage.isEmpty()) {
prop.put("import_prevStatus", 1);
prop.put("import_prevStatus_message", jobErrorMessage);
}
}
if (post == null) { if (post == null) {
prop.put("import_status", 0); prop.put("import_status", 0);
@ -118,8 +115,9 @@ public class IndexImportMediawiki_p {
MediawikiImporter.job = new MediawikiImporter(sourceURL, sb.surrogatesInPath); MediawikiImporter.job = new MediawikiImporter(sourceURL, sb.surrogatesInPath);
MediawikiImporter.job.start(); MediawikiImporter.job.start();
prop.put("import_dump", MediawikiImporter.job.source()); prop.put("import_dump", MediawikiImporter.job.source());
prop.put("import_thread", "started"); prop.put("import_thread", 1);
prop.put("import", 1); prop.put("import", 1);
prop.put("refresh", 1);
} else { } else {
prop.put("import_status", status); prop.put("import_status", status);
prop.put("import_status_sourceFile", sourceFilePath); prop.put("import_status_sourceFile", sourceFilePath);

Write Preview
Loading…
Cancel
Save