boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Allow edtion of scheduled next execution dates for finer control

Browse Source 
Can be useful more especially when scheduling many API calls over a long
period of time to precisely adjust each scheduled date/time.
pull/186/head
luccioman 7 years ago
parent 40e8c7b89b
commit 9bc7b6c39d
3 changed files with 100 additions and 17 deletions
Show Stats Download Patch File Download Diff File

32
htroot/Table_API_p.html
Unescape View File

@ -1,5 +1,5 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<!DOCTYPE html>
<html lang="en">
<head>
<title>YaCy '#[clientname]#': Peer Steering</title>
@ -76,7 +76,11 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<th>Call<br/>Count</th>
<th>Recording<br/>Date</th>
<th>Last&nbsp;Exec<br/>Date</th>
<th>Next&nbsp;Exec<br/>Date</th>
<th>Next&nbsp;Exec<br/>Date#(hasEditableNextExecDate)#:: <button type="submit" name="submitNextExecDates" class="btn btn-default btn-xs" title="Apply edited next execution dates">
<span class="glyphicon glyphicon-ok"></span>
<span> Apply</span>
</button>#(/hasEditableNextExecDate)#
</th>
<th class="sorttable_nosort">Event Trigger</th>
<th class="sorttable_nosort">Scheduler</th>
#(inline)#<th class="sorttable_nosort">URL</th>::#(/inline)#
@ -113,7 +117,14 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<td valign="top">#[callcount]#</td>
<td valign="top">#[dateRecording]#</td>
<td valign="top">#[dateLastExec]#</td>
<td valign="top">#[dateNextExec]#</td>
<td valign="top">#(editableDateNext)# #[dateNextExec]#
::
#(dateFormatError)#::<div class="alert alert-danger" role="alert">Malformed date "#[invalidDate]#".</div>#(/dateFormatError)#
#(dateBeforeNowError)#::<div class="alert alert-danger" role="alert">"#[invalidDate]#" is before the current date.</div>#(/dateBeforeNowError)#
<input name="date_next_exec_#[pk]#" type="text" required="required"
maxlength="19" minlength="19" pattern="#[dateLastExecPattern]#"
value="#[dateNextExec]#" size="19" title="yyyy/MM/dd HH:mm:ss"/>#(/editableDateNext)#
</td>
<td valign="top">
#(event)#
<select name="event_select_#[pk]#" onchange='submitchange("#[pk]#")'>
@ -194,14 +205,11 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<input type="hidden" name="scheduleeventaction" id="scheduleeventaction" value="false" />
<input type="hidden" name="current_pk" id="current_pk" value="" />
<input type="hidden" name="num" value="#[num]#" />
<p>
<div class="form-horizontal">
<div class="form-group col-sm-12">
<input type="submit" name="execrows" value="Execute Selected Actions" class="btn btn-success"/>&nbsp;&nbsp;&nbsp;
<input type="submit" name="deleterows" value="Delete Selected Actions" class="btn btn-danger" onclick="return confirm('Confirm Deletion')"/>&nbsp;&nbsp;&nbsp;
</div>
</p>
<p>
<div class="form-group">
<div class="form-group col-sm-6">
<input type="submit" name="deleteold" value="Delete all Actions which had been created before " class="btn btn-warning" style="float:left;" onclick="return confirm('Confirm Deletion')"/>
<select name="deleteoldtime" class="form-control selectWidth input-sm" style="width:auto;">
<option value="1">1 day</option><option value="2">2 days</option><option value="3">3 days</option>
@ -212,7 +220,11 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<option value="365">1 year</option><option value="730">2 years</option>
</select>
</div>
</p>
#(hasEditableNextExecDate)#::
<div class="form-group col-sm-6">
<input type="submit" name="submitNextExecDates" class="btn btn-default" value="Apply edited next execution dates"/>
</div>
#(/hasEditableNextExecDate)#
</form>
#(/showtable)#

79
htroot/Table_API_p.java
Unescape View File

@ -18,8 +18,11 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -35,6 +38,7 @@ import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.data.TransactionManager;
import net.yacy.data.WorkTables;
import net.yacy.kelondro.blob.Tables;
@ -157,6 +161,50 @@ public class Table_API_p {
} catch (final Throwable e) { ConcurrentLog.logException(e); }
}
/* Edition of scheduled next execution dates */
final Map<String, String> invalidNextExecDateFormats = new HashMap<>();
final Map<String, String> nextExecDatesBeforeNow = new HashMap<>();
if (post != null && post.containsKey("submitNextExecDates")) {
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
final String dateNexExecPrefix = "date_next_exec_";
final Date now = new Date();
for (final Map.Entry<String, String> entry : post.entrySet()) {
if (entry.getKey().startsWith(dateNexExecPrefix) && entry.getValue() != null) {
try {
final String rowPkStr = entry.getKey().substring(dateNexExecPrefix.length());
final Tables.Row row = sb.tables.select(WorkTables.TABLE_API_NAME,
rowPkStr.getBytes(StandardCharsets.UTF_8));
if(row != null) {
final int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 0);
final String dateNextExecStr = entry.getValue().trim();
try {
final Date dateNextExec = GenericFormatter.FORMAT_SIMPLE.parse(dateNextExecStr);
if(time != 0) { // Check there is effectively a schedule period on this row
if(dateNextExec.before(now)) {
nextExecDatesBeforeNow.put(rowPkStr, dateNextExecStr);
} else {
row.put(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, dateNextExec);
sb.tables.update(WorkTables.TABLE_API_NAME, row);
}
}
} catch (final ParseException e) {
invalidNextExecDateFormats.put(rowPkStr, dateNextExecStr);
}
}
} catch (final IOException | SpaceExceededException e) {
ConcurrentLog.logException(e);
}
}
}
}
if (post != null && !post.get("deleterows", "").isEmpty()) {
/* Check this is a valid transaction */
@ -250,7 +298,7 @@ public class Table_API_p {
}
prop.put("showexec_list", count);
}
// generate table
prop.put("showtable", 1);
prop.put("showtable_inline", inline ? 1 : 0);
@ -328,12 +376,14 @@ public class Table_API_p {
// then work on the list
for (final Tables.Row row : table) {
final String rowPKStr = UTF8.String(row.getPK());
final Date now = new Date();
final Date date = row.containsKey(WorkTables.TABLE_API_COL_DATE) ? row.get(WorkTables.TABLE_API_COL_DATE, now) : null;
final Date date_recording = row.get(WorkTables.TABLE_API_COL_DATE_RECORDING, date);
final Date date_last_exec = row.get(WorkTables.TABLE_API_COL_DATE_LAST_EXEC, date);
final Date date_next_exec = row.get(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, (Date) null);
final int callcount = row.get(WorkTables.TABLE_API_COL_APICALL_COUNT, 1);
final int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 0);
prop.put("showtable_list_" + count + "_inline", inline ? 1 : 0);
prop.put("showtable_list_" + count + "_dark", dark ? 1 : 0);
dark = !dark;
@ -342,7 +392,24 @@ public class Table_API_p {
prop.put("showtable_list_" + count + "_callcount", callcount);
prop.put("showtable_list_" + count + "_dateRecording", date_recording == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_recording));
prop.put("showtable_list_" + count + "_dateLastExec", date_last_exec == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_last_exec));
prop.put("showtable_list_" + count + "_dateNextExec", date_next_exec == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_next_exec));
prop.put("showtable_list_" + count + "_editableDateNext", time != 0);
final String enteredDateBeforeNow = nextExecDatesBeforeNow.get(rowPKStr);
prop.put("showtable_list_" + count + "_editableDateNext_dateBeforeNowError", enteredDateBeforeNow != null);
if(enteredDateBeforeNow != null) {
prop.put("showtable_list_" + count + "_editableDateNext_dateBeforeNowError_invalidDate", enteredDateBeforeNow);
}
final String invalidEnteredDate = invalidNextExecDateFormats.get(rowPKStr);
prop.put("showtable_list_" + count + "_editableDateNext_dateFormatError", invalidEnteredDate != null);
if(invalidEnteredDate != null) {
prop.put("showtable_list_" + count + "_editableDateNext_dateFormatError_invalidDate", invalidEnteredDate);
}
prop.put("showtable_list_" + count + "_editableDateNext_dateLastExecPattern", GenericFormatter.PATTERN_SIMPLE_REGEX);
prop.put("showtable_list_" + count + "_editableDateNext_dateNextExec", date_next_exec == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_next_exec));
prop.put("showtable_list_" + count + "_editableDateNext_pk", rowPKStr);
prop.put("showtable_list_" + count + "_type", row.get(WorkTables.TABLE_API_COL_TYPE));
prop.putHTML("showtable_list_" + count + "_comment", row.get(WorkTables.TABLE_API_COL_COMMENT));
// check type & action to link crawl start URLs back to CrawlStartExpert.html
@ -352,7 +419,7 @@ public class Table_API_p {
if (editUrl.length() > 1000) {
final MultiProtocolURL u = new MultiProtocolURL("http://localhost:8090" + editUrl);
prop.put("showtable_list_" + count + "_isCrawlerStart", 2);
prop.put("showtable_list_" + count + "_isCrawlerStart_pk", UTF8.String(row.getPK()));
prop.put("showtable_list_" + count + "_isCrawlerStart_pk", rowPKStr);
prop.put("showtable_list_" + count + "_isCrawlerStart_servlet", "CrawlStartExpert.html");
Map<String, String> attr = u.getAttributes();
int ac = 0;
@ -383,7 +450,7 @@ public class Table_API_p {
// events
final String kind = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, "off");
final String action = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_ACTION, "startup");
prop.put("showtable_list_" + count + "_event_pk", UTF8.String(row.getPK()));
prop.put("showtable_list_" + count + "_event_pk", rowPKStr);
boolean schedulerDisabled = "regular".equals(kind);
if ("off".equals(kind)) {
prop.put("showtable_list_" + count + "_event", 0);
@ -403,11 +470,10 @@ public class Table_API_p {
// scheduler
final String unit = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_UNIT, "days");
final int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 0);
prop.put("showtable_list_" + count + "_selectedMinutes", unit.equals("minutes") ? 1 : 0);
prop.put("showtable_list_" + count + "_selectedHours", unit.equals("hours") ? 1 : 0);
prop.put("showtable_list_" + count + "_selectedDays", (unit.isEmpty() || unit.equals("days")) ? 1 : 0);
prop.put("showtable_list_" + count + "_scheduler_pk", UTF8.String(row.getPK()));
prop.put("showtable_list_" + count + "_scheduler_pk", rowPKStr);
prop.put("showtable_list_" + count + "_scheduler_disabled", schedulerDisabled ? 1 : 0);
prop.put("showtable_list_" + count + "_repeatTime", time);
if (time == 0) {
@ -448,6 +514,7 @@ public class Table_API_p {
}
count++;
}
prop.put("showtable_hasEditableNextExecDate", scheduledactions);
if (scheduledactions) {
prop.put("showschedulerhint", 1);
prop.put("showschedulerhint_tfminutes", sb.getConfigLong(SwitchboardConstants.CLEANUP_BUSYSLEEP, 300000) / 60000);

6
source/net/yacy/cora/date/GenericFormatter.java
Unescape View File

@ -44,6 +44,10 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
public static final String PATTERN_ANSIC = "EEE MMM d HH:mm:ss yyyy";
public static final String PATTERN_SIMPLE = "yyyy/MM/dd HH:mm:ss";
/** A regular expression matching the PATTERN_SIMPLE pattern (does not control last day of month (30/31 or 28/29 for february).
* Can be used as a HTML5 input field validation pattern */
public static final String PATTERN_SIMPLE_REGEX ="[0-9]{4}/(0[1-9]|1[012])/(0[1-9]|1[0-9]|2[0-9]|3[01]) (0[0-9]|1[0-9]|2[0-3])(:[0-5][0-9]){2}";
public static final SimpleDateFormat FORMAT_SHORT_DAY = new SimpleDateFormat(PATTERN_SHORT_DAY, Locale.US);
public static final SimpleDateFormat FORMAT_SHORT_MINUTE = new SimpleDateFormat(PATTERN_SHORT_MINUTE, Locale.US);
public static final SimpleDateFormat FORMAT_SHORT_SECOND = new SimpleDateFormat(PATTERN_SHORT_SECOND, Locale.US);
@ -157,7 +161,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
else throw new IllegalArgumentException("UTC String malformed (wrong sign):" + diffString);
final int oh = NumberTools.parseIntDecSubstring(diffString, 1, 3);
final int om = NumberTools.parseIntDecSubstring(diffString, 3);
return (int) ( ((ahead) ? 1 : -1) * (oh * 60 + om));
return ((ahead) ? 1 : -1) * (oh * 60 + om);
}
/**

Write Preview
Loading…
Cancel
Save