From 68afe900d079219721180e769eea16a5a91aaa96 Mon Sep 17 00:00:00 2001 From: luccioman Date: Fri, 27 Jan 2017 15:47:15 +0100 Subject: [PATCH] Added user-friendly controls over disk usage configuration settings. As mentioned in issue #103, control settings over YaCy disk usage already existed but lacked a user-friendly way to set them. I added it to the Performance_p.html administration page with a little refactoring on the "Resource Observer" fieldset for improved accessibility and HTML standards respect. Also added the possibility to enable/disable the autoregulation fonction from this page. --- htroot/PerformanceQueues_p.java | 30 +++++- htroot/Performance_p.html | 100 +++++++++++++++--- htroot/env/base.css | 4 +- locales/cn.lng | 13 ++- locales/de.lng | 13 ++- locales/master.lng.xlf | 14 +-- locales/ru.lng | 12 +-- locales/uk.lng | 13 ++- source/net/yacy/search/ResourceObserver.java | 31 +++--- .../net/yacy/search/SwitchboardConstants.java | 37 ++++++- 10 files changed, 197 insertions(+), 70 deletions(-) diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index df8a94bef..23dec663a 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -72,11 +72,29 @@ public class PerformanceQueues_p { prop.put("setStartupCommit", "1"); } if(post.containsKey("diskFree")) { - sb.setConfig(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_STEADYSTATE, post.getInt("diskFree", 3000)); + sb.setConfig(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_STEADYSTATE, post.getLong("diskFree", SwitchboardConstants.RESOURCE_DISK_FREE_MIN_STEADYSTATE_DEFAULT)); } if(post.containsKey("diskFreeHardlimit")) { - sb.setConfig(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_UNDERSHOT, post.getInt("diskFreeHardlimit", 1000)); + sb.setConfig(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_UNDERSHOT, post.getLong("diskFreeHardlimit", SwitchboardConstants.RESOURCE_DISK_FREE_MIN_UNDERSHOT_DEFAULT)); + + /* This is a checkbox in Performance_p.html : when not checked the value is not in post parameters, + * so we take only in account when the relate diskFreeHardlimit is set */ + sb.setConfig(SwitchboardConstants.RESOURCE_DISK_FREE_AUTOREGULATE, + post.getBoolean("diskFreeAutoregulate")); } + if (post.containsKey("diskUsed")) { + sb.setConfig(SwitchboardConstants.RESOURCE_DISK_USED_MAX_STEADYSTATE, + post.getLong("diskUsed", SwitchboardConstants.RESOURCE_DISK_USED_MAX_STEADYSTATE_DEFAULT)); + } + if (post.containsKey("diskUsedHardlimit")) { + sb.setConfig(SwitchboardConstants.RESOURCE_DISK_USED_MAX_OVERSHOT, post.getLong("diskUsedHardlimit", + SwitchboardConstants.RESOURCE_DISK_USED_MAX_OVERSHOT_DEFAULT)); + + /* This is a checkbox in Performance_p.html : when not checked the value is not in post parameters, + * so we take only in account when the related diskFreeHardlimit is set */ + sb.setConfig(SwitchboardConstants.RESOURCE_DISK_USED_AUTOREGULATE, + post.getBoolean("diskUsedAutoregulate")); + } if(post.containsKey("memoryAcceptDHT")) { sb.setConfig(SwitchboardConstants.MEMORY_ACCEPTDHT, post.getInt("memoryAcceptDHT", 50)); } @@ -308,6 +326,14 @@ public class PerformanceQueues_p { final boolean observerTrigger = !MemoryControl.properState(); prop.put("diskFree", diskFree); prop.put("diskFreeHardlimit", diskFreeHardlimit); + prop.put("diskFreeAutoregulate", sb.getConfigBool(SwitchboardConstants.RESOURCE_DISK_FREE_AUTOREGULATE, + SwitchboardConstants.RESOURCE_DISK_FREE_AUTOREGULATE_DEFAULT) ? 1 : 0); + prop.put("diskUsed", sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_USED_MAX_STEADYSTATE, + SwitchboardConstants.RESOURCE_DISK_USED_MAX_STEADYSTATE_DEFAULT)); + prop.put("diskUsedHardlimit", sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_USED_MAX_OVERSHOT, + SwitchboardConstants.RESOURCE_DISK_USED_MAX_OVERSHOT_DEFAULT)); + prop.put("diskUsedAutoregulate", sb.getConfigBool(SwitchboardConstants.RESOURCE_DISK_USED_AUTOREGULATE, + SwitchboardConstants.RESOURCE_DISK_USED_AUTOREGULATE_DEFAULT) ? 1 : 0); prop.put("memoryAcceptDHT", memoryAcceptDHT); if(observerTrigger) prop.put("observerTrigger", "1"); diff --git a/htroot/Performance_p.html b/htroot/Performance_p.html index 498ae8421..605dbfe0b 100644 --- a/htroot/Performance_p.html +++ b/htroot/Performance_p.html @@ -1,5 +1,5 @@ - - + + YaCy '#[clientname]#': Performance Settings #%env/templates/metas.template%# @@ -26,7 +26,7 @@
Memory Settings
-
+
MByte    #(setStartupCommit)#::
Accepted change. This will take effect after restart of YaCy.
restart now
::
#(/setStartupCommit)# @@ -38,17 +38,87 @@
Resource Observer -
-
 
-
DHT-Trigger #(observerTrigger)#not triggered::triggered#(/observerTrigger)# [reset state]
-
-
disable crawls below MiB free space,
- disable DHT-in below MiB free space
-
-
disable DHT-in below MiB free space
-
 
-
-
+
+ DHT-Trigger #(observerTrigger)#not triggered::triggered#(/observerTrigger)# [reset state] +
+
+
+ Free space disk +
+ + MiB. Disable crawls when free space is below. +
+
+ + MiB. Disable DHT-in when free space is below. +
+
+ + when absolute minimum limit has been reached. +
+ Free space disk autoregulation info +
+ The autoregulation task performs the following sequence of operations, stopping once free space disk is over the steady-state value : +
    +
  1. delete old releases
    2. +
  2. delete logs
    3. +
  3. delete robots.txt table
    4. +
  4. delete news
    5. +
  5. clear HTCACHE
    6. +
  6. clear citations
    7. +
  7. throw away large crawl queues
    8. +
  8. cut away too large RWIs
    9. +
+
+
+
+
+
+
+
+ Used space disk +
+ + MiB. Disable crawls when used space is over. +
+
+ + MiB. Disable DHT-in when used space is over. +
+
+ + when absolute maximum limit has been reached. +
+ Used space disk autoregulation info +
+ The autoregulation task performs the following sequence of operations, stopping once used space disk is below the steady-state value: +
    +
  1. delete old releases
    2. +
  2. delete logs
    3. +
  3. delete robots.txt table
    4. +
  4. delete news
    5. +
  5. clear HTCACHE
    6. +
  6. clear citations
    7. +
  7. throw away large crawl queues
    8. +
  8. cut away too large RWIs
    9. +
+
+
+
+
+
+
+
+ RAM +
+ + MiB free space +
+
+
+
+ +
@@ -79,7 +149,7 @@ - + Changes take effect immediately diff --git a/htroot/env/base.css b/htroot/env/base.css index 924de111a..c0fef583d 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -1038,11 +1038,11 @@ div#tagcloud { margin: 10px 0; } -.info span { +.info span, .info div { display: none; } -.info:hover span { +.info:hover span, .info:hover div { display: block; position: absolute; font-size:1em; diff --git a/locales/cn.lng b/locales/cn.lng index fadd57f80..f99640206 100644 --- a/locales/cn.lng +++ b/locales/cn.lng @@ -1996,20 +1996,19 @@ Attributes==属性 #File: Performance_p.html #--------------------------- +== Performance Settings==性能设置 Memory Settings==内存设置 -Memory reserved for JVM==JVM内存设置 +Memory reserved for JVM==JVM内存设置 "Set"=="设置" Resource Observer==资源查看 -DHT-Trigger==DHT-触发 +DHT-Trigger==DHT-触发 not triggered:==未触发: >triggered==>已触发 reset state==重置 -HDD==硬盘 -disable crawls below==停止crawl当低于 -free space==空闲空间 -disable DHT-in below==停止接收DHT当低于 -RAM==内存 +> free space==> 空闲空间 +disable DHT-in below==停止接收DHT当低于 +RAM==内存 Accepted change. This will take effect after restart of YaCy==已接受改变. 在YaCy重启后生效 restart now==立即重启 Confirm Restart==确定重启 diff --git a/locales/de.lng b/locales/de.lng index b4eb77909..a1006c368 100644 --- a/locales/de.lng +++ b/locales/de.lng @@ -2431,20 +2431,19 @@ Attributes==Attribute #File: Performance_p.html #--------------------------- +== Performance Settings==Leistungseinstellungen Memory Settings==Speicher Einstellungen -Memory reserved for JVM==Für JVM reservierter Speicher +Memory reserved for JVM==Für JVM reservierter Speicher "Set"=="Setzen" Resource Observer==Ressourcen Beobachter -DHT-Trigger==DHT-Umschalter +DHT-Trigger==DHT-Umschalter not triggered:==nicht ausgelöst: >triggered==>ausgelöst reset state==Status zurücksetzen -HDD==Festplattenspeicher -disable crawls below==Deaktivere Crawls unter -free space==freiem Speicher -disable DHT-in below==Deaktivere eingehende DHT Transfers unter -RAM==Arbeitsspeicher +> free space==> freiem Speicher +disable DHT-in below==Deaktivere eingehende DHT Transfers unter +RAM==Arbeitsspeicher Accepted change. This will take effect after restart of YaCy==Änderung akzeptiert. Diese werden erst nach einem Neustart von YaCy wirksam restart now==jetzt neustarten Confirm Restart==Bestätige Neustart diff --git a/locales/master.lng.xlf b/locales/master.lng.xlf index a5a96a3cd..7a3c4a531 100644 --- a/locales/master.lng.xlf +++ b/locales/master.lng.xlf @@ -6499,7 +6499,7 @@ Memory Settings - Memory reserved for JVM + Memory reserved for <abbr title="Java Virtual Machine">JVM</abbr> "Set" @@ -6508,7 +6508,7 @@ Resource Observer - DHT-Trigger + <abbr title="Distributed Hash Table">DHT</abbr>-Trigger not triggered: @@ -6520,19 +6520,19 @@ reset state - HDD + Free space disk - disable crawls below + Disable crawls when free space is below - free space + > free space - disable DHT-in below + disable <abbr title="Distributed Hash Table">DHT</abbr>-in below - RAM + <abbr title="Random Access Memory">RAM</abbr> Accepted change. This will take effect after <strong>restart</strong> of YaCy diff --git a/locales/ru.lng b/locales/ru.lng index f18ae1771..6a8ea15c1 100644 --- a/locales/ru.lng +++ b/locales/ru.lng @@ -2535,20 +2535,18 @@ Attributes==Аттрибуты #File: Performance_p.html #--------------------------- +== Performance Settings==Настройки производительности Memory Settings==Настройки оперативной памяти -Memory reserved for JVM==Резервирование динамической памяти для Java +Memory reserved for JVM==Резервирование динамической памяти для Java "Set"=="Сохранить" Resource Observer==Обзор ресурсов -DHT-Trigger==DHT-триггер +DHT-Trigger==DHT-триггер not triggered:==не срабатывает: >triggered==>ausgelöst reset state==сбросить состояние -HDD==Жесткий диск -disable crawls below==Остановить индексаторы при достижении -free space==свободного места -disable DHT-in below==Остановить DHT-приём при достижении -RAM==RAM +> free space==> свободного места +disable DHT-in below==Остановить DHT-приём при достижении Accepted change. This will take effect after restart of YaCy==Изменения будут применены после перезапуска YaCy restart now==Перезапустить сейчас Confirm Restart==Подтвердить перезапуск diff --git a/locales/uk.lng b/locales/uk.lng index eb1d0458a..cc51211e5 100644 --- a/locales/uk.lng +++ b/locales/uk.lng @@ -2167,20 +2167,19 @@ Attributes==Властивості #File: Performance_p.html #--------------------------- +== Performance Settings==Установки живлення Memory Settings==Налаштування пам’яті -Memory reserved for JVM==Пам’ять, утримувана для JVM +Memory reserved for JVM==Пам’ять, утримувана для JVM "Set"=="Виставити" Resource Observer==Спостерігач за ресурсами -DHT-Trigger==DHT-перемикач +DHT-Trigger==DHT-перемикач not triggered:==не перемкнено: >triggered==>перемкнено reset state==відновити стан -HDD==Місце на жорсткому диску -disable crawls below==Вимкнути сканування при менше -free space==вільної пам’яті -disable DHT-in below==Вимкнути отримання DHT при менше -RAM==Основна пам’ять +> free space==> вільної пам’яті +disable DHT-in below==Вимкнути отримання DHT при менше +RAM==Основна пам’ять Accepted change. This will take effect after restart of YaCy==Зміни прийняті. Вони вступлять в силу тільки після перезапуску YaCy restart now==перезапустити зараз Confirm Restart==Підтрвердіть перезапуск diff --git a/source/net/yacy/search/ResourceObserver.java b/source/net/yacy/search/ResourceObserver.java index b4ca16ab9..fb27d82b9 100644 --- a/source/net/yacy/search/ResourceObserver.java +++ b/source/net/yacy/search/ResourceObserver.java @@ -45,11 +45,14 @@ public class ResourceObserver { public static final ConcurrentLog log = new ConcurrentLog("RESOURCE OBSERVER"); - // status type for which shows where in the control-circuit model a memory state can be categorized + /** status type for which shows where in the control-circuit model a memory state can be categorized */ public enum Space implements Comparable { - EXHAUSTED, // smallest space state, outside of over/undershot - NOMINAL, // wanted-space state between steady-state and under/overshot - AMPLE; // largest space state, below steady-state + /** smallest space state, outside of over/undershot */ + EXHAUSTED, + /** wanted-space state between steady-state and under/overshot */ + NOMINAL, + /** largest space state, below steady-state */ + AMPLE; } private final Switchboard sb; @@ -103,8 +106,8 @@ public class ResourceObserver { } // shrink resources if space is EXHAUSTED - if ((this.normalizedDiskFree == Space.EXHAUSTED && this.sb.getConfigBool(SwitchboardConstants.RESOURCE_DISK_FREE_AUTOREGULATE, false)) || - (this.normalizedDiskUsed == Space.EXHAUSTED && this.sb.getConfigBool(SwitchboardConstants.RESOURCE_DISK_USED_AUTOREGULATE, false))) { + if ((this.normalizedDiskFree == Space.EXHAUSTED && this.sb.getConfigBool(SwitchboardConstants.RESOURCE_DISK_FREE_AUTOREGULATE, SwitchboardConstants.RESOURCE_DISK_FREE_AUTOREGULATE_DEFAULT)) || + (this.normalizedDiskUsed == Space.EXHAUSTED && this.sb.getConfigBool(SwitchboardConstants.RESOURCE_DISK_USED_AUTOREGULATE, SwitchboardConstants.RESOURCE_DISK_USED_AUTOREGULATE_DEFAULT))) { shrinkmethods: while (true /*this is not a loop, just a construct that we can leave with a break*/) { // delete old releases if (yacyRelease.deleteOldDownloads(sb.releasePath, 1)) log.warn("DISK SPACE EXHAUSTED - deleting downloaded releases files"); @@ -316,31 +319,35 @@ public class ResourceObserver { } /** - * @return amount of space (bytes) that should be used in steady state + * @return the maximum amount of space (bytes) that should be used as steady state */ public long getMaxUsedDiskSteadystate() { - return this.sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_USED_MAX_STEADYSTATE, 524288) /* MB */ * 1024L * 1024L; + return this.sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_USED_MAX_STEADYSTATE, + SwitchboardConstants.RESOURCE_DISK_USED_MAX_STEADYSTATE_DEFAULT) /* MB */ * 1024L * 1024L; } /** - * @return amount of space (bytes) that should at least be kept free as hard limit; the limit when autoregulation to steady state should start + * @return the maximum amount of space (bytes) that should be used as hard limit; the limit when autoregulation to steady state should start */ public long getMaxUsedDiskOvershot() { - return this.sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_USED_MAX_OVERSHOT, 1048576) /* MB */ * 1024L * 1024L; + return this.sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_USED_MAX_OVERSHOT, + SwitchboardConstants.RESOURCE_DISK_USED_MAX_OVERSHOT_DEFAULT) /* MB */ * 1024L * 1024L; } /** * @return amount of space (bytes) that should be kept free as steady state */ public long getMinFreeDiskSteadystate() { - return this.sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_STEADYSTATE, 2048) /* MB */ * 1024L * 1024L; + return this.sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_STEADYSTATE, + SwitchboardConstants.RESOURCE_DISK_FREE_MIN_STEADYSTATE_DEFAULT) /* MB */ * 1024L * 1024L; } /** * @return amount of space (bytes) that should at least be kept free as hard limit; the limit when autoregulation to steady state should start */ public long getMinFreeDiskUndershot() { - return this.sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_UNDERSHOT, 1024) /* MB */ * 1024L * 1024L; + return this.sb.getConfigLong(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_UNDERSHOT, + SwitchboardConstants.RESOURCE_DISK_FREE_MIN_UNDERSHOT_DEFAULT) /* MB */ * 1024L * 1024L; } /** diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index ddc10283c..3a20a4682 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -409,19 +409,48 @@ public final class SwitchboardConstants { public static final String WORK_PATH_DEFAULT = "DATA/WORK"; - /** + /* * ResourceObserver * We apply the naming of control circuit states to resources observer limit values (steady-state value, over/undershot) * under/overshot states in the system are supposed to be regulated to match the steady-state value * ATTENTION: be aware that using the autoregulate-option causes that the search index data is DELETED as soon as threshold-values are reached! */ + + /** Setting key to enable auto-regulation on disk free threshold values */ public static final String RESOURCE_DISK_FREE_AUTOREGULATE = "resource.disk.free.autoregulate"; - public static final String RESOURCE_DISK_FREE_MIN_STEADYSTATE = "resource.disk.free.min.steadystate"; // the target steady-state of minimum disk space left - public static final String RESOURCE_DISK_FREE_MIN_UNDERSHOT = "resource.disk.free.min.undershot"; // the undershot below the steady-state of minimum disk free as absolute size + /** Default disk free auto-regulation activation setting */ + public static final boolean RESOURCE_DISK_FREE_AUTOREGULATE_DEFAULT = false; + + /** Setting key for the target steady-state of minimum disk space left */ + public static final String RESOURCE_DISK_FREE_MIN_STEADYSTATE = "resource.disk.free.min.steadystate"; + + /** Default value for target steady-state of minimum disk space left */ + public static final long RESOURCE_DISK_FREE_MIN_STEADYSTATE_DEFAULT = 2048L; + + /** Setting key for the undershot below the steady-state of minimum disk free as absolute size */ + public static final String RESOURCE_DISK_FREE_MIN_UNDERSHOT = "resource.disk.free.min.undershot"; + + /** Default value for undershot below the steady-state of minimum disk free as absolute size */ + public static final long RESOURCE_DISK_FREE_MIN_UNDERSHOT_DEFAULT = 1024L; + + /** Setting key to enable auto-regulation on disk used threshold values */ public static final String RESOURCE_DISK_USED_AUTOREGULATE = "resource.disk.used.autoregulate"; + + /** Default disk used auto-regulation activation setting */ + public static final boolean RESOURCE_DISK_USED_AUTOREGULATE_DEFAULT = false; + + /** Setting key for the disk used maximum steady state value */ public static final String RESOURCE_DISK_USED_MAX_STEADYSTATE = "resource.disk.used.max.steadystate"; - public static final String RESOURCE_DISK_USED_MAX_OVERSHOT = "resource.disk.used.max.overshot"; + + /** Default disk used maximum steady state value (in mebibyte)*/ + public static final long RESOURCE_DISK_USED_MAX_STEADYSTATE_DEFAULT = 524288L; + + /** Setting key for the disk used hard upper limit value */ + public static final String RESOURCE_DISK_USED_MAX_OVERSHOT = "resource.disk.used.max.overshot"; + + /** Default disk used hard upper limit value (in mebibyte) */ + public static final long RESOURCE_DISK_USED_MAX_OVERSHOT_DEFAULT = 1048576L; public static final String MEMORY_ACCEPTDHT = "memory.acceptDHTabove"; // minimum memory to accept dht-in (MiB) public static final String INDEX_RECEIVE_AUTODISABLED = "memory.disabledDHT"; // set if DHT was disabled by ResourceObserver