You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
229 lines
8.2 KiB
229 lines
8.2 KiB
// ResourceObserver.java
|
|
// -----------------------
|
|
// part of YaCy
|
|
// (C) by Detlef Reichl; detlef!reichl()gmx!org
|
|
// Pforzheim, Germany, 2008
|
|
//
|
|
// changes by David Wieditz
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
package de.anomic.crawler;
|
|
|
|
|
|
import java.io.IOException;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.Map;
|
|
|
|
import net.yacy.kelondro.logging.Log;
|
|
import net.yacy.kelondro.util.DiskSpace;
|
|
import net.yacy.kelondro.util.MemoryControl;
|
|
|
|
import de.anomic.search.Switchboard;
|
|
import de.anomic.search.SwitchboardConstants;
|
|
|
|
public final class ResourceObserver {
|
|
// Unknown for now
|
|
//private static final static long MIN_FREE_MEMORY = 0;
|
|
// We are called with the cleanup job every five minutes;
|
|
// the disk usage should be checked with every run
|
|
private static final int CHECK_DISK_USAGE_FREQ = 1;
|
|
// The memory usage should be checked on every run
|
|
private static final int CHECK_MEMORY_USAGE_FREQ = 1;
|
|
|
|
// return values for available disk/memory
|
|
private static final int LOW = 0;
|
|
private static final int MEDIUM = 1;
|
|
private static final int HIGH = 2;
|
|
|
|
public static final Log log = new Log("RESOURCE OBSERVER");
|
|
private final Switchboard sb;
|
|
|
|
private int checkDiskUsageCount;
|
|
private int checkMemoryUsageCount;
|
|
private int disksFree;
|
|
private int memoryFree;
|
|
private boolean disabledDHT = false;
|
|
|
|
/**
|
|
* The ResourceObserver checks the resources
|
|
* and pauses crawls if necessary
|
|
* @param sb the plasmaSwitchboard
|
|
*/
|
|
public ResourceObserver(final Switchboard sb) {
|
|
this.sb = sb;
|
|
log.logInfo("initializing the resource observer");
|
|
|
|
final ArrayList<String> pathsToCheck = new ArrayList<String>();
|
|
// FIXME whats about the secondary path???
|
|
// = (getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, "");
|
|
final String[] pathes = {
|
|
SwitchboardConstants.HTDOCS_PATH,
|
|
SwitchboardConstants.INDEX_PRIMARY_PATH,
|
|
SwitchboardConstants.LISTS_PATH,
|
|
SwitchboardConstants.RANKING_PATH,
|
|
SwitchboardConstants.WORK_PATH};
|
|
String path;
|
|
for (final String element : pathes) {
|
|
try {
|
|
path = sb.getConfigPath(element, "").getCanonicalPath();
|
|
if (path.length() > 0) pathsToCheck.add(path);
|
|
} catch (final IOException e) {}
|
|
}
|
|
|
|
DiskSpace.init(pathsToCheck);
|
|
|
|
if (!DiskSpace.isUsable())
|
|
log.logWarning("Disk usage returned: " + DiskSpace.getErrorMessage());
|
|
|
|
checkDiskUsageCount = 0;
|
|
checkMemoryUsageCount = 0;
|
|
disksFree = HIGH;
|
|
memoryFree = HIGH;
|
|
}
|
|
|
|
public static void initThread() {
|
|
Switchboard sb = Switchboard.getSwitchboard();
|
|
// initializing the resourceObserver
|
|
sb.observer = new ResourceObserver(sb);
|
|
// run the oberver here a first time
|
|
sb.observer.resourceObserverJob();
|
|
}
|
|
|
|
/**
|
|
* checks the resources and pauses crawls if necessary
|
|
*/
|
|
public void resourceObserverJob() {
|
|
MemoryControl.setDHTkbytes(getMinFreeMemory());
|
|
|
|
checkDiskUsageCount++;
|
|
checkMemoryUsageCount++;
|
|
int tmpDisksFree = HIGH;
|
|
int tmpMemoryFree = HIGH;
|
|
if (checkDiskUsageCount >= CHECK_DISK_USAGE_FREQ) {
|
|
checkDiskUsageCount = 0;
|
|
tmpDisksFree = checkDisks();
|
|
disksFree = tmpDisksFree;
|
|
}
|
|
if (checkMemoryUsageCount >= CHECK_MEMORY_USAGE_FREQ) {
|
|
checkMemoryUsageCount = 0;
|
|
tmpMemoryFree = checkMemory();
|
|
memoryFree = tmpMemoryFree;
|
|
}
|
|
|
|
if (tmpDisksFree < HIGH || tmpMemoryFree < HIGH) {
|
|
if (tmpDisksFree < HIGH && !sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
|
|
log.logInfo("pausing local crawls");
|
|
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
|
|
}
|
|
if (tmpDisksFree < HIGH && !sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
|
|
log.logInfo("pausing remote triggered crawls");
|
|
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
|
}
|
|
if ((tmpDisksFree == LOW || tmpMemoryFree < HIGH) && sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) {
|
|
log.logInfo("disabling index receive");
|
|
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
|
|
sb.peers.mySeed().setFlagAcceptRemoteIndex(false);
|
|
disabledDHT = true;
|
|
}
|
|
}
|
|
else {
|
|
if (DiskSpace.isUsable()) {
|
|
if(disabledDHT) { // we were wrong!
|
|
log.logInfo("enabling index receive");
|
|
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
|
|
sb.peers.mySeed().setFlagAcceptRemoteIndex(true);
|
|
disabledDHT = false;
|
|
}
|
|
log.logInfo("run completed; everything in order");
|
|
}
|
|
else
|
|
log.logInfo("The observer is out of order: " + DiskSpace.getErrorMessage());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @return <code>true</code> if disk space is available
|
|
*/
|
|
public boolean getDisksOK () {
|
|
return disksFree == HIGH;
|
|
}
|
|
|
|
/**
|
|
* @return <code>true</code> if memory is available
|
|
*/
|
|
public boolean getMemoryOK () {
|
|
return memoryFree == HIGH;
|
|
}
|
|
|
|
/**
|
|
* @return amount of space (bytes) that should be kept free
|
|
*/
|
|
public long getMinFreeDiskSpace () {
|
|
return sb.getConfigLong(SwitchboardConstants.DISK_FREE, 3000) /* MiB */ * 1024L * 1024L;
|
|
}
|
|
|
|
/**
|
|
* @return amount of space (bytes) that should at least be kept free
|
|
*/
|
|
public long getMinFreeDiskSpace_hardlimit () {
|
|
return sb.getConfigLong(SwitchboardConstants.DISK_FREE_HARDLIMIT, 100) /* MiB */ * 1024L * 1024L;
|
|
}
|
|
|
|
/**
|
|
* @return amount of space (KiB) that should at least be free
|
|
*/
|
|
public long getMinFreeMemory() {
|
|
return sb.getConfigLong(SwitchboardConstants.MEMORY_ACCEPTDHT, 0);
|
|
}
|
|
|
|
/**
|
|
* returns the amount of disk space available
|
|
* @return <ul>
|
|
* <li><code>HIGH</code> if disk space is available</li>
|
|
* <li><code>MEDIUM</code> if low disk space is available</li>
|
|
* <li><code>LOW</code> if lower than hardlimit or 1/5 disk space is available (the max)</li>
|
|
* </ul>
|
|
*/
|
|
private int checkDisks() {
|
|
int ret = HIGH;
|
|
|
|
if (!DiskSpace.isUsable ())
|
|
return HIGH;
|
|
|
|
final HashMap<String, long[]> usage = DiskSpace.getDiskUsage();
|
|
long[] val;
|
|
for (final Map.Entry<String, long[]> entry: usage.entrySet()) {
|
|
val = entry.getValue();
|
|
log.logInfo("df of Volume " + entry.getKey() + ": " + (val[1] / 1024 / 1024) + " MB");
|
|
if (val[1] < getMinFreeDiskSpace()) {
|
|
log.logWarning("Volume " + entry.getKey() + ": free space (" + (val[1] / 1024 / 1024) + " MB) is too low (< " + (getMinFreeDiskSpace() / 1024 / 1024) + " MB)");
|
|
ret = MEDIUM;
|
|
}
|
|
if (val[1] < Math.max(getMinFreeDiskSpace() / 5L, getMinFreeDiskSpace_hardlimit())) {
|
|
ret = LOW;
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
private int checkMemory() {
|
|
if(!MemoryControl.getDHTallowed()) return LOW;
|
|
return HIGH;
|
|
}
|
|
}
|
|
|