From 69aac0d74c58f97ab636f2e0b8b1c4483edfba2d Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 30 Jun 2008 21:47:53 +0000 Subject: [PATCH] modified the diskUsage class regarding the following two aspects: 1. The usage and dependency of the plasmaSwitchboad was used many times in the past but this was a bad mistake. The classes should be independent from the switchboard to support a better abstraction. Therefore the object was removed. The parameters from the switchboard are computed outside and then handed over. 2. the class is considered as a tightly connected to hardware resources. Classes which handle data that cannot be replicated because it would need to replicate hadware should not support dynamic object allocation, but should be coded as collection of private static methods. Therefore all class objects had been transformed into static private objects. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4961 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../de/anomic/plasma/plasmaSwitchboard.java | 8 +- source/de/anomic/tools/consoleInterface.java | 123 +++++++ source/de/anomic/tools/diskUsage.java | 319 +++++++----------- source/de/anomic/yacy/resourceObserver.java | 42 ++- source/de/anomic/yacy/yacyDHTAction.java | 1 + 5 files changed, 280 insertions(+), 213 deletions(-) create mode 100755 source/de/anomic/tools/consoleInterface.java diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index b535102b9..0df164baa 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -876,9 +876,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch output = new ArrayList(); + private final Semaphore dataIsRead = new Semaphore(1); + /** + * FIXME just for debugging + */ + private final String name; + private serverLog log; + + + public consoleInterface (final InputStream stream, String name, serverLog log) + { + this.log = log; + this.stream = stream; + this.name = name; + // block reading {@see getOutput()} + try { + dataIsRead.acquire(); + } catch (InterruptedException e) { + // this should never happen because this is a constructor + e.printStackTrace(); + } + } + + public void run() { + // a second run adds data! a output.clear() maybe needed + try { + final InputStreamReader input = new InputStreamReader(stream); + final BufferedReader buffer = new BufferedReader(input); + String line = null; + int tries = 0; + while (tries < 1000) { + tries++; + try { + // may block! + Thread.sleep(1); + } catch (InterruptedException e) { + // just stop sleeping + } + if (buffer.ready()) + break; + } + log.logInfo("logpoint 3 "+ name +" needed " + tries + " tries"); + while((line = buffer.readLine()) != null) { + output.add(line); + } + dataIsRead.release(); + } catch(final IOException ix) { log.logWarning("logpoint 6 " + ix.getMessage());} + } + + /** + * waits until the stream is read and returns all data + * + * @return lines of text in stream + */ + public List getOutput(){ + // wait that data is ready + try { + log.logInfo("logpoint 4 waiting for data of '"+ name +"'"); + final long start = System.currentTimeMillis(); + dataIsRead.acquire(); + log.logInfo("logpoint 5 data ready for '"+ name +"' after "+ (System.currentTimeMillis() - start) +" ms"); + } catch (InterruptedException e) { + // after interrupt just return what is available (maybe nothing) + } + // is just for checking availability, so release it immediatly + dataIsRead.release(); + return output; + } +} \ No newline at end of file diff --git a/source/de/anomic/tools/diskUsage.java b/source/de/anomic/tools/diskUsage.java index a0a59eb40..1e204f116 100644 --- a/source/de/anomic/tools/diskUsage.java +++ b/source/de/anomic/tools/diskUsage.java @@ -46,22 +46,17 @@ package de.anomic.tools; -import java.io.BufferedReader; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.concurrent.Semaphore; -import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.logging.serverLog; public class diskUsage { - serverLog log = new serverLog("DISK USAGE"); - private static final HashMap diskUsages = new HashMap(); + + private static serverLog log = new serverLog("DISK USAGE"); private static final List allVolumes = new ArrayList(); private static final List allMountPoints = new ArrayList(); @@ -70,44 +65,45 @@ public class diskUsage { private static final List yacyUsedVolumes = new ArrayList(); private static final List yacyUsedMountPoints = new ArrayList(); - private static plasmaSwitchboard sb; - private static int usedOS; - private static boolean usable; - private static String windowsCommand; - private static String errorMessage; - private static boolean consoleError; + private static int usedOS = -1; + private static boolean usable = false; + private static String windowsCommand = null; + private static String errorMessage = ""; + private static boolean consoleError = false; // Unix-like - private final int AIX = 0; // IBM - private final int BS2000 = 1; // Fujitsu Siemens (oficial BS2000/OSD) - private final int BSD = 2; // all kind of BSD - private final int HAIKU = 3; // like BeOS; does not have a JRE til now, but they are working on it - private final int HP_UX = 4; // Hewlett-Packard - private final int TRU64 = 5; // Hewlett-Packard - private final int IRIX = 6; // sgi - private final int LINUX = 7; // all kind of linux - private final int MAC_OS_X = 8; // Apple - private final int MINIX = 9; // don't know if there even is a JRE for minix... - private final int SOLARIS = 10; // SUN - private final int SUNOS = 11; // The latest SunOS version is from 1990 but the Solaris java refferer remains SunOS - private final int UNICOS = 12; // cray - - private final int UNIX_END = UNICOS; + private static final int AIX = 0; // IBM + private static final int BS2000 = 1; // Fujitsu Siemens (oficial BS2000/OSD) + //private static final int BSD = 2; // all kind of BSD + private static final int HAIKU = 3; // like BeOS; does not have a JRE til now, but they are working on it + //private static final int HP_UX = 4; // Hewlett-Packard + private static final int TRU64 = 5; // Hewlett-Packard + //private static final int IRIX = 6; // sgi + //private static final int LINUX = 7; // all kind of linux + //private static final int MAC_OS_X = 8; // Apple + private static final int MINIX = 9; // don't know if there even is a JRE for minix... + //private static final int SOLARIS = 10; // SUN + //private static final int SUNOS = 11; // The latest SunOS version is from 1990 but the Solaris java refferer remains SunOS + private static final int UNICOS = 12; // cray + + private static final int UNIX_END = UNICOS; // Windows dos based - private final int WINDOWS_95 = 13; - private final int WINDOWS_98 = 14; - private final int WINDOWS_ME = 15; + //private static final int WINDOWS_95 = 13; + //private static final int WINDOWS_98 = 14; + //private static final int WINDOWS_ME = 15; // Windows WinNT based - private final int WINDOWS_NT = 16; - private final int WINDOWS_2000 = 17; - private final int WINDOWS_XP = 18; - private final int WINDOWS_SERVER = 19; - private final int WINDOWS_VISTA = 20; + //private static final int WINDOWS_NT = 16; + //private static final int WINDOWS_2000 = 17; + //private static final int WINDOWS_XP = 18; + //private static final int WINDOWS_SERVER = 19; + //private static final int WINDOWS_VISTA = 20; - String[] OSname = {"aix", "bs2000", "bsd", "haiku", "hp-ux", "tru64", "irix", "linux", "mac os x", "minix", + // don't change order of names! + private static final String[] OSname = { + "aix", "bs2000", "bsd", "haiku", "hp-ux", "tru64", "irix", "linux", "mac os x", "minix", "solaris", "sunos", "unicos", "windows 95", "windows 98", "windows me", "windows nt", "windows 2000", "windows xp", "windows server", "windows vista"}; @@ -116,22 +112,22 @@ public class diskUsage { // public API // ////////////////// - public diskUsage (final plasmaSwitchboard sb) { + public static void init(ArrayList pathsToCheck) { errorMessage = null; - diskUsage.sb = sb; + if (usedOS >= 0) return; // prevent double initialization usedOS = getOS(); if (usedOS == -1) { usable = false; } else { usable = true; - // some kind of *nix if (usedOS <= UNIX_END) { - dfUnix (true); + // some kind of *nix + dfUnixGetVolumes(); for (int i = 0; i < allMountPoints.size(); i++) usedVolumes.add(false); checkVolumesInUseUnix ("DATA"); - checkMapedSubDirs (); + checkMappedSubDirs(pathsToCheck); for (int i = 0; i < allVolumes.size(); i++){ if (usedVolumes.get(i) == true) { @@ -139,52 +135,94 @@ public class diskUsage { yacyUsedMountPoints.add(allMountPoints.get (i)); } } - - // all Windows version } else { - checkWindowsCommandVersion(); + // all Windows versions + initWindowsCommandVersion(); checkStartVolume(); - checkMapedSubDirs (); + checkMappedSubDirs(pathsToCheck); } if (yacyUsedVolumes.size() < 1) usable = false; } } - public HashMap getDiskUsage () { + public static HashMap getDiskUsage () { if (!usable) return null; if (usedOS <= UNIX_END) - dfUnix(false); + return dfUnix(); else - dfWindows (); - return diskUsages; + return dfWindows(); } - public boolean isUsable () { + public static boolean isUsable () { return usable; } - public String getErrorMessage () { + public static String getErrorMessage () { return errorMessage; } - public int getNumUsedVolumes () { + public static int getNumUsedVolumes () { return yacyUsedVolumes.size(); } - - - //////////// // Unix // //////////// - private void dfUnix(boolean getVolumesOnly) { - if (!getVolumesOnly) - diskUsages.clear (); + private static HashMap dfUnix() { + HashMap diskUsages = new HashMap(); + final List lines = dfUnixExec(); + if (consoleError) { + errorMessage = "df:"; + for (final String line: lines){ + errorMessage += "\n" + line; + } + usable = false; + return diskUsages; + } + nextLine: for (final String line: lines){ + if (line.charAt(0) != '/') continue; + final String[] tokens = line.split(" +", 6); + if (tokens.length < 6) continue; + for (int i = 0; i < yacyUsedVolumes.size(); i++){ + if (yacyUsedVolumes.get(i).equals(tokens[0])) { + final long[] vals = new long[2]; + try { vals[0] = new Long(tokens[1]); } catch (final NumberFormatException e) { continue nextLine; } + try { vals[1] = new Long(tokens[3]); } catch (final NumberFormatException e) { continue nextLine; } + vals[0] *= 1024; + vals[1] *= 1024; + diskUsages.put(yacyUsedMountPoints.get(i), vals); + } + } + } + return diskUsages; + } + + private static void dfUnixGetVolumes() { + final List lines = dfUnixExec(); + + nextLine: for (final String line: lines){ + if (line.charAt(0) != '/') continue; + final String[] tokens = line.split(" +", 6); + if (tokens.length < 6) continue; + for (int i = 0; i < allMountPoints.size(); i++) { + if (tokens[5].trim().compareTo(allMountPoints.get(i)) > 0) { + allMountPoints.add(i, tokens[5].trim()); + allVolumes.add(i, tokens[0]); + continue nextLine; + } + } + allMountPoints.add(allMountPoints.size(), tokens[5]); + allVolumes.add(allVolumes.size(), tokens[0]); + } + } + + private static List dfUnixExec() { + // -k set blocksize to 1024 // confirmed with tests: // Linux @@ -219,44 +257,13 @@ public class diskUsage { errorMessage += "\n" + line; } usable = false; - return; + lines.clear(); } - for (final String line: lines){ - if (line.charAt(0) != '/') - continue; - final String[] tokens = line.split(" +", 6); - if (tokens.length < 6) - continue; -nextLine: - if (getVolumesOnly) { - for (int i = 0; i < allMountPoints.size(); i++) { - if (tokens[5].trim().compareTo(allMountPoints.get(i)) > 0) { - allMountPoints.add(i, tokens[5].trim()); - allVolumes.add(i, tokens[0]); - break nextLine; - } - } - allMountPoints.add(allMountPoints.size(), tokens[5]); - allVolumes.add(allVolumes.size(), tokens[0]); - } else { - for (int i = 0; i < yacyUsedVolumes.size(); i++){ - if (yacyUsedVolumes.get(i).equals(tokens[0])) { - final long[] vals = new long[2]; - try { vals[0] = new Long(tokens[1]); } catch (final NumberFormatException e) { break nextLine; } - try { vals[1] = new Long(tokens[3]); } catch (final NumberFormatException e) { break nextLine; } - vals[0] *= 1024; - vals[1] *= 1024; - diskUsages.put (yacyUsedMountPoints.get(i), vals); - } - } - } - } + return lines; } - - - private void checkVolumesInUseUnix (final String path) { + private static void checkVolumesInUseUnix (final String path) { final File file = new File(path); final File[] fileList = file.listFiles(); String base; @@ -292,13 +299,11 @@ nextLine: } } - - /////////////// // Windows // /////////////// - private void checkWindowsCommandVersion () { + private static void initWindowsCommandVersion () { windowsCommand = null; final String os = System.getProperty("os.name").toLowerCase(); final String[] oses = {"windows 95", "windows 98", "windows me"}; @@ -313,7 +318,7 @@ nextLine: windowsCommand = "cmd.exe"; } - private void checkStartVolume() { + private static void checkStartVolume() { final File file = new File("DATA"); String path = null; @@ -327,7 +332,8 @@ nextLine: yacyUsedVolumes.add(path.substring(0, 1)); } - public void dfWindows () { + public static HashMap dfWindows() { + HashMap diskUsages = new HashMap(); for (int i = 0; i < yacyUsedVolumes.size(); i++){ final List processArgs = new ArrayList(); processArgs.add(windowsCommand); @@ -342,7 +348,7 @@ nextLine: errorMessage += "\n" + line; } usable = false; - return; + return diskUsages; } String line = ""; @@ -354,7 +360,7 @@ nextLine: if (line.length() == 0) { errorMessage = "unable to get free size of volume " + yacyUsedVolumes.get(i); usable = false; - return; + return diskUsages; } String[] tokens = line.trim().split(" ++"); @@ -363,14 +369,14 @@ nextLine: try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (final NumberFormatException e) {continue;} diskUsages.put (yacyUsedVolumes.get(i), vals); } + return diskUsages; } - ///////////// // common // ///////////// - private int getOS () { + private static int getOS () { final String os = System.getProperty("os.name").toLowerCase(); for (int i = 0; i < OSname.length; i++) { @@ -381,32 +387,16 @@ nextLine: return -1; } - private void checkMapedSubDirs () { - // FIXME whats about the secondary path??? - // = (getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, ""); - final String[] pathes = {plasmaSwitchboard.HTDOCS_PATH, - plasmaSwitchboard.INDEX_PRIMARY_PATH, - plasmaSwitchboard.LISTS_PATH, - plasmaSwitchboard.PLASMA_PATH, - plasmaSwitchboard.RANKING_PATH, - plasmaSwitchboard.WORK_PATH}; - - String path; - for (final String element : pathes) { - path = null; - try { - path = sb.getConfigPath(element, "").getCanonicalPath().toString(); - } catch (final IOException e) { continue; } - if (path.length() > 0) { - if (usedOS <= UNIX_END) - checkPathUsageUnix (path); - else - checkPathUsageWindows (path); - } + private static void checkMappedSubDirs (ArrayList pathsToCheck) { + for (final String path : pathsToCheck) { + if (usedOS <= UNIX_END) + checkPathUsageUnix (path); + else + checkPathUsageWindows (path); } } - private void checkPathUsageUnix (final String path) { + private static void checkPathUsageUnix (final String path) { for (int i = 0; i < allMountPoints.size(); i++){ if (path.startsWith (allMountPoints.get(i))) { usedVolumes.set(i, true); @@ -415,9 +405,9 @@ nextLine: } } - private void checkPathUsageWindows (final String path) { + private static void checkPathUsageWindows (final String path) { int index = -1; - String sub = path.substring(0, 1); + String sub = path.substring(0, 1); // ?? nur ein character? try { index = yacyUsedVolumes.indexOf(sub); } catch (IndexOutOfBoundsException e) { errorMessage = "internal error while checking used windows volumes"; usable = false; @@ -427,7 +417,7 @@ nextLine: yacyUsedVolumes.add(sub); } - private List getConsoleOutput (final List processArgs) { + private static List getConsoleOutput (final List processArgs) { final ProcessBuilder processBuilder = new ProcessBuilder(processArgs); Process process = null; consoleInterface inputStream = null; @@ -437,8 +427,8 @@ nextLine: try { process = processBuilder.start(); - inputStream = new consoleInterface(process.getInputStream(), "input"); - errorStream = new consoleInterface(process.getErrorStream(), "error"); + inputStream = new consoleInterface(process.getInputStream(), "input", log); + errorStream = new consoleInterface(process.getErrorStream(), "error", log); inputStream.start(); errorStream.start(); @@ -467,74 +457,5 @@ nextLine: return list; } - public class consoleInterface extends Thread - { - private final InputStream stream; - private final List output = new ArrayList(); - private final Semaphore dataIsRead = new Semaphore(1); - /** - * FIXME just for debugging - */ - private final String name; - - public consoleInterface (final InputStream stream, String name) - { - this.stream = stream; - this.name = name; - // block reading {@see getOutput()} - try { - dataIsRead.acquire(); - } catch (InterruptedException e) { - // this should never happen because this is a constructor - e.printStackTrace(); - } - } - - public void run() { - // a second run adds data! a output.clear() maybe needed - try { - final InputStreamReader input = new InputStreamReader(stream); - final BufferedReader buffer = new BufferedReader(input); - String line = null; - int tries = 0; - while (tries < 1000) { - tries++; - try { - // may block! - Thread.sleep(1); - } catch (InterruptedException e) { - // just stop sleeping - } - if (buffer.ready()) - break; - } - log.logInfo("logpoint 3 "+ name +" needed " + tries + " tries"); - while((line = buffer.readLine()) != null) { - output.add(line); - } - dataIsRead.release(); - } catch(final IOException ix) { log.logWarning("logpoint 6 " + ix.getMessage());} - } - - /** - * waits until the stream is read and returns all data - * - * @return lines of text in stream - */ - public List getOutput(){ - // wait that data is ready - try { - log.logInfo("logpoint 4 waiting for data of '"+ name +"'"); - final long start = System.currentTimeMillis(); - dataIsRead.acquire(); - log.logInfo("logpoint 5 data ready for '"+ name +"' after "+ (System.currentTimeMillis() - start) +" ms"); - } catch (InterruptedException e) { - // after interrupt just return what is available (maybe nothing) - } - // is just for checking availability, so release it immediatly - dataIsRead.release(); - return output; - } - } } diff --git a/source/de/anomic/yacy/resourceObserver.java b/source/de/anomic/yacy/resourceObserver.java index d3e4bde0b..c961df3a5 100644 --- a/source/de/anomic/yacy/resourceObserver.java +++ b/source/de/anomic/yacy/resourceObserver.java @@ -39,6 +39,8 @@ package de.anomic.yacy; +import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -51,7 +53,7 @@ public final class resourceObserver { // TODO make it configurable private final static long MIN_FREE_DISK_SPACE = 100L /* MiB */ * 1024L * 1024L; // Unknown for now - private final static long MIN_FREE_MEMORY = 0; + //private final static long MIN_FREE_MEMORY = 0; // We are called with the cleanup job every five minutes; // the disk usage should be checked with every run private final int CHECK_DISK_USAGE_FREQ = 1; @@ -59,7 +61,6 @@ public final class resourceObserver { private final int CHECK_MEMORY_USAGE_FREQ = 1; private final serverLog log = new serverLog("RESOURCE OBSERVER"); - private final diskUsage du; private final plasmaSwitchboard sb; private int checkDiskUsageCount; @@ -67,13 +68,31 @@ public final class resourceObserver { private boolean disksOK; private boolean memoryOK; - public resourceObserver(final plasmaSwitchboard sb) { + public resourceObserver(plasmaSwitchboard sb) { this.sb = sb; this.log.logInfo("initializing the resource observer"); - du = new diskUsage(sb); + + ArrayList pathsToCheck = new ArrayList(); + // FIXME whats about the secondary path??? + // = (getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, ""); + final String[] pathes = {plasmaSwitchboard.HTDOCS_PATH, + plasmaSwitchboard.INDEX_PRIMARY_PATH, + plasmaSwitchboard.LISTS_PATH, + plasmaSwitchboard.PLASMA_PATH, + plasmaSwitchboard.RANKING_PATH, + plasmaSwitchboard.WORK_PATH}; + String path; + for (final String element : pathes) { + try { + path = sb.getConfigPath(element, "").getCanonicalPath().toString(); + if (path.length() > 0) pathsToCheck.add(path); + } catch (final IOException e) {} + } + + diskUsage.init(pathsToCheck); - if (!du.isUsable ()) - this.log.logWarning("Disk usage returned: " + du.getErrorMessage()); + if (!diskUsage.isUsable ()) + this.log.logWarning("Disk usage returned: " + diskUsage.getErrorMessage()); checkDiskUsageCount = 0; checkMemoryUsageCount = 0; @@ -108,7 +127,7 @@ public final class resourceObserver { } } else { - if (du.isUsable ()) + if (diskUsage.isUsable()) this.log.logInfo("run completed; everything in order"); else this.log.logInfo("The observer is out of order"); @@ -131,20 +150,21 @@ public final class resourceObserver { } /** - * @return enough disk space availabe? + * @return enough disk space available? */ private boolean checkDisks() { boolean below = false; - if (!du.isUsable ()) + if (!diskUsage.isUsable ()) return true; - final HashMap usage = du.getDiskUsage(); + final HashMap usage = diskUsage.getDiskUsage(); long[] val; for (Map.Entry entry: usage.entrySet()) { val = entry.getValue(); + this.log.logInfo("df of Volume " + entry.getKey() + ": " + (val[1] / 1024 / 1024) + " MB"); if (val[1] < MIN_FREE_DISK_SPACE) { - this.log.logWarning("Volume " + entry.getKey() + ": free space is too low"); + this.log.logWarning("Volume " + entry.getKey() + ": free space (" + (val[1] / 1024 / 1024) + " MB) is too low (< " + (MIN_FREE_DISK_SPACE / 1024 / 1024) + " MB)"); below = true; } } diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index f909eabd4..5e9ded635 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -373,6 +373,7 @@ public class yacyDHTAction { if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + firstKey + ", last_hash = " + lastKey); while ((e.hasNext()) && (seeds.size() < (primaryPeerCount + reservePeerCount)) && (maxloop-- > 0)) { seed = e.next(); + if (seed == null || seed.hash == null) continue; firstdist = yacyDHTAction.dhtDistance(seed.hash, firstKey); lastdist = yacyDHTAction.dhtDistance(seed.hash, lastKey); if (lastdist > maxDist) {