From 6b7e8739626fbb3aa09cb5fbebc74b823a70261d Mon Sep 17 00:00:00 2001 From: danielr Date: Thu, 19 Jun 2008 12:40:44 +0000 Subject: [PATCH] resourceObserver refactoring and some synchronisation for console output git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4939 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/tools/diskUsage.java | 189 +++++++++++--------- source/de/anomic/yacy/resourceObserver.java | 37 ++-- 2 files changed, 121 insertions(+), 105 deletions(-) diff --git a/source/de/anomic/tools/diskUsage.java b/source/de/anomic/tools/diskUsage.java index 09fd84b99..84d85ba2e 100644 --- a/source/de/anomic/tools/diskUsage.java +++ b/source/de/anomic/tools/diskUsage.java @@ -53,10 +53,9 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; +import java.util.List; import de.anomic.plasma.plasmaSwitchboard; -// FIXME entfernen import de.anomic.server.logging.serverLog; public class diskUsage { @@ -64,12 +63,12 @@ public class diskUsage { serverLog log = new serverLog("DISK USAGE"); private static final HashMap diskUsages = new HashMap(); - private static final ArrayList allVolumes = new ArrayList(); - private static final ArrayList allMountPoints = new ArrayList(); - private static final ArrayList usedVolumes = new ArrayList(); + private static final List allVolumes = new ArrayList(); + private static final List allMountPoints = new ArrayList(); + private static final List usedVolumes = new ArrayList(); - private static final ArrayList yacyUsedVolumes = new ArrayList(); - private static final ArrayList yacyUsedMountPoints = new ArrayList(); + private static final List yacyUsedVolumes = new ArrayList(); + private static final List yacyUsedMountPoints = new ArrayList(); private static plasmaSwitchboard sb; private static int usedOS; @@ -117,7 +116,7 @@ public class diskUsage { // public API // ////////////////// - public diskUsage (plasmaSwitchboard sb) { + public diskUsage (final plasmaSwitchboard sb) { errorMessage = null; diskUsage.sb = sb; usedOS = getOS(); @@ -170,7 +169,7 @@ public class diskUsage { return diskUsages; } - public boolean getUsable () { + public boolean isUsable () { return usable; } @@ -208,7 +207,7 @@ public class diskUsage { // please report all successes or fails for non-confirmed systems to // detlef!reichl()gmx!org. Thanks! - ArrayList processArgs = new ArrayList(); + final List processArgs = new ArrayList(); processArgs.add("df"); processArgs.add("-k"); // Some systems need the additional -P parameter to return the data in Posix format. @@ -220,23 +219,20 @@ public class diskUsage { if (usedOS != TRU64 && usedOS != HAIKU) processArgs.add("-l"); - ArrayList lines = getConsoleOutput(processArgs); + final List lines = getConsoleOutput(processArgs); if (consoleError) { errorMessage = "df:"; - Iterator iter = lines.iterator(); - while (iter.hasNext()){ - errorMessage += "\n" + iter.next(); - usable = false; - return; + for (final String line: lines){ + errorMessage += "\n" + line; } + usable = false; + return; } - Iterator iter = lines.iterator(); - while (iter.hasNext()){ - String line = iter.next(); - if (! line.startsWith ("/")) + for (final String line: lines){ + if (line.charAt(0) != '/') continue; - String[] tokens = line.split(" ++", 6); + final String[] tokens = line.split(" +", 6); if (tokens.length < 6) continue; nextLine: @@ -245,7 +241,7 @@ nextLine: if (tokens[5].trim().compareTo(allMountPoints.get(i)) > 0) { allMountPoints.add(i, tokens[5].trim()); allVolumes.add(i, tokens[0]); - break nextLine; + break nextLine;// TODO what does this mean? continue? stop parsing lines? --danielr } } allMountPoints.add(allMountPoints.size(), tokens[5]); @@ -253,9 +249,9 @@ nextLine: } else { for (int i = 0; i < yacyUsedVolumes.size(); i++){ if (yacyUsedVolumes.get(i).equals(tokens[0])) { - long[] vals = new long[2]; - try { vals[0] = new Long(tokens[1]); } catch (NumberFormatException e) { break nextLine; } - try { vals[1] = new Long(tokens[3]); } catch (NumberFormatException e) { break nextLine; } + final long[] vals = new long[2]; + try { vals[0] = new Long(tokens[1]); } catch (final NumberFormatException e) { break nextLine; } + try { vals[1] = new Long(tokens[3]); } catch (final NumberFormatException e) { break nextLine; } vals[0] *= 1024; vals[1] *= 1024; diskUsages.put (yacyUsedMountPoints.get(i), vals); @@ -267,34 +263,34 @@ nextLine: - private void checkVolumesInUseUnix (String path) { - File file = new File(path); - File[] fileList = file.listFiles(); + private void checkVolumesInUseUnix (final String path) { + final File file = new File(path); + final File[] fileList = file.listFiles(); String base; String dir; - for (int i = 0; i < fileList.length; i++) { + for (final File element : fileList) { // ATTENTION! THIS LOOP NEEDS A TIME-OUT - if (fileList[i].isDirectory()) { + if (element.isDirectory()) { try { - dir = fileList[i].getCanonicalPath(); - } catch (IOException e) { + dir = element.getCanonicalPath(); + } catch (final IOException e) { usable = false; break; } - if (!dir.endsWith ("HTCACHE") - && !dir.endsWith ("LOCALE") - && !dir.endsWith ("RANKING") - && !dir.endsWith ("RELEASE") - && !dir.endsWith ("collection.0028.commons")) { - checkVolumesInUseUnix (dir); - } else { + if (dir.endsWith ("HTCACHE") + || dir.endsWith ("LOCALE") + || dir.endsWith ("RANKING") + || dir.endsWith ("RELEASE") + || dir.endsWith ("collection.0028.commons")) { checkPathUsage (dir); + } else { + checkVolumesInUseUnix (dir); } } else { try { - base = fileList[i].getCanonicalPath(); - } catch (IOException e) { + base = element.getCanonicalPath(); + } catch (final IOException e) { usable = false; break; } @@ -311,12 +307,11 @@ nextLine: private void checkWindowsCommandVersion () { windowsCommand = null; - String os = System.getProperty("os.name").toLowerCase(); - String[] oses = {"windows 95", "windows 98", "windows me"}; + final String os = System.getProperty("os.name").toLowerCase(); + final String[] oses = {"windows 95", "windows 98", "windows me"}; - for (int i = 0; i < oses.length; i++) - { - if (os.indexOf(oses[i]) >= 0){ + for (final String element : oses) { + if (os.indexOf(element) >= 0){ windowsCommand = "command.com"; break; } @@ -325,27 +320,27 @@ nextLine: windowsCommand = "cmd.exe"; } - private String dfWindowsGetConsoleOutput (String device) { - ArrayList processArgs = new ArrayList(); + private String dfWindowsGetConsoleOutput (final String device) { + final List processArgs = new ArrayList(); processArgs.add(windowsCommand); processArgs.add("/c"); processArgs.add("dir"); processArgs.add(device); - ProcessBuilder processBuilder = new ProcessBuilder(processArgs); + final ProcessBuilder processBuilder = new ProcessBuilder(processArgs); Process process; try { process = processBuilder.start(); - } catch (IOException e) {return null;} + } catch (final IOException e) {return null;} - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); String line; String lastLine = null; while (true) { try { line = bufferedReader.readLine (); - } catch (IOException e) { return null; } + } catch (final IOException e) { return null; } if (line == null) break; if (line.trim().length() > 0) @@ -355,8 +350,9 @@ nextLine: } private void getAllVolumesWindows () { + String dirName; for (char c = 'C'; c <= 'Z'; c++) { // A and B are reserved for floppy on Windows - String dirName = c + ":\\"; + dirName = c + ":\\"; if (dfWindowsGetConsoleOutput (dirName) != null) { allVolumes.add(String.valueOf(c)); } @@ -364,10 +360,10 @@ nextLine: } private void checkStartVolume() { - File file = new File("DATA"); + final File file = new File("DATA"); String path = null; - try { path = file.getCanonicalPath().toString(); } catch (IOException e) { + try { path = file.getCanonicalPath().toString(); } catch (final IOException e) { errorMessage = "Cant get DATA directory"; usable = false; return; @@ -376,7 +372,7 @@ nextLine: return; int index = -1; - try { index = allVolumes.indexOf(path.substring(0, 1)); } catch (IndexOutOfBoundsException e) { + try { index = allVolumes.indexOf(path.substring(0, 1)); } catch (final IndexOutOfBoundsException e) { errorMessage = "Start volume not found in all volumes"; usable = false; return; @@ -392,12 +388,12 @@ nextLine: public void dfWindows () { for (int i = 0; i < yacyUsedVolumes.size(); i++){ // in yacyUsedMountPoints aendern - String line = dfWindowsGetConsoleOutput(yacyUsedVolumes.get(i) + ":\\"); - String[] tokens = line.trim().split(" ++"); + final String line = dfWindowsGetConsoleOutput(yacyUsedVolumes.get(i) + ":\\"); + final String[] tokens = line.trim().split(" ++"); - long[] vals = new long[2]; + final long[] vals = new long[2]; vals[0] = -1; - try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (NumberFormatException e) {continue;} + try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (final NumberFormatException e) {continue;} diskUsages.put (yacyUsedVolumes.get(i), vals); } } @@ -409,7 +405,7 @@ nextLine: ///////////// private int getOS () { - String os = System.getProperty("os.name").toLowerCase(); + final String os = System.getProperty("os.name").toLowerCase(); for (int i = 0; i < OSname.length; i++) { if (os.indexOf(OSname[i]) >= 0) @@ -422,24 +418,25 @@ nextLine: private void checkMapedSubDirs () { // FIXME whats about the secondary path??? // = (getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, ""); - String[] pathes = {plasmaSwitchboard.HTDOCS_PATH, + final String[] pathes = {plasmaSwitchboard.HTDOCS_PATH, plasmaSwitchboard.INDEX_PRIMARY_PATH, plasmaSwitchboard.LISTS_PATH, plasmaSwitchboard.PLASMA_PATH, plasmaSwitchboard.RANKING_PATH, plasmaSwitchboard.WORK_PATH}; - for (int i = 0; i < pathes.length; i++) { - String path = null; + String path; + for (final String element : pathes) { + path = null; try { - path = sb.getConfigPath(pathes[i], "").getCanonicalPath().toString(); - } catch (IOException e) { continue; } + path = sb.getConfigPath(element, "").getCanonicalPath().toString(); + } catch (final IOException e) { continue; } if (path.length() > 0) checkPathUsage (path); } } - private void checkPathUsage (String path) { + private void checkPathUsage (final String path) { for (int i = 0; i < allMountPoints.size(); i++){ if (path.startsWith (allMountPoints.get(i))) { usedVolumes.set(i, true); @@ -449,9 +446,8 @@ nextLine: } - private ArrayList getConsoleOutput (ArrayList processArgs) { - ArrayList list = new ArrayList(); - ProcessBuilder processBuilder = new ProcessBuilder(processArgs); + private List getConsoleOutput (final List processArgs) { + final ProcessBuilder processBuilder = new ProcessBuilder(processArgs); Process process = null; consoleInterface inputStream = null; consoleInterface errorStream = null; @@ -460,26 +456,28 @@ nextLine: try { process = processBuilder.start(); - inputStream = new consoleInterface(process.getInputStream()); - errorStream = new consoleInterface(process.getErrorStream()); + inputStream = new consoleInterface(process.getInputStream(), "input"); + errorStream = new consoleInterface(process.getErrorStream(), "error"); inputStream.start(); errorStream.start(); /*int retval =*/ process.waitFor(); - } catch(IOException iox) { + } catch(final IOException iox) { consoleError = true; log.logWarning("logpoint 0 " + iox.getMessage()); + List list = new ArrayList(); list.add(iox.getMessage()); return list; - } catch(InterruptedException ix) { + } catch(final InterruptedException ix) { consoleError = true; log.logWarning("logpoint 1 " + ix.getMessage()); + List list = new ArrayList(); list.add(ix.getMessage()); return list; } - list = inputStream.getOutput(); + List list = inputStream.getOutput(); if (list.isEmpty()) { consoleError = true; log.logWarning("logpoint 2 "); @@ -490,36 +488,53 @@ nextLine: public class consoleInterface extends Thread { - private InputStream stream; - private boolean getInputStream; - private ArrayList output; + private final InputStream stream; + private final List output = new ArrayList(); + private final String name; + private boolean done = false; - public consoleInterface (InputStream stream) + public consoleInterface (final InputStream stream, String name) { this.stream = stream; - output = new ArrayList(); + this.name = name; } public void run() { try { - InputStreamReader input = new InputStreamReader(stream); - BufferedReader buffer = new BufferedReader(input); + final InputStreamReader input = new InputStreamReader(stream); + final BufferedReader buffer = new BufferedReader(input); String line = null; - int tries = 1000; - while (tries-- > 0) { + int tries = 0; + while (tries < 1000) { + tries++; + try { + Thread.sleep(1); + } catch (InterruptedException e) { + // just stop sleeping + } if (buffer.ready()) break; } - log.logInfo("logpoint 3 " + tries + " tries"); + log.logInfo("logpoint 3 "+ name +" needed " + tries + " tries"); while((line = buffer.readLine()) != null) { output.add(line); } - } catch(IOException ix) { log.logWarning("logpoint 4 " + ix.getMessage());} + log.logInfo("logpoint 4 output done of '"+ name +"'"); + done = true; + } catch(final IOException ix) { log.logWarning("logpoint 5 " + ix.getMessage());} } - public ArrayList getOutput(){ + public List getOutput(){ + log.logInfo("logpoint 6 getOutput() of '"+ name +"' requested"); + while(!isDone()) { + // wait + } return output; } + + private boolean isDone() { + return done; + } } } diff --git a/source/de/anomic/yacy/resourceObserver.java b/source/de/anomic/yacy/resourceObserver.java index 90133ef06..68df46fdc 100644 --- a/source/de/anomic/yacy/resourceObserver.java +++ b/source/de/anomic/yacy/resourceObserver.java @@ -40,7 +40,6 @@ package de.anomic.yacy; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import de.anomic.plasma.plasmaSwitchboard; @@ -59,21 +58,21 @@ public final class resourceObserver { // The memory usage should be checked on every run private final int CHECK_MEMORY_USAGE_FREQ = 1; - private serverLog log = new serverLog("RESOURCE OBSERVER"); - private diskUsage du; - private plasmaSwitchboard sb; + private final serverLog log = new serverLog("RESOURCE OBSERVER"); + private final diskUsage du; + private final plasmaSwitchboard sb; private int checkDiskUsageCount; private int checkMemoryUsageCount; private boolean disksOK; private boolean memoryOK; - public resourceObserver(plasmaSwitchboard sb) { + public resourceObserver(final plasmaSwitchboard sb) { this.sb = sb; this.log.logInfo("initializing the resource observer"); du = new diskUsage(sb); - if (!du.getUsable ()) + if (!du.isUsable ()) this.log.logWarning("Disk usage returned: " + du.getErrorMessage()); checkDiskUsageCount = 0; @@ -82,7 +81,7 @@ public final class resourceObserver { memoryOK = true; } - public boolean resourceObserverJob() { + public void resourceObserverJob() { checkDiskUsageCount++; checkMemoryUsageCount++; boolean tmpDisksOK = true; @@ -109,12 +108,11 @@ public final class resourceObserver { } } else { - if (du.getUsable ()) + if (du.isUsable ()) this.log.logInfo("run completed; everything in order"); else this.log.logInfo("The observer is out of order"); } - return true; } public boolean getDisksOK () { @@ -125,25 +123,28 @@ public final class resourceObserver { return memoryOK; } + /** + * @return amount of space that should be kept free + */ public long getMinFreeDiskSpace () { return MIN_FREE_DISK_SPACE; } + /** + * @return enough disk space availabe? + */ private boolean checkDisks() { boolean below = false; - if (!du.getUsable ()) + if (!du.isUsable ()) return true; - HashMap usage = du.getDiskUsage(); - Iterator> iter = usage.entrySet().iterator(); - Map.Entry entry; - while (iter.hasNext()) { - entry = iter.next (); - String key = entry.getKey(); - long[] val = entry.getValue(); + final HashMap usage = du.getDiskUsage(); + long[] val; + for (Map.Entry entry: usage.entrySet()) { + val = entry.getValue(); if (val[1] < MIN_FREE_DISK_SPACE) { - this.log.logWarning("Volume " + key + ": free space is too low"); + this.log.logWarning("Volume " + entry.getKey() + ": free space is too low"); below = true; } }