boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

resourceObserver refactoring and some synchronisation for console output

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4939 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
danielr 17 years ago
parent 01e9c7320e
commit 6b7e873962
2 changed files with 121 additions and 105 deletions
Show Stats Download Patch File Download Diff File

189
source/de/anomic/tools/diskUsage.java
Unescape View File

@ -53,10 +53,9 @@ import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.List;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
// FIXME entfernen
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
public class diskUsage { public class diskUsage {
@ -64,12 +63,12 @@ public class diskUsage {
serverLog log = new serverLog("DISK USAGE"); serverLog log = new serverLog("DISK USAGE");
private static final HashMap<String, long[]> diskUsages = new HashMap<String, long[]>(); private static final HashMap<String, long[]> diskUsages = new HashMap<String, long[]>();
private static final ArrayList<String> allVolumes = new ArrayList<String>(); private static final List<String> allVolumes = new ArrayList<String>();
private static final ArrayList<String> allMountPoints = new ArrayList<String>(); private static final List<String> allMountPoints = new ArrayList<String>();
private static final ArrayList<Boolean> usedVolumes = new ArrayList<Boolean>(); private static final List<Boolean> usedVolumes = new ArrayList<Boolean>();
private static final ArrayList<String> yacyUsedVolumes = new ArrayList<String>(); private static final List<String> yacyUsedVolumes = new ArrayList<String>();
private static final ArrayList<String> yacyUsedMountPoints = new ArrayList<String>(); private static final List<String> yacyUsedMountPoints = new ArrayList<String>();
private static plasmaSwitchboard sb; private static plasmaSwitchboard sb;
private static int usedOS; private static int usedOS;
@ -117,7 +116,7 @@ public class diskUsage {
// public API // // public API //
////////////////// //////////////////
public diskUsage (plasmaSwitchboard sb) { public diskUsage (final plasmaSwitchboard sb) {
errorMessage = null; errorMessage = null;
diskUsage.sb = sb; diskUsage.sb = sb;
usedOS = getOS(); usedOS = getOS();
@ -170,7 +169,7 @@ public class diskUsage {
return diskUsages; return diskUsages;
} }
public boolean getUsable () { public boolean isUsable () {
return usable; return usable;
} }
@ -208,7 +207,7 @@ public class diskUsage {
// please report all successes or fails for non-confirmed systems to // please report all successes or fails for non-confirmed systems to
// detlef!reichl()gmx!org. Thanks! // detlef!reichl()gmx!org. Thanks!
ArrayList<String> processArgs = new ArrayList<String>(); final List<String> processArgs = new ArrayList<String>();
processArgs.add("df"); processArgs.add("df");
processArgs.add("-k"); processArgs.add("-k");
// Some systems need the additional -P parameter to return the data in Posix format. // Some systems need the additional -P parameter to return the data in Posix format.
@ -220,23 +219,20 @@ public class diskUsage {
if (usedOS != TRU64 && usedOS != HAIKU) if (usedOS != TRU64 && usedOS != HAIKU)
processArgs.add("-l"); processArgs.add("-l");
ArrayList<String> lines = getConsoleOutput(processArgs); final List<String> lines = getConsoleOutput(processArgs);
if (consoleError) { if (consoleError) {
errorMessage = "df:"; errorMessage = "df:";
Iterator<String> iter = lines.iterator(); for (final String line: lines){
while (iter.hasNext()){ errorMessage += "\n" + line;
errorMessage += "\n" + iter.next();
usable = false;
return;
} }
usable = false;
return;
} }
Iterator<String> iter = lines.iterator(); for (final String line: lines){
while (iter.hasNext()){ if (line.charAt(0) != '/')
String line = iter.next();
if (! line.startsWith ("/"))
continue; continue;
String[] tokens = line.split(" ++", 6); final String[] tokens = line.split(" +", 6);
if (tokens.length < 6) if (tokens.length < 6)
continue; continue;
nextLine: nextLine:
@ -245,7 +241,7 @@ nextLine:
if (tokens[5].trim().compareTo(allMountPoints.get(i)) > 0) { if (tokens[5].trim().compareTo(allMountPoints.get(i)) > 0) {
allMountPoints.add(i, tokens[5].trim()); allMountPoints.add(i, tokens[5].trim());
allVolumes.add(i, tokens[0]); allVolumes.add(i, tokens[0]);
break nextLine; break nextLine;// TODO what does this mean? continue? stop parsing lines? --danielr
} }
} }
allMountPoints.add(allMountPoints.size(), tokens[5]); allMountPoints.add(allMountPoints.size(), tokens[5]);
@ -253,9 +249,9 @@ nextLine:
} else { } else {
for (int i = 0; i < yacyUsedVolumes.size(); i++){ for (int i = 0; i < yacyUsedVolumes.size(); i++){
if (yacyUsedVolumes.get(i).equals(tokens[0])) { if (yacyUsedVolumes.get(i).equals(tokens[0])) {
long[] vals = new long[2]; final long[] vals = new long[2];
try { vals[0] = new Long(tokens[1]); } catch (NumberFormatException e) { break nextLine; } try { vals[0] = new Long(tokens[1]); } catch (final NumberFormatException e) { break nextLine; }
try { vals[1] = new Long(tokens[3]); } catch (NumberFormatException e) { break nextLine; } try { vals[1] = new Long(tokens[3]); } catch (final NumberFormatException e) { break nextLine; }
vals[0] *= 1024; vals[0] *= 1024;
vals[1] *= 1024; vals[1] *= 1024;
diskUsages.put (yacyUsedMountPoints.get(i), vals); diskUsages.put (yacyUsedMountPoints.get(i), vals);
@ -267,34 +263,34 @@ nextLine:
private void checkVolumesInUseUnix (String path) { private void checkVolumesInUseUnix (final String path) {
File file = new File(path); final File file = new File(path);
File[] fileList = file.listFiles(); final File[] fileList = file.listFiles();
String base; String base;
String dir; String dir;
for (int i = 0; i < fileList.length; i++) { for (final File element : fileList) {
// ATTENTION! THIS LOOP NEEDS A TIME-OUT // ATTENTION! THIS LOOP NEEDS A TIME-OUT
if (fileList[i].isDirectory()) { if (element.isDirectory()) {
try { try {
dir = fileList[i].getCanonicalPath(); dir = element.getCanonicalPath();
} catch (IOException e) { } catch (final IOException e) {
usable = false; usable = false;
break; break;
} }
if (!dir.endsWith ("HTCACHE") if (dir.endsWith ("HTCACHE")
&& !dir.endsWith ("LOCALE") || dir.endsWith ("LOCALE")
&& !dir.endsWith ("RANKING") || dir.endsWith ("RANKING")
&& !dir.endsWith ("RELEASE") || dir.endsWith ("RELEASE")
&& !dir.endsWith ("collection.0028.commons")) { || dir.endsWith ("collection.0028.commons")) {
checkVolumesInUseUnix (dir);
} else {
checkPathUsage (dir); checkPathUsage (dir);
} else {
checkVolumesInUseUnix (dir);
} }
} else { } else {
try { try {
base = fileList[i].getCanonicalPath(); base = element.getCanonicalPath();
} catch (IOException e) { } catch (final IOException e) {
usable = false; usable = false;
break; break;
} }
@ -311,12 +307,11 @@ nextLine:
private void checkWindowsCommandVersion () { private void checkWindowsCommandVersion () {
windowsCommand = null; windowsCommand = null;
String os = System.getProperty("os.name").toLowerCase(); final String os = System.getProperty("os.name").toLowerCase();
String[] oses = {"windows 95", "windows 98", "windows me"}; final String[] oses = {"windows 95", "windows 98", "windows me"};
for (int i = 0; i < oses.length; i++) for (final String element : oses) {
{ if (os.indexOf(element) >= 0){
if (os.indexOf(oses[i]) >= 0){
windowsCommand = "command.com"; windowsCommand = "command.com";
break; break;
} }
@ -325,27 +320,27 @@ nextLine:
windowsCommand = "cmd.exe"; windowsCommand = "cmd.exe";
} }
private String dfWindowsGetConsoleOutput (String device) { private String dfWindowsGetConsoleOutput (final String device) {
ArrayList<String> processArgs = new ArrayList<String>(); final List<String> processArgs = new ArrayList<String>();
processArgs.add(windowsCommand); processArgs.add(windowsCommand);
processArgs.add("/c"); processArgs.add("/c");
processArgs.add("dir"); processArgs.add("dir");
processArgs.add(device); processArgs.add(device);
ProcessBuilder processBuilder = new ProcessBuilder(processArgs); final ProcessBuilder processBuilder = new ProcessBuilder(processArgs);
Process process; Process process;
try { try {
process = processBuilder.start(); process = processBuilder.start();
} catch (IOException e) {return null;} } catch (final IOException e) {return null;}
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line; String line;
String lastLine = null; String lastLine = null;
while (true) { while (true) {
try { try {
line = bufferedReader.readLine (); line = bufferedReader.readLine ();
} catch (IOException e) { return null; } } catch (final IOException e) { return null; }
if (line == null) if (line == null)
break; break;
if (line.trim().length() > 0) if (line.trim().length() > 0)
@ -355,8 +350,9 @@ nextLine:
} }
private void getAllVolumesWindows () { private void getAllVolumesWindows () {
String dirName;
for (char c = 'C'; c <= 'Z'; c++) { // A and B are reserved for floppy on Windows for (char c = 'C'; c <= 'Z'; c++) { // A and B are reserved for floppy on Windows
String dirName = c + ":\\"; dirName = c + ":\\";
if (dfWindowsGetConsoleOutput (dirName) != null) { if (dfWindowsGetConsoleOutput (dirName) != null) {
allVolumes.add(String.valueOf(c)); allVolumes.add(String.valueOf(c));
} }
@ -364,10 +360,10 @@ nextLine:
} }
private void checkStartVolume() { private void checkStartVolume() {
File file = new File("DATA"); final File file = new File("DATA");
String path = null; String path = null;
try { path = file.getCanonicalPath().toString(); } catch (IOException e) { try { path = file.getCanonicalPath().toString(); } catch (final IOException e) {
errorMessage = "Cant get DATA directory"; errorMessage = "Cant get DATA directory";
usable = false; usable = false;
return; return;
@ -376,7 +372,7 @@ nextLine:
return; return;
int index = -1; int index = -1;
try { index = allVolumes.indexOf(path.substring(0, 1)); } catch (IndexOutOfBoundsException e) { try { index = allVolumes.indexOf(path.substring(0, 1)); } catch (final IndexOutOfBoundsException e) {
errorMessage = "Start volume not found in all volumes"; errorMessage = "Start volume not found in all volumes";
usable = false; usable = false;
return; return;
@ -392,12 +388,12 @@ nextLine:
public void dfWindows () { public void dfWindows () {
for (int i = 0; i < yacyUsedVolumes.size(); i++){ for (int i = 0; i < yacyUsedVolumes.size(); i++){
// in yacyUsedMountPoints aendern // in yacyUsedMountPoints aendern
String line = dfWindowsGetConsoleOutput(yacyUsedVolumes.get(i) + ":\\"); final String line = dfWindowsGetConsoleOutput(yacyUsedVolumes.get(i) + ":\\");
String[] tokens = line.trim().split(" ++"); final String[] tokens = line.trim().split(" ++");
long[] vals = new long[2]; final long[] vals = new long[2];
vals[0] = -1; vals[0] = -1;
try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (NumberFormatException e) {continue;} try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (final NumberFormatException e) {continue;}
diskUsages.put (yacyUsedVolumes.get(i), vals); diskUsages.put (yacyUsedVolumes.get(i), vals);
} }
} }
@ -409,7 +405,7 @@ nextLine:
///////////// /////////////
private int getOS () { private int getOS () {
String os = System.getProperty("os.name").toLowerCase(); final String os = System.getProperty("os.name").toLowerCase();
for (int i = 0; i < OSname.length; i++) for (int i = 0; i < OSname.length; i++)
{ {
if (os.indexOf(OSname[i]) >= 0) if (os.indexOf(OSname[i]) >= 0)
@ -422,24 +418,25 @@ nextLine:
private void checkMapedSubDirs () { private void checkMapedSubDirs () {
// FIXME whats about the secondary path??? // FIXME whats about the secondary path???
// = (getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, ""); // = (getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, "");
String[] pathes = {plasmaSwitchboard.HTDOCS_PATH, final String[] pathes = {plasmaSwitchboard.HTDOCS_PATH,
plasmaSwitchboard.INDEX_PRIMARY_PATH, plasmaSwitchboard.INDEX_PRIMARY_PATH,
plasmaSwitchboard.LISTS_PATH, plasmaSwitchboard.LISTS_PATH,
plasmaSwitchboard.PLASMA_PATH, plasmaSwitchboard.PLASMA_PATH,
plasmaSwitchboard.RANKING_PATH, plasmaSwitchboard.RANKING_PATH,
plasmaSwitchboard.WORK_PATH}; plasmaSwitchboard.WORK_PATH};
for (int i = 0; i < pathes.length; i++) { String path;
String path = null; for (final String element : pathes) {
path = null;
try { try {
path = sb.getConfigPath(pathes[i], "").getCanonicalPath().toString(); path = sb.getConfigPath(element, "").getCanonicalPath().toString();
} catch (IOException e) { continue; } } catch (final IOException e) { continue; }
if (path.length() > 0) if (path.length() > 0)
checkPathUsage (path); checkPathUsage (path);
} }
} }
private void checkPathUsage (String path) { private void checkPathUsage (final String path) {
for (int i = 0; i < allMountPoints.size(); i++){ for (int i = 0; i < allMountPoints.size(); i++){
if (path.startsWith (allMountPoints.get(i))) { if (path.startsWith (allMountPoints.get(i))) {
usedVolumes.set(i, true); usedVolumes.set(i, true);
@ -449,9 +446,8 @@ nextLine:
} }
private ArrayList<String> getConsoleOutput (ArrayList<String> processArgs) { private List<String> getConsoleOutput (final List<String> processArgs) {
ArrayList<String> list = new ArrayList<String>(); final ProcessBuilder processBuilder = new ProcessBuilder(processArgs);
ProcessBuilder processBuilder = new ProcessBuilder(processArgs);
Process process = null; Process process = null;
consoleInterface inputStream = null; consoleInterface inputStream = null;
consoleInterface errorStream = null; consoleInterface errorStream = null;
@ -460,26 +456,28 @@ nextLine:
try { try {
process = processBuilder.start(); process = processBuilder.start();
inputStream = new consoleInterface(process.getInputStream()); inputStream = new consoleInterface(process.getInputStream(), "input");
errorStream = new consoleInterface(process.getErrorStream()); errorStream = new consoleInterface(process.getErrorStream(), "error");
inputStream.start(); inputStream.start();
errorStream.start(); errorStream.start();
/*int retval =*/ process.waitFor(); /*int retval =*/ process.waitFor();
} catch(IOException iox) { } catch(final IOException iox) {
consoleError = true; consoleError = true;
log.logWarning("logpoint 0 " + iox.getMessage()); log.logWarning("logpoint 0 " + iox.getMessage());
List<String> list = new ArrayList<String>();
list.add(iox.getMessage()); list.add(iox.getMessage());
return list; return list;
} catch(InterruptedException ix) { } catch(final InterruptedException ix) {
consoleError = true; consoleError = true;
log.logWarning("logpoint 1 " + ix.getMessage()); log.logWarning("logpoint 1 " + ix.getMessage());
List<String> list = new ArrayList<String>();
list.add(ix.getMessage()); list.add(ix.getMessage());
return list; return list;
} }
list = inputStream.getOutput(); List<String> list = inputStream.getOutput();
if (list.isEmpty()) { if (list.isEmpty()) {
consoleError = true; consoleError = true;
log.logWarning("logpoint 2 "); log.logWarning("logpoint 2 ");
@ -490,36 +488,53 @@ nextLine:
public class consoleInterface extends Thread public class consoleInterface extends Thread
{ {
private InputStream stream; private final InputStream stream;
private boolean getInputStream; private final List<String> output = new ArrayList<String>();
private ArrayList<String> output; private final String name;
private boolean done = false;
public consoleInterface (InputStream stream) public consoleInterface (final InputStream stream, String name)
{ {
this.stream = stream; this.stream = stream;
output = new ArrayList<String>(); this.name = name;
} }
public void run() { public void run() {
try { try {
InputStreamReader input = new InputStreamReader(stream); final InputStreamReader input = new InputStreamReader(stream);
BufferedReader buffer = new BufferedReader(input); final BufferedReader buffer = new BufferedReader(input);
String line = null; String line = null;
int tries = 1000; int tries = 0;
while (tries-- > 0) { while (tries < 1000) {
tries++;
try {
Thread.sleep(1);
} catch (InterruptedException e) {
// just stop sleeping
}
if (buffer.ready()) if (buffer.ready())
break; break;
} }
log.logInfo("logpoint 3 " + tries + " tries"); log.logInfo("logpoint 3 "+ name +" needed " + tries + " tries");
while((line = buffer.readLine()) != null) { while((line = buffer.readLine()) != null) {
output.add(line); output.add(line);
} }
} catch(IOException ix) { log.logWarning("logpoint 4 " + ix.getMessage());} log.logInfo("logpoint 4 output done of '"+ name +"'");
done = true;
} catch(final IOException ix) { log.logWarning("logpoint 5 " + ix.getMessage());}
} }
public ArrayList<String> getOutput(){ public List<String> getOutput(){
log.logInfo("logpoint 6 getOutput() of '"+ name +"' requested");
while(!isDone()) {
// wait
}
return output; return output;
} }
private boolean isDone() {
return done;
}
} }
} }

37
source/de/anomic/yacy/resourceObserver.java
Unescape View File

@ -40,7 +40,6 @@
package de.anomic.yacy; package de.anomic.yacy;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator;
import java.util.Map; import java.util.Map;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
@ -59,21 +58,21 @@ public final class resourceObserver {
// The memory usage should be checked on every run // The memory usage should be checked on every run
private final int CHECK_MEMORY_USAGE_FREQ = 1; private final int CHECK_MEMORY_USAGE_FREQ = 1;
private serverLog log = new serverLog("RESOURCE OBSERVER"); private final serverLog log = new serverLog("RESOURCE OBSERVER");
private diskUsage du; private final diskUsage du;
private plasmaSwitchboard sb; private final plasmaSwitchboard sb;
private int checkDiskUsageCount; private int checkDiskUsageCount;
private int checkMemoryUsageCount; private int checkMemoryUsageCount;
private boolean disksOK; private boolean disksOK;
private boolean memoryOK; private boolean memoryOK;
public resourceObserver(plasmaSwitchboard sb) { public resourceObserver(final plasmaSwitchboard sb) {
this.sb = sb; this.sb = sb;
this.log.logInfo("initializing the resource observer"); this.log.logInfo("initializing the resource observer");
du = new diskUsage(sb); du = new diskUsage(sb);
if (!du.getUsable ()) if (!du.isUsable ())
this.log.logWarning("Disk usage returned: " + du.getErrorMessage()); this.log.logWarning("Disk usage returned: " + du.getErrorMessage());
checkDiskUsageCount = 0; checkDiskUsageCount = 0;
@ -82,7 +81,7 @@ public final class resourceObserver {
memoryOK = true; memoryOK = true;
} }
public boolean resourceObserverJob() { public void resourceObserverJob() {
checkDiskUsageCount++; checkDiskUsageCount++;
checkMemoryUsageCount++; checkMemoryUsageCount++;
boolean tmpDisksOK = true; boolean tmpDisksOK = true;
@ -109,12 +108,11 @@ public final class resourceObserver {
} }
} }
else { else {
if (du.getUsable ()) if (du.isUsable ())
this.log.logInfo("run completed; everything in order"); this.log.logInfo("run completed; everything in order");
else else
this.log.logInfo("The observer is out of order"); this.log.logInfo("The observer is out of order");
} }
return true;
} }
public boolean getDisksOK () { public boolean getDisksOK () {
@ -125,25 +123,28 @@ public final class resourceObserver {
return memoryOK; return memoryOK;
} }
/**
* @return amount of space that should be kept free
*/
public long getMinFreeDiskSpace () { public long getMinFreeDiskSpace () {
return MIN_FREE_DISK_SPACE; return MIN_FREE_DISK_SPACE;
} }
/**
* @return enough disk space availabe?
*/
private boolean checkDisks() { private boolean checkDisks() {
boolean below = false; boolean below = false;
if (!du.getUsable ()) if (!du.isUsable ())
return true; return true;
HashMap<String, long[]> usage = du.getDiskUsage(); final HashMap<String, long[]> usage = du.getDiskUsage();
Iterator<Map.Entry<String, long[]>> iter = usage.entrySet().iterator(); long[] val;
Map.Entry<String, long[]> entry; for (Map.Entry<String, long[]> entry: usage.entrySet()) {
while (iter.hasNext()) { val = entry.getValue();
entry = iter.next ();
String key = entry.getKey();
long[] val = entry.getValue();
if (val[1] < MIN_FREE_DISK_SPACE) { if (val[1] < MIN_FREE_DISK_SPACE) {
this.log.logWarning("Volume " + key + ": free space is too low"); this.log.logWarning("Volume " + entry.getKey() + ": free space is too low");
below = true; below = true;
} }
} }

Write Preview
Loading…
Cancel
Save