- refactoring of resourceObserver (moved it to crawler)

- partly redesign of diskUsage: little bit more functional behavior, less side effects, better error case handling
- the resourceObserver can now show a error message if the diskUsage is 'out of order'

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4973 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent f0012554de
commit 1400cdc91e

@ -1,43 +1,5 @@
// resourceObserver.java package de.anomic.crawler;
// -----------------------
// part of YaCy
// (C) by Detlef Reichl; detlef!reichl()gmx!org
// Pforzheim, Germany, 2008
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.yacy;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -48,7 +10,7 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.tools.diskUsage; import de.anomic.tools.diskUsage;
public final class resourceObserver { public final class ResourceObserver {
// The minimal free space on every used volume, for now set to 100 MB. // The minimal free space on every used volume, for now set to 100 MB.
// TODO make it configurable // TODO make it configurable
private final static long MIN_FREE_DISK_SPACE = 100L /* MiB */ * 1024L * 1024L; private final static long MIN_FREE_DISK_SPACE = 100L /* MiB */ * 1024L * 1024L;
@ -68,7 +30,7 @@ public final class resourceObserver {
private boolean disksOK; private boolean disksOK;
private boolean memoryOK; private boolean memoryOK;
public resourceObserver(plasmaSwitchboard sb) { public ResourceObserver(plasmaSwitchboard sb) {
this.sb = sb; this.sb = sb;
this.log.logInfo("initializing the resource observer"); this.log.logInfo("initializing the resource observer");
@ -91,7 +53,7 @@ public final class resourceObserver {
diskUsage.init(pathsToCheck); diskUsage.init(pathsToCheck);
if (!diskUsage.isUsable ()) if (!diskUsage.isUsable())
this.log.logWarning("Disk usage returned: " + diskUsage.getErrorMessage()); this.log.logWarning("Disk usage returned: " + diskUsage.getErrorMessage());
checkDiskUsageCount = 0; checkDiskUsageCount = 0;
@ -130,7 +92,7 @@ public final class resourceObserver {
if (diskUsage.isUsable()) if (diskUsage.isUsable())
this.log.logInfo("run completed; everything in order"); this.log.logInfo("run completed; everything in order");
else else
this.log.logInfo("The observer is out of order"); this.log.logInfo("The observer is out of order: " + diskUsage.getErrorMessage());
} }
} }

@ -85,6 +85,7 @@
package de.anomic.plasma; package de.anomic.plasma;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
@ -120,6 +121,7 @@ import de.anomic.crawler.ResultImages;
import de.anomic.crawler.ResultURLs; import de.anomic.crawler.ResultURLs;
import de.anomic.crawler.RobotsTxt; import de.anomic.crawler.RobotsTxt;
import de.anomic.crawler.ZURL; import de.anomic.crawler.ZURL;
import de.anomic.crawler.ResourceObserver;
import de.anomic.data.URLLicense; import de.anomic.data.URLLicense;
import de.anomic.data.blogBoard; import de.anomic.data.blogBoard;
import de.anomic.data.blogBoardComments; import de.anomic.data.blogBoardComments;
@ -163,7 +165,6 @@ import de.anomic.server.serverThread;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.tools.crypt; import de.anomic.tools.crypt;
import de.anomic.tools.nxTools; import de.anomic.tools.nxTools;
import de.anomic.yacy.resourceObserver;
import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsPool;
@ -225,7 +226,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
public plasmaParser parser; public plasmaParser parser;
public volatile long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess; public volatile long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess;
public yacyCore yc; public yacyCore yc;
public resourceObserver observer; public ResourceObserver observer;
public userDB userDB; public userDB userDB;
public bookmarksDB bookmarksDB; public bookmarksDB bookmarksDB;
public plasmaWebStructure webStructure; public plasmaWebStructure webStructure;
@ -1114,7 +1115,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
} }
// initializing the resourceObserver // initializing the resourceObserver
this.observer = new resourceObserver(this); this.observer = new ResourceObserver(this);
// run the oberver here a first time // run the oberver here a first time
this.observer.resourceObserverJob(); this.observer.resourceObserverJob();

@ -4,6 +4,8 @@
// (C) by Detlef Reichl; detlef!reichl()gmx!org // (C) by Detlef Reichl; detlef!reichl()gmx!org
// Pforzheim, Germany, 2008 // Pforzheim, Germany, 2008
// //
// [MC] made many changes to remove side-effect-based routines towards a more functional programming style
//
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by // it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or // the Free Software Foundation; either version 2 of the License, or
@ -17,25 +19,6 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
// The HashMap contains the following values: // The HashMap contains the following values:
@ -66,10 +49,8 @@ public class diskUsage {
private static final List<String> yacyUsedMountPoints = new ArrayList<String>(); private static final List<String> yacyUsedMountPoints = new ArrayList<String>();
private static int usedOS = -1; private static int usedOS = -1;
private static boolean usable = false; private static String usageError = null;
private static String windowsCommand = null; private static String windowsCommand = null;
private static String errorMessage = "";
private static boolean consoleError = false;
// Unix-like // Unix-like
@ -113,13 +94,12 @@ public class diskUsage {
////////////////// //////////////////
public static void init(ArrayList<String> pathsToCheck) { public static void init(ArrayList<String> pathsToCheck) {
errorMessage = null;
if (usedOS >= 0) return; // prevent double initialization if (usedOS >= 0) return; // prevent double initialization
usedOS = getOS(); usedOS = getOS();
if (usedOS == -1) { if (usedOS == -1) {
usable = false; return;
} else { } else {
usable = true; usageError = null;
if (usedOS <= UNIX_END) { if (usedOS <= UNIX_END) {
// some kind of *nix // some kind of *nix
@ -142,13 +122,12 @@ public class diskUsage {
checkMappedSubDirs(pathsToCheck); checkMappedSubDirs(pathsToCheck);
} }
if (yacyUsedVolumes.size() < 1) if (yacyUsedVolumes.size() < 1)
usable = false; usageError = "found no volumes";
} }
} }
public static HashMap<String, long[]> getDiskUsage () { public static HashMap<String, long[]> getDiskUsage () {
if (!usable) if (usageError != null) return null;
return null;
if (usedOS <= UNIX_END) if (usedOS <= UNIX_END)
return dfUnix(); return dfUnix();
@ -156,34 +135,26 @@ public class diskUsage {
return dfWindows(); return dfWindows();
} }
public static boolean isUsable () { public static boolean isUsable() {
return usable; return usageError == null;
}
public static String getErrorMessage () {
return errorMessage;
} }
public static int getNumUsedVolumes () { public static int getNumUsedVolumes () {
return yacyUsedVolumes.size(); return yacyUsedVolumes.size();
} }
public static String getErrorMessage() {
return usageError;
}
//////////// ////////////
// Unix // // Unix //
//////////// ////////////
private static HashMap<String, long[]> dfUnix() { private static HashMap<String, long[]> dfUnix() {
HashMap<String, long[]> diskUsages = new HashMap<String, long[]>(); HashMap<String, long[]> diskUsages = new HashMap<String, long[]>();
try {
final List<String> lines = dfUnixExec(); final List<String> lines = dfUnixExec();
if (consoleError) {
errorMessage = "df:";
for (final String line: lines){
errorMessage += "\n" + line;
}
usable = false;
return diskUsages;
}
nextLine: for (final String line: lines){ nextLine: for (final String line: lines){
if (line.charAt(0) != '/') continue; if (line.charAt(0) != '/') continue;
final String[] tokens = line.split(" +", 6); final String[] tokens = line.split(" +", 6);
@ -200,9 +171,14 @@ public class diskUsage {
} }
} }
return diskUsages; return diskUsages;
} catch (IOException e) {
usageError = "dfUnix: " + e.getMessage();
return diskUsages;
}
} }
private static void dfUnixGetVolumes() { private static void dfUnixGetVolumes() {
try {
final List<String> lines = dfUnixExec(); final List<String> lines = dfUnixExec();
nextLine: for (final String line: lines){ nextLine: for (final String line: lines){
@ -219,9 +195,12 @@ public class diskUsage {
allMountPoints.add(allMountPoints.size(), tokens[5]); allMountPoints.add(allMountPoints.size(), tokens[5]);
allVolumes.add(allVolumes.size(), tokens[0]); allVolumes.add(allVolumes.size(), tokens[0]);
} }
} catch (IOException e) {
usageError = "error during dfUnixGetVolumes: " + e.getMessage();
}
} }
private static List<String> dfUnixExec() { private static List<String> dfUnixExec() throws IOException {
// -k set blocksize to 1024 // -k set blocksize to 1024
// confirmed with tests: // confirmed with tests:
@ -251,15 +230,6 @@ public class diskUsage {
processArgs.add("-l"); processArgs.add("-l");
final List<String> lines = getConsoleOutput(processArgs); final List<String> lines = getConsoleOutput(processArgs);
if (consoleError) {
errorMessage = "df:";
for (final String line: lines){
errorMessage += "\n" + line;
}
usable = false;
lines.clear();
}
return lines; return lines;
} }
@ -274,7 +244,7 @@ public class diskUsage {
try { try {
dir = element.getCanonicalPath(); dir = element.getCanonicalPath();
} catch (final IOException e) { } catch (final IOException e) {
usable = false; usageError = "checkVolumesInUseUnix(1): " + e.getMessage();
break; break;
} }
if (dir.endsWith ("HTCACHE") if (dir.endsWith ("HTCACHE")
@ -291,7 +261,7 @@ public class diskUsage {
try { try {
base = element.getCanonicalPath(); base = element.getCanonicalPath();
} catch (final IOException e) { } catch (final IOException e) {
usable = false; usageError = "checkVolumesInUseUnix(2): " + e.getMessage();
break; break;
} }
checkPathUsageUnix (base); checkPathUsageUnix (base);
@ -323,8 +293,7 @@ public class diskUsage {
String path = null; String path = null;
try { path = file.getCanonicalPath().toString(); } catch (final IOException e) { try { path = file.getCanonicalPath().toString(); } catch (final IOException e) {
errorMessage = "Cant get DATA directory"; usageError = "Cant get DATA directory";
usable = false;
return; return;
} }
if (path.length() < 6) if (path.length() < 6)
@ -341,25 +310,16 @@ public class diskUsage {
processArgs.add("dir"); processArgs.add("dir");
processArgs.add(yacyUsedVolumes.get(i) + ":\\"); processArgs.add(yacyUsedVolumes.get(i) + ":\\");
try {
final List<String> lines = getConsoleOutput(processArgs); final List<String> lines = getConsoleOutput(processArgs);
if (consoleError) {
errorMessage = "df:";
for (final String line: lines){
errorMessage += "\n" + line;
}
usable = false;
return diskUsages;
}
String line = ""; String line = "";
for (int l = lines.size() - 1; l >= 0; l--) { for (int l = lines.size() - 1; l >= 0; l--) {
line = lines.get(l).trim(); line = lines.get(l).trim();
if (line.length() > 0) if (line.length() > 0) break;
break;
} }
if (line.length() == 0) { if (line.length() == 0) {
errorMessage = "unable to get free size of volume " + yacyUsedVolumes.get(i); usageError = "unable to get free size of volume " + yacyUsedVolumes.get(i);
usable = false;
return diskUsages; return diskUsages;
} }
@ -368,6 +328,11 @@ public class diskUsage {
vals[0] = -1; vals[0] = -1;
try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (final NumberFormatException e) {continue;} try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (final NumberFormatException e) {continue;}
diskUsages.put (yacyUsedVolumes.get(i), vals); diskUsages.put (yacyUsedVolumes.get(i), vals);
} catch (IOException e) {
usageError = "dfWindows: " + e.getMessage();
return diskUsages;
}
} }
return diskUsages; return diskUsages;
} }
@ -376,14 +341,14 @@ public class diskUsage {
// common // // common //
///////////// /////////////
private static int getOS () { private static int getOS() {
final String os = System.getProperty("os.name").toLowerCase(); final String os = System.getProperty("os.name").toLowerCase();
for (int i = 0; i < OSname.length; i++) for (int i = 0; i < OSname.length; i++)
{ {
if (os.indexOf(OSname[i]) >= 0) if (os.indexOf(OSname[i]) >= 0)
return i; return i;
} }
errorMessage = "unknown operating system (" + System.getProperty("os.name") + ")"; usageError = "unknown operating system (" + System.getProperty("os.name") + ")";
return -1; return -1;
} }
@ -405,24 +370,24 @@ public class diskUsage {
} }
} }
private static void checkPathUsageWindows (final String path) { private static void checkPathUsageWindows(final String path) {
int index = -1; int index = -1;
String sub = path.substring(0, 1); // ?? nur ein character? String sub = path.substring(0, 1); // ?? nur ein character?
try { index = yacyUsedVolumes.indexOf(sub); } catch (IndexOutOfBoundsException e) { try {
errorMessage = "internal error while checking used windows volumes"; index = yacyUsedVolumes.indexOf(sub);
usable = false; } catch (IndexOutOfBoundsException e) {
usageError = "internal error while checking used windows volumes";
return; return;
} }
if (index < 0) if (index < 0)
yacyUsedVolumes.add(sub); yacyUsedVolumes.add(sub);
} }
private static List<String> getConsoleOutput (final List<String> processArgs) { private static List<String> getConsoleOutput(final List<String> processArgs) throws IOException {
final ProcessBuilder processBuilder = new ProcessBuilder(processArgs); final ProcessBuilder processBuilder = new ProcessBuilder(processArgs);
Process process = null; Process process = null;
consoleInterface inputStream = null; consoleInterface inputStream = null;
consoleInterface errorStream = null; consoleInterface errorStream = null;
consoleError = false;
try { try {
process = processBuilder.start(); process = processBuilder.start();
@ -435,24 +400,18 @@ public class diskUsage {
/*int retval =*/ process.waitFor(); /*int retval =*/ process.waitFor();
} catch(final IOException iox) { } catch (final IOException iox) {
consoleError = true;
log.logWarning("logpoint 0 " + iox.getMessage()); log.logWarning("logpoint 0 " + iox.getMessage());
List<String> list = new ArrayList<String>(); throw new IOException(iox.getMessage());
list.add(iox.getMessage()); } catch (final InterruptedException ix) {
return list;
} catch(final InterruptedException ix) {
consoleError = true;
log.logWarning("logpoint 1 " + ix.getMessage()); log.logWarning("logpoint 1 " + ix.getMessage());
List<String> list = new ArrayList<String>(); throw new IOException(ix.getMessage());
list.add(ix.getMessage());
return list;
} }
List<String> list = inputStream.getOutput(); List<String> list = inputStream.getOutput();
if (list.isEmpty()) { if (list.isEmpty()) {
consoleError = true; String error = errorStream.getOutput().toString();
log.logWarning("logpoint 2 "); log.logWarning("logpoint 2: "+ error);
return errorStream.getOutput(); throw new IOException("empty list: " + error);
} else } else
return list; return list;
} }

Loading…
Cancel
Save