boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

modified the diskUsage class regarding the following two aspects:

Browse Source 
1. The usage and dependency of the plasmaSwitchboad was used many times in the past but this was
a bad mistake. The classes should be independent from the switchboard to support a better abstraction. Therefore the object was removed. The parameters from the switchboard are computed outside and then handed over.
2. the class is considered as a tightly connected to hardware resources. Classes which handle data that cannot be replicated because it would need to replicate hadware should not support dynamic object allocation, but should be coded as collection of private static methods. Therefore all class objects had been transformed into static private objects.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4961 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent da917cf4b1
commit 69aac0d74c
5 changed files with 280 additions and 213 deletions
Show Stats Download Patch File Download Diff File

8
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -876,9 +876,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
this.robotstxtConfig = httpdRobotsTxtConfig.init(this); this.robotstxtConfig = httpdRobotsTxtConfig.init(this);
// setting timestamp of last proxy access // setting timestamp of last proxy access
this.proxyLastAccess = System.currentTimeMillis() - 60000; this.proxyLastAccess = System.currentTimeMillis() - 10000;
this.localSearchLastAccess = System.currentTimeMillis() - 60000; this.localSearchLastAccess = System.currentTimeMillis() - 10000;
this.remoteSearchLastAccess = System.currentTimeMillis() - 60000; this.remoteSearchLastAccess = System.currentTimeMillis() - 10000;
this.webStructure = new plasmaWebStructure(log, rankingPath, "LOCAL/010_cr/", getConfig("CRDist0Path", plasmaRankingDistribution.CR_OWN), new File(plasmaPath, "webStructure.map")); this.webStructure = new plasmaWebStructure(log, rankingPath, "LOCAL/010_cr/", getConfig("CRDist0Path", plasmaRankingDistribution.CR_OWN), new File(plasmaPath, "webStructure.map"));
// configuring list path // configuring list path
@ -1115,6 +1115,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// initializing the resourceObserver // initializing the resourceObserver
this.observer = new resourceObserver(this); this.observer = new resourceObserver(this);
// run the oberver here a first time
this.observer.resourceObserverJob();
// initializing the stackCrawlThread // initializing the stackCrawlThread
this.crawlStacker = new CrawlStacker(this, this.plasmaPath, (int) getConfigLong("tableTypeForPreNURL", 0), (((int) getConfigLong("tableTypeForPreNURL", 0) == 0) && (getConfigLong(CRAWLSTACK_BUSYSLEEP, 0) <= 100))); this.crawlStacker = new CrawlStacker(this, this.plasmaPath, (int) getConfigLong("tableTypeForPreNURL", 0), (((int) getConfigLong("tableTypeForPreNURL", 0) == 0) && (getConfigLong(CRAWLSTACK_BUSYSLEEP, 0) <= 100)));

123
source/de/anomic/tools/consoleInterface.java
Unescape View File

@ -0,0 +1,123 @@
//consoleInterface.java
//-----------------------
//part of YaCy
//(C) by Detlef Reichl; detlef!reichl()gmx!org
//Pforzheim, Germany, 2008
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
//the Free Software Foundation; either version 2 of the License, or
//(at your option) any later version.
//
//This program is distributed in the hope that it will be useful,
//but WITHOUT ANY WARRANTY; without even the implied warranty of
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//GNU General Public License for more details.
//
//You should have received a copy of the GNU General Public License
//along with this program; if not, write to the Free Software
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
//Using this software in any meaning (reading, learning, copying, compiling,
//running) means that you agree that the Author(s) is (are) not responsible
//for cost, loss of data or any harm that may be caused directly or indirectly
//by usage of this softare or this documentation. The usage of this software
//is on your own risk. The installation and usage (starting/running) of this
//software may allow other people or application to access your computer and
//any attached devices and is highly dependent on the configuration of the
//software which must be done by the user of the software; the author(s) is
//(are) also not responsible for proper configuration and usage of the
//software, even if provoked by documentation provided together with
//the software.
//
//Any changes to this file according to the GPL as documented in the file
//gpl.txt aside this file in the shipment you received can be done to the
//lines that follows this copyright notice here, but changes must not be
//done inside the copyright notive above. A re-distribution must contain
//the intact and unchanged copyright notice.
//Contributions and changes to the program code must be marked as such.
package de.anomic.tools;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Semaphore;
import de.anomic.server.logging.serverLog;
public class consoleInterface extends Thread
{
private final InputStream stream;
private final List<String> output = new ArrayList<String>();
private final Semaphore dataIsRead = new Semaphore(1);
/**
* FIXME just for debugging
*/
private final String name;
private serverLog log;
public consoleInterface (final InputStream stream, String name, serverLog log)
{
this.log = log;
this.stream = stream;
this.name = name;
// block reading {@see getOutput()}
try {
dataIsRead.acquire();
} catch (InterruptedException e) {
// this should never happen because this is a constructor
e.printStackTrace();
}
}
public void run() {
// a second run adds data! a output.clear() maybe needed
try {
final InputStreamReader input = new InputStreamReader(stream);
final BufferedReader buffer = new BufferedReader(input);
String line = null;
int tries = 0;
while (tries < 1000) {
tries++;
try {
// may block!
Thread.sleep(1);
} catch (InterruptedException e) {
// just stop sleeping
}
if (buffer.ready())
break;
}
log.logInfo("logpoint 3 "+ name +" needed " + tries + " tries");
while((line = buffer.readLine()) != null) {
output.add(line);
}
dataIsRead.release();
} catch(final IOException ix) { log.logWarning("logpoint 6 " + ix.getMessage());}
}
/**
* waits until the stream is read and returns all data
*
* @return lines of text in stream
*/
public List<String> getOutput(){
// wait that data is ready
try {
log.logInfo("logpoint 4 waiting for data of '"+ name +"'");
final long start = System.currentTimeMillis();
dataIsRead.acquire();
log.logInfo("logpoint 5 data ready for '"+ name +"' after "+ (System.currentTimeMillis() - start) +" ms");
} catch (InterruptedException e) {
// after interrupt just return what is available (maybe nothing)
}
// is just for checking availability, so release it immediatly
dataIsRead.release();
return output;
}
}

313
source/de/anomic/tools/diskUsage.java
Unescape View File

@ -46,22 +46,17 @@
package de.anomic.tools; package de.anomic.tools;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.concurrent.Semaphore;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
public class diskUsage { public class diskUsage {
serverLog log = new serverLog("DISK USAGE");
private static final HashMap<String, long[]> diskUsages = new HashMap<String, long[]>(); private static serverLog log = new serverLog("DISK USAGE");
private static final List<String> allVolumes = new ArrayList<String>(); private static final List<String> allVolumes = new ArrayList<String>();
private static final List<String> allMountPoints = new ArrayList<String>(); private static final List<String> allMountPoints = new ArrayList<String>();
@ -70,44 +65,45 @@ public class diskUsage {
private static final List<String> yacyUsedVolumes = new ArrayList<String>(); private static final List<String> yacyUsedVolumes = new ArrayList<String>();
private static final List<String> yacyUsedMountPoints = new ArrayList<String>(); private static final List<String> yacyUsedMountPoints = new ArrayList<String>();
private static plasmaSwitchboard sb; private static int usedOS = -1;
private static int usedOS; private static boolean usable = false;
private static boolean usable; private static String windowsCommand = null;
private static String windowsCommand; private static String errorMessage = "";
private static String errorMessage; private static boolean consoleError = false;
private static boolean consoleError;
// Unix-like // Unix-like
private final int AIX = 0; // IBM private static final int AIX = 0; // IBM
private final int BS2000 = 1; // Fujitsu Siemens (oficial BS2000/OSD) private static final int BS2000 = 1; // Fujitsu Siemens (oficial BS2000/OSD)
private final int BSD = 2; // all kind of BSD //private static final int BSD = 2; // all kind of BSD
private final int HAIKU = 3; // like BeOS; does not have a JRE til now, but they are working on it private static final int HAIKU = 3; // like BeOS; does not have a JRE til now, but they are working on it
private final int HP_UX = 4; // Hewlett-Packard //private static final int HP_UX = 4; // Hewlett-Packard
private final int TRU64 = 5; // Hewlett-Packard private static final int TRU64 = 5; // Hewlett-Packard
private final int IRIX = 6; // sgi //private static final int IRIX = 6; // sgi
private final int LINUX = 7; // all kind of linux //private static final int LINUX = 7; // all kind of linux
private final int MAC_OS_X = 8; // Apple //private static final int MAC_OS_X = 8; // Apple
private final int MINIX = 9; // don't know if there even is a JRE for minix... private static final int MINIX = 9; // don't know if there even is a JRE for minix...
private final int SOLARIS = 10; // SUN //private static final int SOLARIS = 10; // SUN
private final int SUNOS = 11; // The latest SunOS version is from 1990 but the Solaris java refferer remains SunOS //private static final int SUNOS = 11; // The latest SunOS version is from 1990 but the Solaris java refferer remains SunOS
private final int UNICOS = 12; // cray private static final int UNICOS = 12; // cray
private final int UNIX_END = UNICOS; private static final int UNIX_END = UNICOS;
// Windows dos based // Windows dos based
private final int WINDOWS_95 = 13; //private static final int WINDOWS_95 = 13;
private final int WINDOWS_98 = 14; //private static final int WINDOWS_98 = 14;
private final int WINDOWS_ME = 15; //private static final int WINDOWS_ME = 15;
// Windows WinNT based // Windows WinNT based
private final int WINDOWS_NT = 16; //private static final int WINDOWS_NT = 16;
private final int WINDOWS_2000 = 17; //private static final int WINDOWS_2000 = 17;
private final int WINDOWS_XP = 18; //private static final int WINDOWS_XP = 18;
private final int WINDOWS_SERVER = 19; //private static final int WINDOWS_SERVER = 19;
private final int WINDOWS_VISTA = 20; //private static final int WINDOWS_VISTA = 20;
String[] OSname = {"aix", "bs2000", "bsd", "haiku", "hp-ux", "tru64", "irix", "linux", "mac os x", "minix", // don't change order of names!
private static final String[] OSname = {
"aix", "bs2000", "bsd", "haiku", "hp-ux", "tru64", "irix", "linux", "mac os x", "minix",
"solaris", "sunos", "unicos", "solaris", "sunos", "unicos",
"windows 95", "windows 98", "windows me", "windows 95", "windows 98", "windows me",
"windows nt", "windows 2000", "windows xp", "windows server", "windows vista"}; "windows nt", "windows 2000", "windows xp", "windows server", "windows vista"};
@ -116,22 +112,22 @@ public class diskUsage {
// public API // // public API //
////////////////// //////////////////
public diskUsage (final plasmaSwitchboard sb) { public static void init(ArrayList<String> pathsToCheck) {
errorMessage = null; errorMessage = null;
diskUsage.sb = sb; if (usedOS >= 0) return; // prevent double initialization
usedOS = getOS(); usedOS = getOS();
if (usedOS == -1) { if (usedOS == -1) {
usable = false; usable = false;
} else { } else {
usable = true; usable = true;
// some kind of *nix
if (usedOS <= UNIX_END) { if (usedOS <= UNIX_END) {
dfUnix (true); // some kind of *nix
dfUnixGetVolumes();
for (int i = 0; i < allMountPoints.size(); i++) for (int i = 0; i < allMountPoints.size(); i++)
usedVolumes.add(false); usedVolumes.add(false);
checkVolumesInUseUnix ("DATA"); checkVolumesInUseUnix ("DATA");
checkMapedSubDirs (); checkMappedSubDirs(pathsToCheck);
for (int i = 0; i < allVolumes.size(); i++){ for (int i = 0; i < allVolumes.size(); i++){
if (usedVolumes.get(i) == true) { if (usedVolumes.get(i) == true) {
@ -139,51 +135,93 @@ public class diskUsage {
yacyUsedMountPoints.add(allMountPoints.get (i)); yacyUsedMountPoints.add(allMountPoints.get (i));
} }
} }
// all Windows version
} else { } else {
checkWindowsCommandVersion(); // all Windows versions
initWindowsCommandVersion();
checkStartVolume(); checkStartVolume();
checkMapedSubDirs (); checkMappedSubDirs(pathsToCheck);
} }
if (yacyUsedVolumes.size() < 1) if (yacyUsedVolumes.size() < 1)
usable = false; usable = false;
} }
} }
public HashMap<String, long[]> getDiskUsage () { public static HashMap<String, long[]> getDiskUsage () {
if (!usable) if (!usable)
return null; return null;
if (usedOS <= UNIX_END) if (usedOS <= UNIX_END)
dfUnix(false); return dfUnix();
else else
dfWindows (); return dfWindows();
return diskUsages;
} }
public boolean isUsable () { public static boolean isUsable () {
return usable; return usable;
} }
public String getErrorMessage () { public static String getErrorMessage () {
return errorMessage; return errorMessage;
} }
public int getNumUsedVolumes () { public static int getNumUsedVolumes () {
return yacyUsedVolumes.size(); return yacyUsedVolumes.size();
} }
//////////// ////////////
// Unix // // Unix //
//////////// ////////////
private void dfUnix(boolean getVolumesOnly) { private static HashMap<String, long[]> dfUnix() {
if (!getVolumesOnly) HashMap<String, long[]> diskUsages = new HashMap<String, long[]>();
diskUsages.clear (); final List<String> lines = dfUnixExec();
if (consoleError) {
errorMessage = "df:";
for (final String line: lines){
errorMessage += "\n" + line;
}
usable = false;
return diskUsages;
}
nextLine: for (final String line: lines){
if (line.charAt(0) != '/') continue;
final String[] tokens = line.split(" +", 6);
if (tokens.length < 6) continue;
for (int i = 0; i < yacyUsedVolumes.size(); i++){
if (yacyUsedVolumes.get(i).equals(tokens[0])) {
final long[] vals = new long[2];
try { vals[0] = new Long(tokens[1]); } catch (final NumberFormatException e) { continue nextLine; }
try { vals[1] = new Long(tokens[3]); } catch (final NumberFormatException e) { continue nextLine; }
vals[0] *= 1024;
vals[1] *= 1024;
diskUsages.put(yacyUsedMountPoints.get(i), vals);
}
}
}
return diskUsages;
}
private static void dfUnixGetVolumes() {
final List<String> lines = dfUnixExec();
nextLine: for (final String line: lines){
if (line.charAt(0) != '/') continue;
final String[] tokens = line.split(" +", 6);
if (tokens.length < 6) continue;
for (int i = 0; i < allMountPoints.size(); i++) {
if (tokens[5].trim().compareTo(allMountPoints.get(i)) > 0) {
allMountPoints.add(i, tokens[5].trim());
allVolumes.add(i, tokens[0]);
continue nextLine;
}
}
allMountPoints.add(allMountPoints.size(), tokens[5]);
allVolumes.add(allVolumes.size(), tokens[0]);
}
}
private static List<String> dfUnixExec() {
// -k set blocksize to 1024 // -k set blocksize to 1024
// confirmed with tests: // confirmed with tests:
@ -219,44 +257,13 @@ public class diskUsage {
errorMessage += "\n" + line; errorMessage += "\n" + line;
} }
usable = false; usable = false;
return; lines.clear();
} }
for (final String line: lines){ return lines;
if (line.charAt(0) != '/')
continue;
final String[] tokens = line.split(" +", 6);
if (tokens.length < 6)
continue;
nextLine:
if (getVolumesOnly) {
for (int i = 0; i < allMountPoints.size(); i++) {
if (tokens[5].trim().compareTo(allMountPoints.get(i)) > 0) {
allMountPoints.add(i, tokens[5].trim());
allVolumes.add(i, tokens[0]);
break nextLine;
}
}
allMountPoints.add(allMountPoints.size(), tokens[5]);
allVolumes.add(allVolumes.size(), tokens[0]);
} else {
for (int i = 0; i < yacyUsedVolumes.size(); i++){
if (yacyUsedVolumes.get(i).equals(tokens[0])) {
final long[] vals = new long[2];
try { vals[0] = new Long(tokens[1]); } catch (final NumberFormatException e) { break nextLine; }
try { vals[1] = new Long(tokens[3]); } catch (final NumberFormatException e) { break nextLine; }
vals[0] *= 1024;
vals[1] *= 1024;
diskUsages.put (yacyUsedMountPoints.get(i), vals);
} }
}
}
}
}
private void checkVolumesInUseUnix (final String path) { private static void checkVolumesInUseUnix (final String path) {
final File file = new File(path); final File file = new File(path);
final File[] fileList = file.listFiles(); final File[] fileList = file.listFiles();
String base; String base;
@ -292,13 +299,11 @@ nextLine:
} }
} }
/////////////// ///////////////
// Windows // // Windows //
/////////////// ///////////////
private void checkWindowsCommandVersion () { private static void initWindowsCommandVersion () {
windowsCommand = null; windowsCommand = null;
final String os = System.getProperty("os.name").toLowerCase(); final String os = System.getProperty("os.name").toLowerCase();
final String[] oses = {"windows 95", "windows 98", "windows me"}; final String[] oses = {"windows 95", "windows 98", "windows me"};
@ -313,7 +318,7 @@ nextLine:
windowsCommand = "cmd.exe"; windowsCommand = "cmd.exe";
} }
private void checkStartVolume() { private static void checkStartVolume() {
final File file = new File("DATA"); final File file = new File("DATA");
String path = null; String path = null;
@ -327,7 +332,8 @@ nextLine:
yacyUsedVolumes.add(path.substring(0, 1)); yacyUsedVolumes.add(path.substring(0, 1));
} }
public void dfWindows () { public static HashMap<String, long[]> dfWindows() {
HashMap<String, long[]> diskUsages = new HashMap<String, long[]>();
for (int i = 0; i < yacyUsedVolumes.size(); i++){ for (int i = 0; i < yacyUsedVolumes.size(); i++){
final List<String> processArgs = new ArrayList<String>(); final List<String> processArgs = new ArrayList<String>();
processArgs.add(windowsCommand); processArgs.add(windowsCommand);
@ -342,7 +348,7 @@ nextLine:
errorMessage += "\n" + line; errorMessage += "\n" + line;
} }
usable = false; usable = false;
return; return diskUsages;
} }
String line = ""; String line = "";
@ -354,7 +360,7 @@ nextLine:
if (line.length() == 0) { if (line.length() == 0) {
errorMessage = "unable to get free size of volume " + yacyUsedVolumes.get(i); errorMessage = "unable to get free size of volume " + yacyUsedVolumes.get(i);
usable = false; usable = false;
return; return diskUsages;
} }
String[] tokens = line.trim().split(" ++"); String[] tokens = line.trim().split(" ++");
@ -363,14 +369,14 @@ nextLine:
try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (final NumberFormatException e) {continue;} try { vals[1] = new Long(tokens[2].replaceAll("[.,]", "")); } catch (final NumberFormatException e) {continue;}
diskUsages.put (yacyUsedVolumes.get(i), vals); diskUsages.put (yacyUsedVolumes.get(i), vals);
} }
return diskUsages;
} }
///////////// /////////////
// common // // common //
///////////// /////////////
private int getOS () { private static int getOS () {
final String os = System.getProperty("os.name").toLowerCase(); final String os = System.getProperty("os.name").toLowerCase();
for (int i = 0; i < OSname.length; i++) for (int i = 0; i < OSname.length; i++)
{ {
@ -381,32 +387,16 @@ nextLine:
return -1; return -1;
} }
private void checkMapedSubDirs () { private static void checkMappedSubDirs (ArrayList<String> pathsToCheck) {
// FIXME whats about the secondary path??? for (final String path : pathsToCheck) {
// = (getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, "");
final String[] pathes = {plasmaSwitchboard.HTDOCS_PATH,
plasmaSwitchboard.INDEX_PRIMARY_PATH,
plasmaSwitchboard.LISTS_PATH,
plasmaSwitchboard.PLASMA_PATH,
plasmaSwitchboard.RANKING_PATH,
plasmaSwitchboard.WORK_PATH};
String path;
for (final String element : pathes) {
path = null;
try {
path = sb.getConfigPath(element, "").getCanonicalPath().toString();
} catch (final IOException e) { continue; }
if (path.length() > 0) {
if (usedOS <= UNIX_END) if (usedOS <= UNIX_END)
checkPathUsageUnix (path); checkPathUsageUnix (path);
else else
checkPathUsageWindows (path); checkPathUsageWindows (path);
} }
} }
}
private void checkPathUsageUnix (final String path) { private static void checkPathUsageUnix (final String path) {
for (int i = 0; i < allMountPoints.size(); i++){ for (int i = 0; i < allMountPoints.size(); i++){
if (path.startsWith (allMountPoints.get(i))) { if (path.startsWith (allMountPoints.get(i))) {
usedVolumes.set(i, true); usedVolumes.set(i, true);
@ -415,9 +405,9 @@ nextLine:
} }
} }
private void checkPathUsageWindows (final String path) { private static void checkPathUsageWindows (final String path) {
int index = -1; int index = -1;
String sub = path.substring(0, 1); String sub = path.substring(0, 1); // ?? nur ein character?
try { index = yacyUsedVolumes.indexOf(sub); } catch (IndexOutOfBoundsException e) { try { index = yacyUsedVolumes.indexOf(sub); } catch (IndexOutOfBoundsException e) {
errorMessage = "internal error while checking used windows volumes"; errorMessage = "internal error while checking used windows volumes";
usable = false; usable = false;
@ -427,7 +417,7 @@ nextLine:
yacyUsedVolumes.add(sub); yacyUsedVolumes.add(sub);
} }
private List<String> getConsoleOutput (final List<String> processArgs) { private static List<String> getConsoleOutput (final List<String> processArgs) {
final ProcessBuilder processBuilder = new ProcessBuilder(processArgs); final ProcessBuilder processBuilder = new ProcessBuilder(processArgs);
Process process = null; Process process = null;
consoleInterface inputStream = null; consoleInterface inputStream = null;
@ -437,8 +427,8 @@ nextLine:
try { try {
process = processBuilder.start(); process = processBuilder.start();
inputStream = new consoleInterface(process.getInputStream(), "input"); inputStream = new consoleInterface(process.getInputStream(), "input", log);
errorStream = new consoleInterface(process.getErrorStream(), "error"); errorStream = new consoleInterface(process.getErrorStream(), "error", log);
inputStream.start(); inputStream.start();
errorStream.start(); errorStream.start();
@ -467,74 +457,5 @@ nextLine:
return list; return list;
} }
public class consoleInterface extends Thread
{
private final InputStream stream;
private final List<String> output = new ArrayList<String>();
private final Semaphore dataIsRead = new Semaphore(1);
/**
* FIXME just for debugging
*/
private final String name;
public consoleInterface (final InputStream stream, String name)
{
this.stream = stream;
this.name = name;
// block reading {@see getOutput()}
try {
dataIsRead.acquire();
} catch (InterruptedException e) {
// this should never happen because this is a constructor
e.printStackTrace();
}
}
public void run() {
// a second run adds data! a output.clear() maybe needed
try {
final InputStreamReader input = new InputStreamReader(stream);
final BufferedReader buffer = new BufferedReader(input);
String line = null;
int tries = 0;
while (tries < 1000) {
tries++;
try {
// may block!
Thread.sleep(1);
} catch (InterruptedException e) {
// just stop sleeping
}
if (buffer.ready())
break;
}
log.logInfo("logpoint 3 "+ name +" needed " + tries + " tries");
while((line = buffer.readLine()) != null) {
output.add(line);
}
dataIsRead.release();
} catch(final IOException ix) { log.logWarning("logpoint 6 " + ix.getMessage());}
}
/**
* waits until the stream is read and returns all data
*
* @return lines of text in stream
*/
public List<String> getOutput(){
// wait that data is ready
try {
log.logInfo("logpoint 4 waiting for data of '"+ name +"'");
final long start = System.currentTimeMillis();
dataIsRead.acquire();
log.logInfo("logpoint 5 data ready for '"+ name +"' after "+ (System.currentTimeMillis() - start) +" ms");
} catch (InterruptedException e) {
// after interrupt just return what is available (maybe nothing)
}
// is just for checking availability, so release it immediatly
dataIsRead.release();
return output;
}
}
} }

42
source/de/anomic/yacy/resourceObserver.java
Unescape View File

@ -39,6 +39,8 @@
package de.anomic.yacy; package de.anomic.yacy;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -51,7 +53,7 @@ public final class resourceObserver {
// TODO make it configurable // TODO make it configurable
private final static long MIN_FREE_DISK_SPACE = 100L /* MiB */ * 1024L * 1024L; private final static long MIN_FREE_DISK_SPACE = 100L /* MiB */ * 1024L * 1024L;
// Unknown for now // Unknown for now
private final static long MIN_FREE_MEMORY = 0; //private final static long MIN_FREE_MEMORY = 0;
// We are called with the cleanup job every five minutes; // We are called with the cleanup job every five minutes;
// the disk usage should be checked with every run // the disk usage should be checked with every run
private final int CHECK_DISK_USAGE_FREQ = 1; private final int CHECK_DISK_USAGE_FREQ = 1;
@ -59,7 +61,6 @@ public final class resourceObserver {
private final int CHECK_MEMORY_USAGE_FREQ = 1; private final int CHECK_MEMORY_USAGE_FREQ = 1;
private final serverLog log = new serverLog("RESOURCE OBSERVER"); private final serverLog log = new serverLog("RESOURCE OBSERVER");
private final diskUsage du;
private final plasmaSwitchboard sb; private final plasmaSwitchboard sb;
private int checkDiskUsageCount; private int checkDiskUsageCount;
@ -67,13 +68,31 @@ public final class resourceObserver {
private boolean disksOK; private boolean disksOK;
private boolean memoryOK; private boolean memoryOK;
public resourceObserver(final plasmaSwitchboard sb) { public resourceObserver(plasmaSwitchboard sb) {
this.sb = sb; this.sb = sb;
this.log.logInfo("initializing the resource observer"); this.log.logInfo("initializing the resource observer");
du = new diskUsage(sb);
if (!du.isUsable ()) ArrayList<String> pathsToCheck = new ArrayList<String>();
this.log.logWarning("Disk usage returned: " + du.getErrorMessage()); // FIXME whats about the secondary path???
// = (getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, "");
final String[] pathes = {plasmaSwitchboard.HTDOCS_PATH,
plasmaSwitchboard.INDEX_PRIMARY_PATH,
plasmaSwitchboard.LISTS_PATH,
plasmaSwitchboard.PLASMA_PATH,
plasmaSwitchboard.RANKING_PATH,
plasmaSwitchboard.WORK_PATH};
String path;
for (final String element : pathes) {
try {
path = sb.getConfigPath(element, "").getCanonicalPath().toString();
if (path.length() > 0) pathsToCheck.add(path);
} catch (final IOException e) {}
}
diskUsage.init(pathsToCheck);
if (!diskUsage.isUsable ())
this.log.logWarning("Disk usage returned: " + diskUsage.getErrorMessage());
checkDiskUsageCount = 0; checkDiskUsageCount = 0;
checkMemoryUsageCount = 0; checkMemoryUsageCount = 0;
@ -108,7 +127,7 @@ public final class resourceObserver {
} }
} }
else { else {
if (du.isUsable ()) if (diskUsage.isUsable())
this.log.logInfo("run completed; everything in order"); this.log.logInfo("run completed; everything in order");
else else
this.log.logInfo("The observer is out of order"); this.log.logInfo("The observer is out of order");
@ -131,20 +150,21 @@ public final class resourceObserver {
} }
/** /**
* @return enough disk space availabe? * @return enough disk space available?
*/ */
private boolean checkDisks() { private boolean checkDisks() {
boolean below = false; boolean below = false;
if (!du.isUsable ()) if (!diskUsage.isUsable ())
return true; return true;
final HashMap<String, long[]> usage = du.getDiskUsage(); final HashMap<String, long[]> usage = diskUsage.getDiskUsage();
long[] val; long[] val;
for (Map.Entry<String, long[]> entry: usage.entrySet()) { for (Map.Entry<String, long[]> entry: usage.entrySet()) {
val = entry.getValue(); val = entry.getValue();
this.log.logInfo("df of Volume " + entry.getKey() + ": " + (val[1] / 1024 / 1024) + " MB");
if (val[1] < MIN_FREE_DISK_SPACE) { if (val[1] < MIN_FREE_DISK_SPACE) {
this.log.logWarning("Volume " + entry.getKey() + ": free space is too low"); this.log.logWarning("Volume " + entry.getKey() + ": free space (" + (val[1] / 1024 / 1024) + " MB) is too low (< " + (MIN_FREE_DISK_SPACE / 1024 / 1024) + " MB)");
below = true; below = true;
} }
} }

1
source/de/anomic/yacy/yacyDHTAction.java
Unescape View File

@ -373,6 +373,7 @@ public class yacyDHTAction {
if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + firstKey + ", last_hash = " + lastKey); if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + firstKey + ", last_hash = " + lastKey);
while ((e.hasNext()) && (seeds.size() < (primaryPeerCount + reservePeerCount)) && (maxloop-- > 0)) { while ((e.hasNext()) && (seeds.size() < (primaryPeerCount + reservePeerCount)) && (maxloop-- > 0)) {
seed = e.next(); seed = e.next();
if (seed == null || seed.hash == null) continue;
firstdist = yacyDHTAction.dhtDistance(seed.hash, firstKey); firstdist = yacyDHTAction.dhtDistance(seed.hash, firstKey);
lastdist = yacyDHTAction.dhtDistance(seed.hash, lastKey); lastdist = yacyDHTAction.dhtDistance(seed.hash, lastKey);
if (lastdist > maxDist) { if (lastdist > maxDist) {

Write Preview
Loading…
Cancel
Save