diff --git a/defaults/yacy.init b/defaults/yacy.init index 926f37292..40e56793f 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -135,10 +135,10 @@ network.unit.definition = defaults/yacy.network.freeworld.unit # This option is only valid if the network.unit.domain property is set to 'any' network.unit.domain.nocheck = false -# in addition to non-dht networks a client may have its own agent name -# this option is only used if the value is non-empty and network.unit.dht = false -# that means it is not usable in YaCy p2p-configurations, only in private portal configurations -network.unit.tenant.agent = +# A client may have its own agent name. This name can be set by the user and is set to a random value +# if not overwritten by the user. As an alternative, the name can be set with this property. +# This option is only used if the value is non-empty. +network.unit.agent = # Prefer https for in-protocol operations when available on remote peers # A distinct general setting is available to control whether https sould be used for remote search queries : remotesearch.https.preferred diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index 39b7aada7..1f368149d 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -224,7 +224,7 @@ public class ConfigNetwork_p prop.putHTML("network.unit.name", sb.getConfig(SwitchboardConstants.NETWORK_NAME, "")); prop.putHTML("network.unit.description", sb.getConfig("network.unit.description", "")); prop.putHTML("network.unit.domain", sb.getConfig(SwitchboardConstants.NETWORK_DOMAIN, "")); - prop.putHTML("network.unit.dht", sb.getConfig(SwitchboardConstants.DHT_ENABLED, "")); + prop.putHTML("network.unit.dht", sb.getConfig(SwitchboardConstants.NETWORK_UNIT_DHT, "")); networkBootstrapLocations.remove(sb.getConfig("network.unit.definition", "")); int c = 0; for ( final String s : networkBootstrapLocations ) { diff --git a/source/net/yacy/http/servlets/YaCyDefaultServlet.java b/source/net/yacy/http/servlets/YaCyDefaultServlet.java index 92746e05d..0c77d1094 100644 --- a/source/net/yacy/http/servlets/YaCyDefaultServlet.java +++ b/source/net/yacy/http/servlets/YaCyDefaultServlet.java @@ -1039,7 +1039,7 @@ public class YaCyDefaultServlet extends HttpServlet { templatePatterns.put("simpleheadernavbar", sb.getConfig("decoration.simpleheadernavbar", "navbar-default")); // add navigation keys to enable or disable menu items - templatePatterns.put("navigation-p2p", sb.getConfigBool(SwitchboardConstants.DHT_ENABLED, true) || !sb.isRobinsonMode() ? 1 : 0); + templatePatterns.put("navigation-p2p", sb.getConfigBool(SwitchboardConstants.NETWORK_UNIT_DHT, true) || !sb.isRobinsonMode() ? 1 : 0); templatePatterns.put("navigation-p2p_authorized", authorized ? 1 : 0); String submitted = sb.getConfig("server.servlets.submitted", ""); boolean crawler_enabled = true; /* diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 4f61ce188..4038ebb63 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -633,6 +633,8 @@ public final class Switchboard extends serverSwitch { partitionExponent, false, this.exceed134217727); + String agent = getConfig(SwitchboardConstants.NETWORK_UNIT_AGENT, ""); + if (!agent.isEmpty()) this.peers.setMyName(agent); // this can thus be set using the environment variable yacy.network.unit.agent this.localpeers = Collections.newSetFromMap(new ConcurrentHashMap<>()); new OneTimeBusyThread("Switchboard.scanForOtherYaCyInIntranet") { @Override @@ -3473,7 +3475,7 @@ public final class Switchboard extends serverSwitch { condenser, searchEvent, sourceName, - getConfigBool(SwitchboardConstants.DHT_ENABLED, false), + getConfigBool(SwitchboardConstants.NETWORK_UNIT_DHT, false), this.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null, this.getConfig("crawler.http.acceptLanguage", null)); final RSSFeed feed = @@ -4142,7 +4144,7 @@ public final class Switchboard extends serverSwitch { if ( this.peers.noDHTActivity() ) { return "no DHT distribution: network too small"; } - if ( !getConfigBool(SwitchboardConstants.DHT_ENABLED, true) ) { + if ( !getConfigBool(SwitchboardConstants.NETWORK_UNIT_DHT, true) ) { return "no DHT distribution: disabled by network.unit.dht"; } if ( getConfig(SwitchboardConstants.INDEX_DIST_ALLOW, "false").equalsIgnoreCase("false") ) { diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 4f9f9daa0..421b9e6ac 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -267,9 +267,9 @@ public final class SwitchboardConstants { /** Default value for the global HTTP Referrer policy delivered by meta tag */ public static final String REFERRER_META_POLICY_DEFAULT = "origin-when-cross-origin"; - - public static final String DHT_ENABLED = "network.unit.dht"; + public static final String NETWORK_UNIT_DHT = "network.unit.dht"; + public static final String NETWORK_UNIT_AGENT = "network.unit.agent"; public static final String REMOTESEARCH_MAXCOUNT_DEFAULT = "network.unit.remotesearch.maxcount"; public static final String REMOTESEARCH_MAXTIME_DEFAULT = "network.unit.remotesearch.maxtime"; public static final String REMOTESEARCH_MAXCOUNT_USER = "remotesearch.maxcount"; diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index 06b5cfbf3..a41e07573 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -58,249 +58,254 @@ import net.yacy.peers.Seed; import net.yacy.search.SwitchboardConstants; public class serverSwitch { - - /** Key of system property defining locally open http port */ - public static final String LOCAL_PORT_SYSTEM_PROPERTY = "net.yacy.server.localPort"; - - // configuration management - private final File configFile; - private final String configComment; - public final File dataPath; - public final File appPath; - protected boolean firstInit; - public ConcurrentLog log; - protected int serverJobs; - private ConcurrentMap configProps; - private final ConcurrentMap configRemoved; - private final NavigableMap workerThreads; - private YaCyHttpServer httpserver; // implemented HttpServer - private ConcurrentMap upnpPortMap = new ConcurrentHashMap<>(); - private boolean isConnectedViaUpnp; - - public serverSwitch(final File dataPath, final File appPath, - final String initPath, final String configPath) { - // we initialize the switchboard with a property file, - // but maintain these properties then later in a new 'config' file - // to reset all changed configs, the config file must - // be deleted, but not the init file - // the only attribute that will always be read from the init is the - // file name of the config file - this.dataPath = dataPath; - this.appPath = appPath; - this.configComment = "This is an automatically generated file, updated by serverAbstractSwitch and initialized by " - + initPath; - final File initFile = new File(appPath, initPath); - this.configFile = new File(dataPath, configPath); // propertiesFile(config); - this.firstInit = !this.configFile.exists(); // this is true if the - // application was started - // for the first time - new File(this.configFile.getParent()).mkdir(); - - // predefine init's - final ConcurrentMap initProps; - if (initFile.exists()) { - initProps = FileUtils.loadMap(initFile); - } else { - initProps = new ConcurrentHashMap(); - } - - // load config's from last save - if (this.configFile.exists()) { - this.configProps = FileUtils.loadMap(this.configFile); - } else { - this.configProps = new ConcurrentHashMap(); - } - - // remove all values from config that do not appear in init - this.configRemoved = new ConcurrentHashMap(); - Iterator i = this.configProps.keySet().iterator(); - String key; - while (i.hasNext()) { - key = i.next(); - if (!(initProps.containsKey(key))) { - this.configRemoved.put(key, this.configProps.get(key)); - i.remove(); - } - } - - // merge new props from init to config - // this is necessary for migration, when new properties are attached - initProps.putAll(this.configProps); - this.configProps = initProps; - - // save result; this may initially create a config file after - // initialization - saveConfig(); - - // init thread control - this.workerThreads = new TreeMap(); - - // init busy state control - // this.serverJobs = 0; - - // init server tracking - serverAccessTracker.init( - getConfigLong("server.maxTrackingTime", 60 * 60 * 1000), - (int) getConfigLong("server.maxTrackingCount", 1000), - (int) getConfigLong("server.maxTrackingHostCount", 100)); - } - - /** - * get my public IP, either set statically or figure out dynamic This method - * is deprecated because there may be more than one public IPs of this peer, - * i.e. one IPv4 and one IPv6. Please use myPublicIPs() instead - * - * @return the public IP of this peer, if known - */ - public String myPublicIP() { - // if a static IP was configured, we have to return it here ... - final String staticIP = getConfig(SwitchboardConstants.SERVER_STATICIP, ""); - if (staticIP.length() > 0) - return staticIP; - - // otherwise we return the real IP address of this host - final InetAddress pLIP = Domains.myPublicLocalIP(); - if (pLIP != null) - return pLIP.getHostAddress(); - return null; - } - - /** - * Get all my public IPs. If there was a static IP assignment, only one, - * that IP is returned. - * - * @return a set of IPs which are supposed to be my own public IPs - */ - public Set myPublicIPs() { - // if a static IP was configured, we have to return it here ... - final String staticIP = getConfig(SwitchboardConstants.SERVER_STATICIP, ""); - if (staticIP.length() > 0) { - HashSet h = new HashSet<>(); - h.add(staticIP); - return h; - } - - Set h = new LinkedHashSet<>(); - for (InetAddress i : Domains.myPublicIPv6()) { - String s = i.getHostAddress(); - if (Seed.isProperIP(s)) - h.add(Domains.chopZoneID(s)); - } - for (InetAddress i : Domains.myPublicIPv4()) { - String s = i.getHostAddress(); - if (Seed.isProperIP(s)) - h.add(Domains.chopZoneID(s)); - } - return h; - } - - /** - * Gets public port. May differ from local port due to NATting. This method - * will eventually removed once nobody used IPv4 anymore, but until then we - * have to live with it. - * - * @param key - * original key from config (for example "port" or "port.ssl") - * @param dflt - * default value which will be used if no value is found - * @return the public port of this system on its IPv4 address - * - * @see #getLocalPort(String, int) - */ - public int getPublicPort(final String key, final int dflt) { - - if (isConnectedViaUpnp && upnpPortMap.containsKey(key)) { - return upnpPortMap.get(key).intValue(); - } - - // TODO: add way of setting and retrieving port for manual NAT - - return getConfigInt(key, dflt); - } - - /** - * @return local http server port or null if system property is not defined - */ - public Integer getLocalPortSystemProperty() { - String systemDefinedPort = System.getProperty(LOCAL_PORT_SYSTEM_PROPERTY); - Integer localPort = null; - if(systemDefinedPort != null) { - try { - localPort = Integer.parseInt(systemDefinedPort); - } catch(NumberFormatException e) { - log.warn("System property " + LOCAL_PORT_SYSTEM_PROPERTY + " is not valid : it should be a integer."); - } - } - return localPort; - } - - /** - * Wrapper for {@link #getConfigInt(String, int)} to have a more consistent - * API. - * - * Default value 8090 will be used if no value is found in system properties and in configuration. + + /** Key of system property defining locally open http port */ + public static final String LOCAL_PORT_SYSTEM_PROPERTY = "net.yacy.server.localPort"; + + // configuration management + private final File configFile; + private final String configComment; + public final File dataPath; + public final File appPath; + protected boolean firstInit; + public ConcurrentLog log; + protected int serverJobs; + private ConcurrentMap configProps; + private final ConcurrentMap configRemoved; + private final NavigableMap workerThreads; + private YaCyHttpServer httpserver; // implemented HttpServer + private ConcurrentMap upnpPortMap = new ConcurrentHashMap<>(); + private boolean isConnectedViaUpnp; + + public serverSwitch(final File dataPath, final File appPath, final String initPath, final String configPath) { + // we initialize the switchboard with a property file, + // but maintain these properties then later in a new 'config' file + // to reset all changed configs, the config file must + // be deleted, but not the init file + // the only attribute that will always be read from the init is the + // file name of the config file + this.dataPath = dataPath; + this.appPath = appPath; + this.configComment = "This is an automatically generated file, updated by serverAbstractSwitch and initialized by " + initPath; + final File initFile = new File(appPath, initPath); + this.configFile = new File(dataPath, configPath); // propertiesFile(config); + this.firstInit = !this.configFile.exists(); // this is true if the application was started for the first time + new File(this.configFile.getParent()).mkdir(); + + // predefine init's + final ConcurrentMap initProps; + if (initFile.exists()) { + initProps = FileUtils.loadMap(initFile); + } else { + initProps = new ConcurrentHashMap(); + } + + // load config's from last save + if (this.configFile.exists()) { + this.configProps = FileUtils.loadMap(this.configFile); + } else { + this.configProps = new ConcurrentHashMap(); + } + + // overwrite configs with values from environment variables that start with "yacy_" + Properties sysprops = System.getProperties(); + sysprops.forEach((key, value) -> { + String k = (String) key; + if (k.startsWith("yacy.")) { + this.configProps.put(k.substring(5), (String) value); + } + }); + + // remove all values from config that do not appear in init + this.configRemoved = new ConcurrentHashMap(); + Iterator i = this.configProps.keySet().iterator(); + String key; + while (i.hasNext()) { + key = i.next(); + if (!(initProps.containsKey(key))) { + this.configRemoved.put(key, this.configProps.get(key)); + i.remove(); + } + } + + // merge new props from init to config + // this is necessary for migration, when new properties are attached + initProps.putAll(this.configProps); + this.configProps = initProps; + + // save result; this may initially create a config file after + // initialization + saveConfig(); + + // init thread control + this.workerThreads = new TreeMap(); + + // init busy state control + // this.serverJobs = 0; + + // init server tracking + serverAccessTracker.init( + getConfigLong("server.maxTrackingTime", 60 * 60 * 1000), + (int) getConfigLong("server.maxTrackingCount", 1000), + (int) getConfigLong("server.maxTrackingHostCount", 100)); + } + + /** + * get my public IP, either set statically or figure out dynamic This method + * is deprecated because there may be more than one public IPs of this peer, + * i.e. one IPv4 and one IPv6. Please use myPublicIPs() instead + * + * @return the public IP of this peer, if known + */ + public String myPublicIP() { + // if a static IP was configured, we have to return it here ... + final String staticIP = getConfig(SwitchboardConstants.SERVER_STATICIP, ""); + if (staticIP.length() > 0) + return staticIP; + + // otherwise we return the real IP address of this host + final InetAddress pLIP = Domains.myPublicLocalIP(); + if (pLIP != null) + return pLIP.getHostAddress(); + return null; + } + + /** + * Get all my public IPs. If there was a static IP assignment, only one, + * that IP is returned. + * + * @return a set of IPs which are supposed to be my own public IPs + */ + public Set myPublicIPs() { + // if a static IP was configured, we have to return it here ... + final String staticIP = getConfig(SwitchboardConstants.SERVER_STATICIP, ""); + if (staticIP.length() > 0) { + HashSet h = new HashSet<>(); + h.add(staticIP); + return h; + } + + Set h = new LinkedHashSet<>(); + for (InetAddress i : Domains.myPublicIPv6()) { + String s = i.getHostAddress(); + if (Seed.isProperIP(s)) + h.add(Domains.chopZoneID(s)); + } + for (InetAddress i : Domains.myPublicIPv4()) { + String s = i.getHostAddress(); + if (Seed.isProperIP(s)) + h.add(Domains.chopZoneID(s)); + } + return h; + } + + /** + * Gets public port. May differ from local port due to NATting. This method + * will eventually removed once nobody used IPv4 anymore, but until then we + * have to live with it. + * + * @param key + * original key from config (for example "port" or "port.ssl") + * @param dflt + * default value which will be used if no value is found + * @return the public port of this system on its IPv4 address + * + * @see #getLocalPort(String, int) + */ + public int getPublicPort(final String key, final int dflt) { + + if (isConnectedViaUpnp && upnpPortMap.containsKey(key)) { + return upnpPortMap.get(key).intValue(); + } + + // TODO: add way of setting and retrieving port for manual NAT + + return getConfigInt(key, dflt); + } + + /** + * @return local http server port or null if system property is not defined + */ + public Integer getLocalPortSystemProperty() { + String systemDefinedPort = System.getProperty(LOCAL_PORT_SYSTEM_PROPERTY); + Integer localPort = null; + if(systemDefinedPort != null) { + try { + localPort = Integer.parseInt(systemDefinedPort); + } catch(NumberFormatException e) { + log.warn("System property " + LOCAL_PORT_SYSTEM_PROPERTY + " is not valid : it should be a integer."); + } + } + return localPort; + } + + /** + * Wrapper for {@link #getConfigInt(String, int)} to have a more consistent + * API. + * + * Default value 8090 will be used if no value is found in system properties and in configuration. + * + * @return the local http port of this system + * @see #getPublicPort(String, int) + */ + public int getLocalPort() { + + /* A system property "net.yacy.server.localPort" may override configuration + * This is useful when running YaCy inside a container manager such as Heroku which decide which http port to use */ + Integer localPort = getLocalPortSystemProperty(); + if(localPort != null) { + return localPort; + } + return getConfigInt(SwitchboardConstants.SERVER_PORT, 8090); + } + + // a logger for this switchboard + public void setLog(final ConcurrentLog log) { + this.log = log; + } + + public ConcurrentLog getLog() { + return this.log; + } + + /** + * add whole map of key-value pairs to config * - * @return the local http port of this system - * @see #getPublicPort(String, int) - */ - public int getLocalPort() { - - /* A system property "net.yacy.server.localPort" may override configuration - * This is useful when running YaCy inside a container manager such as Heroku which decide which http port to use */ - Integer localPort = getLocalPortSystemProperty(); - if(localPort != null) { - return localPort; - } - return getConfigInt(SwitchboardConstants.SERVER_PORT, 8090); - } - - // a logger for this switchboard - public void setLog(final ConcurrentLog log) { - this.log = log; - } - - public ConcurrentLog getLog() { - return this.log; - } - - /** - * add whole map of key-value pairs to config - * - * @param otherConfigs - */ - public void setConfig(final Map otherConfigs) { - final Iterator> i = otherConfigs.entrySet() - .iterator(); - Map.Entry entry; - while (i.hasNext()) { - entry = i.next(); - setConfig(entry.getKey(), entry.getValue()); - } - } - - public void setConfig(final String key, final boolean value) { - setConfig(key, (value) ? "true" : "false"); - } - - public void setConfig(final String key, final long value) { - setConfig(key, Long.toString(value)); - } - - public void setConfig(final String key, final float value) { - setConfig(key, Float.toString(value)); - } - - public void setConfig(final String key, final double value) { - setConfig(key, Double.toString(value)); - } - - public void setConfig(final String key, final String value) { - // set the value - final String oldValue = this.configProps.put(key, value); - if (oldValue == null || !value.equals(oldValue)) { - saveConfig(); - } - } + * @param otherConfigs + */ + public void setConfig(final Map otherConfigs) { + final Iterator> i = otherConfigs.entrySet() + .iterator(); + Map.Entry entry; + while (i.hasNext()) { + entry = i.next(); + setConfig(entry.getKey(), entry.getValue()); + } + } + + public void setConfig(final String key, final boolean value) { + setConfig(key, (value) ? "true" : "false"); + } + + public void setConfig(final String key, final long value) { + setConfig(key, Long.toString(value)); + } + + public void setConfig(final String key, final float value) { + setConfig(key, Float.toString(value)); + } + + public void setConfig(final String key, final double value) { + setConfig(key, Double.toString(value)); + } + + public void setConfig(final String key, final String value) { + // set the value + final String oldValue = this.configProps.put(key, value); + if (oldValue == null || !value.equals(oldValue)) { + saveConfig(); + } + } public void setConfig(final String key, final String[] value) { StringBuilder sb = new StringBuilder(); @@ -314,134 +319,134 @@ public class serverSwitch { for (String s: value) a[c++] = s; setConfig(key, a); } - - public void removeConfig(final String key) { - this.configProps.remove(key); - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key - * name of the configuration parameter - * @param dflt - * default value which will be used in case parameter can not be - * found or if it is invalid - * @return value if the parameter or default value - */ - public String getConfig(final String key, final String dflt) { - // get the value - final String s = this.configProps.get(key); - - // return value - if (s == null) { - return dflt; - } - return s; - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key - * name of the configuration parameter - * @param dflt - * default value which will be used in case parameter can not be - * found or if it is invalid - * @return value if the parameter or default value - */ - public long getConfigLong(final String key, final long dflt) { - try { - return Long.parseLong(getConfig(key, Long.toString(dflt))); - } catch (final NumberFormatException e) { - return dflt; - } - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key - * name of the configuration parameter - * @param dflt - * default value which will be used in case parameter can not be - * found or if it is invalid - * @return value if the parameter or default value - */ - public float getConfigFloat(final String key, final float dflt) { - try { - return Float.parseFloat(getConfig(key, Float.toString(dflt))); - } catch (final NumberFormatException e) { - return dflt; - } - } - - public boolean isConnectedViaUpnp() { - - return isConnectedViaUpnp; - } - - public void setConnectedViaUpnp(final boolean isConnectedViaUpnp) { - - this.isConnectedViaUpnp = isConnectedViaUpnp; - - if (!isConnectedViaUpnp) { - upnpPortMap.clear(); - } - } - - public void setUpnpPorts(final String key, final int port) { - - upnpPortMap.put(key, Integer.valueOf(port)); - } - - public void removeUpnpPort(final String key) { - upnpPortMap.remove(key); - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key - * name of the configuration parameter - * @param dflt - * default value which will be used in case parameter can not be - * found or if it is invalid - * @return value if the parameter or default value - */ - public int getConfigInt(final String key, final int dflt) { - try { - - return Integer.parseInt(getConfig(key, Integer.toString(dflt))); - - } catch (final NumberFormatException e) { - return dflt; - } - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key - * name of the configuration parameter - * @param dflt - * default value which will be used in case parameter can not be - * found or if it is invalid - * @return value if the parameter or default value - */ - public boolean getConfigBool(final String key, final boolean dflt) { - return Boolean.parseBoolean(getConfig(key, Boolean.toString(dflt))); - } - - /** - * get a configuration parameter list - * @param key - * @param dflt a default list - * @return a list of strings which had been separated by comma in the setting - */ + + public void removeConfig(final String key) { + this.configProps.remove(key); + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public String getConfig(final String key, final String dflt) { + // get the value + final String s = this.configProps.get(key); + + // return value + if (s == null) { + return dflt; + } + return s; + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public long getConfigLong(final String key, final long dflt) { + try { + return Long.parseLong(getConfig(key, Long.toString(dflt))); + } catch (final NumberFormatException e) { + return dflt; + } + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public float getConfigFloat(final String key, final float dflt) { + try { + return Float.parseFloat(getConfig(key, Float.toString(dflt))); + } catch (final NumberFormatException e) { + return dflt; + } + } + + public boolean isConnectedViaUpnp() { + + return isConnectedViaUpnp; + } + + public void setConnectedViaUpnp(final boolean isConnectedViaUpnp) { + + this.isConnectedViaUpnp = isConnectedViaUpnp; + + if (!isConnectedViaUpnp) { + upnpPortMap.clear(); + } + } + + public void setUpnpPorts(final String key, final int port) { + + upnpPortMap.put(key, Integer.valueOf(port)); + } + + public void removeUpnpPort(final String key) { + upnpPortMap.remove(key); + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public int getConfigInt(final String key, final int dflt) { + try { + + return Integer.parseInt(getConfig(key, Integer.toString(dflt))); + + } catch (final NumberFormatException e) { + return dflt; + } + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public boolean getConfigBool(final String key, final boolean dflt) { + return Boolean.parseBoolean(getConfig(key, Boolean.toString(dflt))); + } + + /** + * get a configuration parameter list + * @param key + * @param dflt a default list + * @return a list of strings which had been separated by comma in the setting + */ public String[] getConfigArray(final String key, final String dflt) { return CommonPattern.COMMA.split(this.getConfig(key, dflt)); - } + } /** * get a configuration parameter set @@ -454,70 +459,70 @@ public class serverSwitch { return h; } - /** - * Create a File instance for a configuration setting specifying a path. - * - * @param key - * config key - * @param dflt - * default path value, that is used when there is no value - * key in the configuration. - * @return if the value of the setting is an absolute path String, then the - * returned File is derived from this setting only. Otherwise the - * path's file is constructed from the applications root path + the - * relative path setting. - */ - public File getDataPath(final String key, final String dflt) { - return getFileByPath(key, dflt, this.dataPath); - } - - /** - * return file at path from config entry "key", or fallback to default dflt - * - * @param key - * @param dflt - * @return - */ - public File getAppPath(final String key, final String dflt) { - return getFileByPath(key, dflt, this.appPath); - } - - private File getFileByPath(String key, String dflt, File prefix) { - final String path = getConfig(key, dflt).replace('\\', '/'); - final File f = new File(path); - return (f.isAbsolute() ? new File(f.getAbsolutePath()) : new File( - prefix, path)); - } - - public Iterator configKeys() { - return this.configProps.keySet().iterator(); - } - - /** - * write the changes to permanent storage (File) - */ - private void saveConfig() { - ConcurrentMap configPropsCopy = new ConcurrentHashMap(); - configPropsCopy.putAll(this.configProps); // avoid concurrency problems - FileUtils.saveMap(this.configFile, configPropsCopy, this.configComment); - } - - /** - * Gets configuration parameters which have been removed during - * initialization. - * - * @return contains parameter name as key and parameter value as value - */ - public ConcurrentMap getRemoved() { - return this.configRemoved; - } - - /** - * @return the default configuration properties loaded form the - * defaults/yacy.init file. The properties are empty when the file can - * not be read for some reason. - */ - public Properties loadDefaultConfig() { + /** + * Create a File instance for a configuration setting specifying a path. + * + * @param key + * config key + * @param dflt + * default path value, that is used when there is no value + * key in the configuration. + * @return if the value of the setting is an absolute path String, then the + * returned File is derived from this setting only. Otherwise the + * path's file is constructed from the applications root path + the + * relative path setting. + */ + public File getDataPath(final String key, final String dflt) { + return getFileByPath(key, dflt, this.dataPath); + } + + /** + * return file at path from config entry "key", or fallback to default dflt + * + * @param key + * @param dflt + * @return + */ + public File getAppPath(final String key, final String dflt) { + return getFileByPath(key, dflt, this.appPath); + } + + private File getFileByPath(String key, String dflt, File prefix) { + final String path = getConfig(key, dflt).replace('\\', '/'); + final File f = new File(path); + return (f.isAbsolute() ? new File(f.getAbsolutePath()) : new File( + prefix, path)); + } + + public Iterator configKeys() { + return this.configProps.keySet().iterator(); + } + + /** + * write the changes to permanent storage (File) + */ + private void saveConfig() { + ConcurrentMap configPropsCopy = new ConcurrentHashMap(); + configPropsCopy.putAll(this.configProps); // avoid concurrency problems + FileUtils.saveMap(this.configFile, configPropsCopy, this.configComment); + } + + /** + * Gets configuration parameters which have been removed during + * initialization. + * + * @return contains parameter name as key and parameter value as value + */ + public ConcurrentMap getRemoved() { + return this.configRemoved; + } + + /** + * @return the default configuration properties loaded form the + * defaults/yacy.init file. The properties are empty when the file can + * not be read for some reason. + */ + public Properties loadDefaultConfig() { final Properties config = new Properties(); try (final FileInputStream fis = new FileInputStream(new File(this.appPath, "defaults/yacy.init"))) { config.load(fis); @@ -527,225 +532,225 @@ public class serverSwitch { log.severe("Could not read configuration file."); } return config; - } - - public void deployThread(final String threadName, - final String threadShortDescription, - final String threadLongDescription, final String threadMonitorURL, - final BusyThread newThread, final long startupDelay) { - deployThread( - threadName, - threadShortDescription, - threadLongDescription, - threadMonitorURL, - newThread, - startupDelay, - Long.parseLong(getConfig(threadName + "_idlesleep", "1000")), - Long.parseLong(getConfig(threadName + "_busysleep", "100")), - Long.parseLong(getConfig(threadName + "_memprereq", "1048576")), - Double.parseDouble(getConfig(threadName + "_loadprereq", "9.0"))); - } - - public void deployThread(final String threadName, - final String threadShortDescription, - final String threadLongDescription, final String threadMonitorURL, - final BusyThread newThread, final long startupDelay, - final long initialIdleSleep, final long initialBusySleep, - final long initialMemoryPreRequisite, - final double initialLoadPreRequisite) { - if (newThread.isAlive()) { - throw new RuntimeException( - "undeployed threads must not live; they are started as part of the deployment"); - } - newThread.setStartupSleep(startupDelay); - long x; - try { - x = Long.parseLong(getConfig(threadName + "_idlesleep", "novalue")); - newThread.setIdleSleep(x); - } catch (final NumberFormatException e) { - newThread.setIdleSleep(initialIdleSleep); - setConfig(threadName + "_idlesleep", initialIdleSleep); - } - try { - x = Long.parseLong(getConfig(threadName + "_busysleep", "novalue")); - newThread.setBusySleep(x); - } catch (final NumberFormatException e) { - newThread.setBusySleep(initialBusySleep); - setConfig(threadName + "_busysleep", initialBusySleep); - } - try { - x = Long.parseLong(getConfig(threadName + "_memprereq", "novalue")); - newThread.setMemPreReqisite(x); - } catch (final NumberFormatException e) { - newThread.setMemPreReqisite(initialMemoryPreRequisite); - setConfig(threadName + "_memprereq", initialMemoryPreRequisite); - } - try { - final double load = Double.parseDouble(getConfig(threadName - + "_loadprereq", "novalue")); - newThread.setLoadPreReqisite(load); - } catch (final NumberFormatException e) { - newThread.setLoadPreReqisite(initialLoadPreRequisite); - setConfig(threadName + "_loadprereq", - (float) initialLoadPreRequisite); - } - newThread.setDescription(threadShortDescription, threadLongDescription, - threadMonitorURL); - this.workerThreads.put(threadName, newThread); - // start the thread - if (this.workerThreads.containsKey(threadName)) { - newThread.start(); - } - } - - public BusyThread getThread(final String threadName) { - return this.workerThreads.get(threadName); - } - - public void setThreadPerformance(final String threadName, - final long idleMillis, final long busyMillis, - final long memprereqBytes, final double loadprereq) { - final BusyThread thread = this.workerThreads.get(threadName); - if (thread != null) { - setConfig(threadName + "_idlesleep", - thread.setIdleSleep(idleMillis)); - setConfig(threadName + "_busysleep", - thread.setBusySleep(busyMillis)); - setConfig(threadName + "_memprereq", memprereqBytes); - thread.setMemPreReqisite(memprereqBytes); - setConfig(threadName + "_loadprereq", (float) loadprereq); - thread.setLoadPreReqisite(loadprereq); - } - } - - public synchronized void terminateThread(final String threadName, - final boolean waitFor) { - if (this.workerThreads.containsKey(threadName)) { - ((WorkflowThread) this.workerThreads.get(threadName)) - .terminate(waitFor); - this.workerThreads.remove(threadName); - } - } - - public void intermissionAllThreads(final long pause) { - final Iterator e = this.workerThreads.keySet().iterator(); - while (e.hasNext()) { - this.workerThreads.get(e.next()).intermission(pause); - } - } - - public synchronized void terminateAllThreads(final boolean waitFor) { - Iterator e = this.workerThreads.keySet().iterator(); - while (e.hasNext()) { - ((WorkflowThread) this.workerThreads.get(e.next())) - .terminate(false); - } - if (waitFor) { - e = this.workerThreads.keySet().iterator(); - while (e.hasNext()) { - ((WorkflowThread) this.workerThreads.get(e.next())) - .terminate(true); - e.remove(); - } - } - } - - public Iterator /* of serverThread-Names (String) */threadNames() { - return this.workerThreads.keySet().iterator(); - } - - public File getDataPath() { - return this.dataPath; - } - - public File getAppPath() { - return this.appPath; - } - - @Override - public String toString() { - return this.configProps.toString(); - } - - public void handleBusyState(final int jobs) { - this.serverJobs = jobs; - } - - /** - * Retrieve text data (e. g. config file) from file file may be an url or a - * filename with path relative to rootPath parameter - * - * @param file - * url or filename - * @param rootPath - * searchpath for file - * @param file - * file to use when remote fetching fails (null if unused) - */ - public Reader getConfigFileFromWebOrLocally(final String uri, - final String rootPath, final File file) throws IOException, - FileNotFoundException { - if (uri.startsWith("http://") || uri.startsWith("https://")) { - final String[] uris = CommonPattern.COMMA.split(uri); - for (String netdef : uris) { - netdef = netdef.trim(); - try { - final RequestHeader reqHeader = new RequestHeader(); - reqHeader - .put(HeaderFramework.USER_AGENT, - ClientIdentification.yacyInternetCrawlerAgent.userAgent); - final HTTPClient client = new HTTPClient( - ClientIdentification.yacyInternetCrawlerAgent); - client.setHeader(reqHeader.entrySet()); - byte[] data = client - .GETbytes( - uri, - getConfig( - SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, - "admin"), - getConfig( - SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, - ""), false); - if (data == null || data.length == 0) { - continue; - } - // save locally in case next fetch fails - if (file != null) { - try(/* Automatically closed by this try-with-resource statement */ - FileOutputStream f = new FileOutputStream(file); - ) { - f.write(data); - } - } - return new InputStreamReader(new BufferedInputStream( - new ByteArrayInputStream(data))); - } catch (final Exception e) { - continue; - } - } - if (file != null && file.exists()) { - return new FileReader(file); - } - throw new FileNotFoundException(); - } - final File f = (uri.length() > 0 && uri.startsWith("/")) ? new File(uri) - : new File(rootPath, uri); - if (f.exists()) - return new FileReader(f); - throw new FileNotFoundException(f.toString()); - } - - /** - * set/remember jetty server - * - * @param jettyserver - */ - public void setHttpServer(YaCyHttpServer jettyserver) { - this.httpserver = jettyserver; - } - - public YaCyHttpServer getHttpServer() { - return httpserver; - } + } + + public void deployThread(final String threadName, + final String threadShortDescription, + final String threadLongDescription, final String threadMonitorURL, + final BusyThread newThread, final long startupDelay) { + deployThread( + threadName, + threadShortDescription, + threadLongDescription, + threadMonitorURL, + newThread, + startupDelay, + Long.parseLong(getConfig(threadName + "_idlesleep", "1000")), + Long.parseLong(getConfig(threadName + "_busysleep", "100")), + Long.parseLong(getConfig(threadName + "_memprereq", "1048576")), + Double.parseDouble(getConfig(threadName + "_loadprereq", "9.0"))); + } + + public void deployThread(final String threadName, + final String threadShortDescription, + final String threadLongDescription, final String threadMonitorURL, + final BusyThread newThread, final long startupDelay, + final long initialIdleSleep, final long initialBusySleep, + final long initialMemoryPreRequisite, + final double initialLoadPreRequisite) { + if (newThread.isAlive()) { + throw new RuntimeException( + "undeployed threads must not live; they are started as part of the deployment"); + } + newThread.setStartupSleep(startupDelay); + long x; + try { + x = Long.parseLong(getConfig(threadName + "_idlesleep", "novalue")); + newThread.setIdleSleep(x); + } catch (final NumberFormatException e) { + newThread.setIdleSleep(initialIdleSleep); + setConfig(threadName + "_idlesleep", initialIdleSleep); + } + try { + x = Long.parseLong(getConfig(threadName + "_busysleep", "novalue")); + newThread.setBusySleep(x); + } catch (final NumberFormatException e) { + newThread.setBusySleep(initialBusySleep); + setConfig(threadName + "_busysleep", initialBusySleep); + } + try { + x = Long.parseLong(getConfig(threadName + "_memprereq", "novalue")); + newThread.setMemPreReqisite(x); + } catch (final NumberFormatException e) { + newThread.setMemPreReqisite(initialMemoryPreRequisite); + setConfig(threadName + "_memprereq", initialMemoryPreRequisite); + } + try { + final double load = Double.parseDouble(getConfig(threadName + + "_loadprereq", "novalue")); + newThread.setLoadPreReqisite(load); + } catch (final NumberFormatException e) { + newThread.setLoadPreReqisite(initialLoadPreRequisite); + setConfig(threadName + "_loadprereq", + (float) initialLoadPreRequisite); + } + newThread.setDescription(threadShortDescription, threadLongDescription, + threadMonitorURL); + this.workerThreads.put(threadName, newThread); + // start the thread + if (this.workerThreads.containsKey(threadName)) { + newThread.start(); + } + } + + public BusyThread getThread(final String threadName) { + return this.workerThreads.get(threadName); + } + + public void setThreadPerformance(final String threadName, + final long idleMillis, final long busyMillis, + final long memprereqBytes, final double loadprereq) { + final BusyThread thread = this.workerThreads.get(threadName); + if (thread != null) { + setConfig(threadName + "_idlesleep", + thread.setIdleSleep(idleMillis)); + setConfig(threadName + "_busysleep", + thread.setBusySleep(busyMillis)); + setConfig(threadName + "_memprereq", memprereqBytes); + thread.setMemPreReqisite(memprereqBytes); + setConfig(threadName + "_loadprereq", (float) loadprereq); + thread.setLoadPreReqisite(loadprereq); + } + } + + public synchronized void terminateThread(final String threadName, + final boolean waitFor) { + if (this.workerThreads.containsKey(threadName)) { + ((WorkflowThread) this.workerThreads.get(threadName)) + .terminate(waitFor); + this.workerThreads.remove(threadName); + } + } + + public void intermissionAllThreads(final long pause) { + final Iterator e = this.workerThreads.keySet().iterator(); + while (e.hasNext()) { + this.workerThreads.get(e.next()).intermission(pause); + } + } + + public synchronized void terminateAllThreads(final boolean waitFor) { + Iterator e = this.workerThreads.keySet().iterator(); + while (e.hasNext()) { + ((WorkflowThread) this.workerThreads.get(e.next())) + .terminate(false); + } + if (waitFor) { + e = this.workerThreads.keySet().iterator(); + while (e.hasNext()) { + ((WorkflowThread) this.workerThreads.get(e.next())) + .terminate(true); + e.remove(); + } + } + } + + public Iterator /* of serverThread-Names (String) */threadNames() { + return this.workerThreads.keySet().iterator(); + } + + public File getDataPath() { + return this.dataPath; + } + + public File getAppPath() { + return this.appPath; + } + + @Override + public String toString() { + return this.configProps.toString(); + } + + public void handleBusyState(final int jobs) { + this.serverJobs = jobs; + } + + /** + * Retrieve text data (e. g. config file) from file file may be an url or a + * filename with path relative to rootPath parameter + * + * @param file + * url or filename + * @param rootPath + * searchpath for file + * @param file + * file to use when remote fetching fails (null if unused) + */ + public Reader getConfigFileFromWebOrLocally(final String uri, + final String rootPath, final File file) throws IOException, + FileNotFoundException { + if (uri.startsWith("http://") || uri.startsWith("https://")) { + final String[] uris = CommonPattern.COMMA.split(uri); + for (String netdef : uris) { + netdef = netdef.trim(); + try { + final RequestHeader reqHeader = new RequestHeader(); + reqHeader + .put(HeaderFramework.USER_AGENT, + ClientIdentification.yacyInternetCrawlerAgent.userAgent); + final HTTPClient client = new HTTPClient( + ClientIdentification.yacyInternetCrawlerAgent); + client.setHeader(reqHeader.entrySet()); + byte[] data = client + .GETbytes( + uri, + getConfig( + SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, + "admin"), + getConfig( + SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, + ""), false); + if (data == null || data.length == 0) { + continue; + } + // save locally in case next fetch fails + if (file != null) { + try(/* Automatically closed by this try-with-resource statement */ + FileOutputStream f = new FileOutputStream(file); + ) { + f.write(data); + } + } + return new InputStreamReader(new BufferedInputStream( + new ByteArrayInputStream(data))); + } catch (final Exception e) { + continue; + } + } + if (file != null && file.exists()) { + return new FileReader(file); + } + throw new FileNotFoundException(); + } + final File f = (uri.length() > 0 && uri.startsWith("/")) ? new File(uri) + : new File(rootPath, uri); + if (f.exists()) + return new FileReader(f); + throw new FileNotFoundException(f.toString()); + } + + /** + * set/remember jetty server + * + * @param jettyserver + */ + public void setHttpServer(YaCyHttpServer jettyserver) { + this.httpserver = jettyserver; + } + + public YaCyHttpServer getHttpServer() { + return httpserver; + } } diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 6b9a5e0d2..6871adfde 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -157,13 +157,13 @@ public final class yacy { mkdirsIfNeseccary(appHome); File f = new File(dataHome, "DATA/"); mkdirsIfNeseccary(f); - if (!(f.exists())) { - System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate."); - System.exit(-1); - } - + if (!(f.exists())) { + System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate."); + System.exit(-1); + } + // set jvm tmpdir to a subdir for easy cleanup (as extensive use file.deleteonexit waists memory during long runs, as todelete files names are collected and never cleaned up during runtime) - // keep this as earlier as possible, as any other class can use the "java.io.tmpdir" property, even the log manager, when the log file pattern uses "%t" as an alias for the tmp directory + // keep this as earlier as possible, as any other class can use the "java.io.tmpdir" property, even the log manager, when the log file pattern uses "%t" as an alias for the tmp directory try { tmpdir = java.nio.file.Files.createTempDirectory("yacy-tmp-").toString(); // creates sub dir in jvm's temp (see System.property "java.io.tempdir") System.setProperty("java.io.tmpdir", tmpdir); @@ -173,9 +173,9 @@ public final class yacy { f = new File(dataHome, "DATA/LOG/"); mkdirsIfNeseccary(f); f = new File(f, "yacy.logging"); - final File f0 = new File(appHome, "defaults/yacy.logging"); - if (!f.exists() || f0.lastModified() > f.lastModified()) try { - Files.copy(f0, f); + final File f0 = new File(appHome, "defaults/yacy.logging"); + if (!f.exists() || f0.lastModified() > f.lastModified()) try { + Files.copy(f0, f); } catch (final IOException e){ System.out.println("could not copy yacy.logging: " + e.getMessage()); } @@ -194,17 +194,17 @@ public final class yacy { ConcurrentLog.config("STARTUP", "Maximum file system path length: " + OS.maxPathLength); f = new File(dataHome, "DATA/yacy.running"); - final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator); if (!f.createNewFile()) ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be created!"); try { new FileOutputStream(f).write(Integer.toString(OS.getPID()).getBytes()); } catch (final Exception e) { } // write PID f.deleteOnExit(); FileChannel channel = null; FileLock lock = null; try { - channel = new RandomAccessFile(f,"rw").getChannel(); - lock = channel.tryLock(); // lock yacy.running + channel = new RandomAccessFile(f,"rw").getChannel(); + lock = channel.tryLock(); // lock yacy.running } catch (final Exception e) { } + final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator); try { sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), conf); } catch (final RuntimeException e) { @@ -227,7 +227,7 @@ public final class yacy { sb.setConfig("htTemplatePath", "htroot/env/templates"); double oldVer; - try { + try { String tmpversion = sb.getConfig(Seed.VERSION, ""); if (tmpversion.isEmpty()) { // before 1.83009737 only the svnRevision nr was in config (like 9737) tmpversion = yacyBuildProperties.getVersion(); @@ -242,7 +242,7 @@ public final class yacy { } } catch (final NumberFormatException e) { oldVer = 0.0d; - } + } final double newRev = Double.parseDouble(yacyBuildProperties.getLongVersion()); sb.setConfig(Seed.VERSION, yacyBuildProperties.getLongVersion()); sb.setConfig("applicationRoot", appHome.toString()); @@ -294,7 +294,7 @@ public final class yacy { final int port = sb.getLocalPort(); try { // start http server - YaCyHttpServer httpServer; + YaCyHttpServer httpServer; httpServer = new Jetty9HttpServerImpl(port); httpServer.startupServer(); sb.setHttpServer(httpServer); @@ -312,10 +312,10 @@ public final class yacy { /* YaCy main startup process must not hang because browser opening is long or fails. * Let's open try opening the browser in a separate thread */ new Thread("Browser opening") { - @Override - public void run() { + @Override + public void run() { Browser.openBrowser(("http://localhost:"+port) + "/" + browserPopUpPage); - } + } }.start(); // Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage); } catch (final Throwable e) { @@ -347,11 +347,11 @@ public final class yacy { } catch (final IOException e) { //Error } finally { - try { - br.close(); - } catch(IOException ioe) { - ConcurrentLog.warn("STARTUP", "Could not close " + tmplang + " version file"); - } + try { + br.close(); + } catch(IOException ioe) { + ConcurrentLog.warn("STARTUP", "Could not close " + tmplang + " version file"); + } } if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) { @@ -420,35 +420,35 @@ public final class yacy { } catch (final Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790) } - /** - * @param f - */ - private static void delete(final File f) { - if(!f.delete()) - ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be deleted!"); - } - - /** - * @see File#mkdir() - * @param path - */ - private static void mkdirIfNeseccary(final File path) { - if (!(path.exists())) - if(!path.mkdir()) - ConcurrentLog.warn("STARTUP", "could not create directory "+ path.toString()); - } - - /** - * @see File#mkdirs() - * @param path - */ - public static void mkdirsIfNeseccary(final File path) { - if (!(path.exists())) - if(!path.mkdirs()) - ConcurrentLog.warn("STARTUP", "could not create directories "+ path.toString()); - } - - /** + /** + * @param f + */ + private static void delete(final File f) { + if(!f.delete()) + ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be deleted!"); + } + + /** + * @see File#mkdir() + * @param path + */ + private static void mkdirIfNeseccary(final File path) { + if (!(path.exists())) + if(!path.mkdir()) + ConcurrentLog.warn("STARTUP", "could not create directory "+ path.toString()); + } + + /** + * @see File#mkdirs() + * @param path + */ + public static void mkdirsIfNeseccary(final File path) { + if (!(path.exists())) + if(!path.mkdirs()) + ConcurrentLog.warn("STARTUP", "could not create directories "+ path.toString()); + } + + /** * Loads the configuration from the data-folder. * FIXME: Why is this called over and over again from every method, instead * of setting the configurationdata once for this class in main? @@ -470,8 +470,8 @@ public final class yacy { final Properties config = new Properties(); FileInputStream fis = null; - try { - fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf")); + try { + fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf")); config.load(fis); } catch (final FileNotFoundException e) { ConcurrentLog.severe(mes, "could not find configuration file."); @@ -480,13 +480,13 @@ public final class yacy { ConcurrentLog.severe(mes, "could not read configuration file."); System.exit(-1); } finally { - if(fis != null) { - try { - fis.close(); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } + if(fis != null) { + try { + fis.close(); + } catch (final IOException e) { + ConcurrentLog.logException(e); + } + } } return config; @@ -536,7 +536,7 @@ public final class yacy { final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); // con.setHeader(requestHeader.entrySet()); try { - /* First get a valid transaction token using HTTP GET */ + /* First get a valid transaction token using HTTP GET */ con.GETbytes("http://localhost:"+ port +"/" + path, adminUser, encodedPassword, false); if (con.getStatusCode() != HttpStatus.SC_OK) { @@ -545,7 +545,7 @@ public final class yacy { final Header transactionTokenHeader = con.getHttpResponse().getFirstHeader(HeaderFramework.X_YACY_TRANSACTION_TOKEN); if(transactionTokenHeader == null) { - throw new IOException("Could not retrieve a valid transaction token"); + throw new IOException("Could not retrieve a valid transaction token"); } /* Then POST the request */ @@ -554,36 +554,36 @@ public final class yacy { if (con.getStatusCode() >= HttpStatus.SC_OK && con.getStatusCode() < HttpStatus.SC_MULTIPLE_CHOICES) { ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription); } else { - ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine()); - + ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine()); + try { - HTTPClient.closeConnectionManager(); - } catch (final InterruptedException e1) { - e1.printStackTrace(); - } + HTTPClient.closeConnectionManager(); + } catch (final InterruptedException e1) { + e1.printStackTrace(); + } RemoteInstance.closeConnectionManager(); - + System.exit(-1); } } catch (final IOException e) { ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage()); try { - HTTPClient.closeConnectionManager(); - } catch (final InterruptedException e1) { - e1.printStackTrace(); - } + HTTPClient.closeConnectionManager(); + } catch (final InterruptedException e1) { + e1.printStackTrace(); + } RemoteInstance.closeConnectionManager(); System.exit(-1); } try { - HTTPClient.closeConnectionManager(); - } catch (final InterruptedException e) { - e.printStackTrace(); - } + HTTPClient.closeConnectionManager(); + } catch (final InterruptedException e) { + e.printStackTrace(); + } RemoteInstance.closeConnectionManager(); // finished @@ -609,7 +609,7 @@ public final class yacy { ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription); } else { - ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine()); + ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine()); System.exit(-1); } } catch (final IOException e) { @@ -618,10 +618,10 @@ public final class yacy { } try { - HTTPClient.closeConnectionManager(); - } catch (final InterruptedException e) { - e.printStackTrace(); - } + HTTPClient.closeConnectionManager(); + } catch (final InterruptedException e) { + e.printStackTrace(); + } RemoteInstance.closeConnectionManager(); // finished @@ -678,11 +678,11 @@ public final class yacy { ConcurrentLog.logException(ex); ConcurrentLog.severe("Startup", "cannot read " + configFile.toString() + ", please delete the corrupted file if problem persits"); } finally { - try { - fis.close(); - } catch (IOException e) { - ConcurrentLog.warn("Startup", "Could not close file " + configFile); - } + try { + fis.close(); + } catch (IOException e) { + ConcurrentLog.warn("Startup", "Could not close file " + configFile); + } } } } @@ -696,68 +696,68 @@ public final class yacy { */ public static void main(String args[]) { - try { - // check assertion status - //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true); - boolean assertionenabled = false; - assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement" - if (assertionenabled) System.out.println("Asserts are enabled"); - - // check memory amount - System.gc(); - final long startupMemFree = MemoryControl.free(); - final long startupMemTotal = MemoryControl.total(); - - // maybe go into headless awt mode: we have three cases depending on OS and one exception: - // windows : better do not go into headless mode - // mac : go into headless mode because an application is shown in gui which may not be wanted - // linux : go into headless mode because this does not need any head operation - // exception : if the -gui option is used then do not go into headless mode since that uses a gui - boolean headless = true; - if (OS.isWindows) headless = false; - if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) headless = false; - System.setProperty("java.awt.headless", headless ? "true" : "false"); - - String s = ""; for (final String a: args) s += a + " "; - yacyRelease.startParameter = s.trim(); - - File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/')); - File dataRoot = applicationRoot; - //System.out.println("args.length=" + args.length); - //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]"); - if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-startup") || args[0].equals("-start"))) { - // normal start-up of yacy - if (args.length > 1) { - dataRoot = new File(args[1]); - if(!dataRoot.isAbsolute()) { - /* data root folder provided as a path relative to the user home folder */ - dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]); - } - } + try { + // check assertion status + //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true); + boolean assertionenabled = false; + assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement" + if (assertionenabled) System.out.println("Asserts are enabled"); + + // check memory amount + System.gc(); + final long startupMemFree = MemoryControl.free(); + final long startupMemTotal = MemoryControl.total(); + + // maybe go into headless awt mode: we have three cases depending on OS and one exception: + // windows : better do not go into headless mode + // mac : go into headless mode because an application is shown in gui which may not be wanted + // linux : go into headless mode because this does not need any head operation + // exception : if the -gui option is used then do not go into headless mode since that uses a gui + boolean headless = true; + if (OS.isWindows) headless = false; + if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) headless = false; + System.setProperty("java.awt.headless", headless ? "true" : "false"); + + String s = ""; for (final String a: args) s += a + " "; + yacyRelease.startParameter = s.trim(); + + File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/')); + File dataRoot = applicationRoot; + //System.out.println("args.length=" + args.length); + //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]"); + if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-startup") || args[0].equals("-start"))) { + // normal start-up of yacy + if (args.length > 1) { + dataRoot = new File(args[1]); + if(!dataRoot.isAbsolute()) { + /* data root folder provided as a path relative to the user home folder */ + dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]); + } + } preReadSavedConfigandInit(dataRoot); - startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false); - } else if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) { - // start-up of yacy with gui - if (args.length > 1) { - dataRoot = new File(args[1]); - if(!dataRoot.isAbsolute()) { - /* data root folder provided as a path relative to the user home folder */ - dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]); - } - } + startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false); + } else if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) { + // start-up of yacy with gui + if (args.length > 1) { + dataRoot = new File(args[1]); + if(!dataRoot.isAbsolute()) { + /* data root folder provided as a path relative to the user home folder */ + dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]); + } + } preReadSavedConfigandInit(dataRoot); - startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true); - } else if ((args.length >= 1) && ((args[0].toLowerCase(Locale.ROOT).equals("-shutdown")) || (args[0].equals("-stop")))) { - // normal shutdown of yacy - if (args.length == 2) applicationRoot= new File(args[1]); - shutdown(applicationRoot); - } else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-update"))) { - // aut-update yacy - if (args.length == 2) applicationRoot= new File(args[1]); - update(applicationRoot); - } else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-version"))) { - // show yacy version - System.out.println(copyright); + startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true); + } else if ((args.length >= 1) && ((args[0].toLowerCase(Locale.ROOT).equals("-shutdown")) || (args[0].equals("-stop")))) { + // normal shutdown of yacy + if (args.length == 2) applicationRoot= new File(args[1]); + shutdown(applicationRoot); + } else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-update"))) { + // aut-update yacy + if (args.length == 2) applicationRoot= new File(args[1]); + update(applicationRoot); + } else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-version"))) { + // show yacy version + System.out.println(copyright); } else if ((args.length > 1) && (args[0].toLowerCase(Locale.ROOT).equals("-config"))) { // set config parameter. Special handling of adminAccount=user:pwd (generates md5 encoded password) // on Windows parameter should be enclosed in doublequotes to accept = sign (e.g. -config "port=8090" "port.ssl=8043") @@ -805,15 +805,15 @@ public final class yacy { System.out.println(); } } else { - if (args.length == 1) { - applicationRoot= new File(args[0]); - } + if (args.length == 1) { + applicationRoot= new File(args[0]); + } preReadSavedConfigandInit(dataRoot); - startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false); - } - } finally { - ConcurrentLog.shutdown(); - } + startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false); + } + } finally { + ConcurrentLog.shutdown(); + } } }