From 1e6e69bc40ea5f3a66fc053c20c2e73c7d47aa98 Mon Sep 17 00:00:00 2001 From: Marc Nause Date: Tue, 7 Oct 2014 13:10:06 +0200 Subject: [PATCH] Finished implementation of UPNP: *) will try other ports if YaCy standard ports are not available *) distinguish between internal and external port (not sure if this works 100%) Still to add: propery in config to enter own external port (in case of manually configured NAT) --- htroot/ConfigBasic.java | 4 +- htroot/ConfigPortal.java | 2 +- htroot/ConfigSearchBox.java | 2 +- htroot/CrawlStartScanner_p.java | 4 +- htroot/Load_MediawikiWiki.java | 2 +- htroot/Load_PHPBB3.java | 2 +- htroot/SettingsAck_p.java | 4 +- htroot/Settings_p.java | 2 +- htroot/Table_API_p.java | 4 +- htroot/api/push_p.java | 2 +- htroot/opensearchdescription.java | 2 +- htroot/yacysearch.java | 4 +- htroot/yacysearch_location.java | 4 +- source/net/yacy/gui/Tray.java | 4 +- .../net/yacy/http/Jetty9HttpServerImpl.java | 6 +- source/net/yacy/peers/Seed.java | 4 +- source/net/yacy/peers/SeedDB.java | 2 +- source/net/yacy/search/Switchboard.java | 6 +- source/net/yacy/server/http/HTTPDemon.java | 6 +- source/net/yacy/server/serverSwitch.java | 1165 +++++++++-------- source/net/yacy/utils/upnp/UPnP.java | 131 +- source/net/yacy/yacy.java | 4 +- 22 files changed, 765 insertions(+), 601 deletions(-) diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java index ffacc8309..0e2d5a6fc 100644 --- a/htroot/ConfigBasic.java +++ b/htroot/ConfigBasic.java @@ -100,7 +100,7 @@ public class ConfigBasic { port = post.getLong("port", 8090); ssl = post.getBoolean("withssl"); } else { - port = env.getConfigLong("port", 8090); //this allows a low port, but it will only get one, if the user edits the config himself. + port = env.getLocalPort("port", 8090); //this allows a low port, but it will only get one, if the user edits the config himself. ssl = env.getConfigBool("server.https", false); } if (ssl) prop.put("withsslenabled_sslport",env.getHttpServer().getSslPort()); @@ -266,7 +266,7 @@ public class ConfigBasic { // set default values prop.putHTML("defaultName", sb.peers.mySeed().getName()); - prop.putHTML("defaultPort", env.getConfig("port", "8090")); + prop.put("defaultPort", env.getLocalPort("port", 8090)); prop.put("withsslenabled", env.getConfigBool("server.https", false) ? 1 : 0); lang = env.getConfig("locale.language", "default"); // re-assign lang, may have changed prop.put("lang_de", "0"); diff --git a/htroot/ConfigPortal.java b/htroot/ConfigPortal.java index 61a335d30..474d22644 100644 --- a/htroot/ConfigPortal.java +++ b/htroot/ConfigPortal.java @@ -221,7 +221,7 @@ public class ConfigPortal { String myaddress = (sb.peers == null) ? null : sb.peers.mySeed() == null ? null : sb.peers.mySeed().getPublicAddress(); if (myaddress == null) { - myaddress = "localhost:" + sb.getConfig("port", "8090"); + myaddress = "localhost:" + sb.getLocalPort("port", 8090); } prop.put("myaddress", myaddress); return prop; diff --git a/htroot/ConfigSearchBox.java b/htroot/ConfigSearchBox.java index 834558dd6..393777203 100644 --- a/htroot/ConfigSearchBox.java +++ b/htroot/ConfigSearchBox.java @@ -35,7 +35,7 @@ public class ConfigSearchBox { final Switchboard sb = (Switchboard) env; String myaddress = sb.peers.mySeed().getPublicAddress(); - if (myaddress == null) myaddress = "localhost:" + sb.getConfig("port", "8090"); + if (myaddress == null) myaddress = "localhost:" + sb.getLocalPort("port", 8090); prop.put("myaddress", myaddress); return prop; } diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index a8651b240..ed5386376 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -214,7 +214,7 @@ public class CrawlStartScanner_p path += "&crawlingURL=" + url.toNormalform(true); WorkTables.execAPICall( Domains.LOCALHOST, - (int) sb.getConfigLong("port", 8090), + sb.getLocalPort("port", 8090), path, pk, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), @@ -261,7 +261,7 @@ public class CrawlStartScanner_p path += "&crawlingURL=" + urlString; WorkTables.execAPICall( Domains.LOCALHOST, - (int) sb.getConfigLong("port", 8090), + sb.getLocalPort("port", 8090), path, u.hash(), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), diff --git a/htroot/Load_MediawikiWiki.java b/htroot/Load_MediawikiWiki.java index a183a3c95..d40647951 100644 --- a/htroot/Load_MediawikiWiki.java +++ b/htroot/Load_MediawikiWiki.java @@ -39,7 +39,7 @@ public class Load_MediawikiWiki { // define visible variables String a = sb.peers.mySeed().getPublicAddress(); - if (a == null) a = "localhost:" + sb.getConfig("port", "8090"); + if (a == null) a = "localhost:" + sb.getLocalPort("port", 8090); final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet"); final String repository = "http://" + a + "/repository/"; prop.put("starturl", (intranet) ? repository : "http://"); diff --git a/htroot/Load_PHPBB3.java b/htroot/Load_PHPBB3.java index 6cf03d789..299a2bd97 100644 --- a/htroot/Load_PHPBB3.java +++ b/htroot/Load_PHPBB3.java @@ -39,7 +39,7 @@ public class Load_PHPBB3 { // define visible variables String a = sb.peers.mySeed().getPublicAddress(); - if (a == null) a = "localhost:" + sb.getConfig("port", "8090"); + if (a == null) a = "localhost:" + sb.getLocalPort("port", 8090); final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet"); final String repository = "http://" + a + "/repository/"; prop.put("starturl", (intranet) ? repository : "http://"); diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index 3251b384d..a0006c9e7 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -99,7 +99,7 @@ public class SettingsAck_p { /* * display port info */ - prop.putHTML("info_port", env.getConfig("port", "8090")); + prop.put("info_port", env.getLocalPort("port", 8090)); prop.put("info_restart", "0"); // read and process data @@ -483,7 +483,7 @@ public class SettingsAck_p { // change https port if (post.containsKey("port.ssl")) { int port = post.getInt("port.ssl", 8443); - if (port > 0 && port != env.getConfigInt("port", 8090)) { + if (port > 0 && port != env.getLocalPort("port", 8090)) { env.setConfig("port.ssl", port); } prop.put("info_port.ssl", port); diff --git a/htroot/Settings_p.java b/htroot/Settings_p.java index 3b8eab40c..393efe729 100644 --- a/htroot/Settings_p.java +++ b/htroot/Settings_p.java @@ -73,7 +73,7 @@ public final class Settings_p { prop.put("settingsTables", ""); } - prop.put("port", env.getConfig("port", "8090")); + prop.put("port", env.getLocalPort("port", 8090)); prop.putHTML("peerName", sb.peers.mySeed().getName()); prop.putHTML("staticIP", env.getConfig("staticIP", "")); diff --git a/htroot/Table_API_p.java b/htroot/Table_API_p.java index f4f345a2a..9675a7c93 100644 --- a/htroot/Table_API_p.java +++ b/htroot/Table_API_p.java @@ -206,7 +206,7 @@ public class Table_API_p { } // now call the api URLs and store the result status - final Map l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); + final Map l = sb.tables.execAPICalls(Domains.LOCALHOST, sb.getLocalPort("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); // construct result table prop.put("showexec", l.isEmpty() ? 0 : 1); @@ -297,7 +297,7 @@ public class Table_API_p { } else { prop.put("showtable_list_" + count + "_isCrawlerStart", 0); } - prop.putHTML("showtable_list_" + count + "_inline_url", "http://" + sb.myPublicIP() + ":" + sb.getConfig("port", "8090") + UTF8.String(row.get(WorkTables.TABLE_API_COL_URL))); + prop.putHTML("showtable_list_" + count + "_inline_url", "http://" + sb.myPublicIP() + ":" + sb.getPublicPort("port", 8090) + UTF8.String(row.get(WorkTables.TABLE_API_COL_URL))); prop.put("showtable_list_" + count + "_scheduler_inline", inline ? "true" : "false"); prop.put("showtable_list_" + count + "_scheduler_filter", typefilter.pattern()); prop.put("showtable_list_" + count + "_scheduler_query", query.pattern()); diff --git a/htroot/api/push_p.java b/htroot/api/push_p.java index bcb0002f3..d5bf6f14f 100644 --- a/htroot/api/push_p.java +++ b/htroot/api/push_p.java @@ -122,7 +122,7 @@ public class push_p { sb.indexingDocumentProcessor.enQueue(in); } prop.put("mode_results_" + i + "_success", "1"); - prop.put("mode_results_" + i + "_success_message", "http://" + Domains.myPublicLocalIP().getHostAddress() + ":" + sb.getConfigInt("port", 8090) + "/solr/select?q=sku:%22" + u + "%22"); + prop.put("mode_results_" + i + "_success_message", "http://" + Domains.myPublicLocalIP().getHostAddress() + ":" + sb.getLocalPort("port", 8090) + "/solr/select?q=sku:%22" + u + "%22"); countsuccess++; } catch (MalformedURLException e) { e.printStackTrace(); diff --git a/htroot/opensearchdescription.java b/htroot/opensearchdescription.java index 8f576acd5..baa39ad2c 100644 --- a/htroot/opensearchdescription.java +++ b/htroot/opensearchdescription.java @@ -40,7 +40,7 @@ public class opensearchdescription { if (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", ""); String thisaddress = header.get("Host", Domains.LOCALHOST); - if (thisaddress.indexOf(':',0) == -1) thisaddress += ":" + env.getConfig("port", "8090"); + if (thisaddress.indexOf(':',0) == -1) thisaddress += ":" + env.getLocalPort("port", 8090); String thisprotocol = env.getConfigBool("server.https", false) ? "https" : "http"; final serverObjects prop = new serverObjects(); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index c856421ae..22b006143 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -146,7 +146,7 @@ public class yacysearch { // adding some additional properties needed for the rss feed String hostName = header.get("Host", Domains.LOCALHOST); if ( hostName.indexOf(':', 0) == -1 ) { - hostName += ":" + env.getConfig("port", "8090"); + hostName += ":" + env.getLocalPort("port", 8090); } prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html"); prop.put("rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.png"); @@ -880,7 +880,7 @@ public class yacysearch { // hostname and port (assume locahost if nothing helps) final String hostIP = sb.peers.mySeed().getIP(); prop.put("myhost", hostIP != null ? hostIP : Domains.LOCALHOST); - prop.put("myport", sb.getConfig("port", "8090")); + prop.put("myport", Domains.LOCALHOST.equals(hostIP) ? sb.getLocalPort("port", 8090) : sb.getPublicPort("port", 8090)); // return rewrite properties return prop; diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index 40a2893c8..a47a73237 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -91,7 +91,7 @@ public class yacysearch_location { if (query.length() > 0 && (metatag || search_title || search_publisher || search_creator || search_subject)) try { // get a queue of search results - final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getConfig("port", "8090") + "/yacysearch.rss"; + final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getLocalPort("port", 8090) + "/yacysearch.rss"; final BlockingQueue results = new LinkedBlockingQueue(); SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, lon == 0.0d && lat == 0.0d ? query : query + " /radius/" + lat + "/" + lon + "/" + radius, maximumTime, Integer.MAX_VALUE, null, false, ClientIdentification.yacyInternetCrawlerAgent); @@ -121,7 +121,7 @@ public class yacysearch_location { String promoteSearchPageGreeting = env.getConfig(SwitchboardConstants.GREETING, ""); if (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", ""); String hostName = header.get("Host", Domains.LOCALHOST); - if (hostName.indexOf(':',0) == -1) hostName += ":" + env.getConfig("port", "8090"); + if (hostName.indexOf(':',0) == -1) hostName += ":" + env.getLocalPort("port", "8090"); final String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); // SRU compliance final boolean global = post.get("kml_resource", "local").equals("global"); diff --git a/source/net/yacy/gui/Tray.java b/source/net/yacy/gui/Tray.java index 411dfbe95..423f627a6 100644 --- a/source/net/yacy/gui/Tray.java +++ b/source/net/yacy/gui/Tray.java @@ -239,8 +239,8 @@ public final class Tray { } private String readyMessage() { - if (deutsch) return "YaCy laeuft unter http://localhost:" + sb.getConfig("port", "8090"); - return "YaCy is running at http://localhost:" + sb.getConfig("port", "8090"); + if (deutsch) return "YaCy laeuft unter http://localhost:" + sb.getLocalPort("port", 8090); + return "YaCy is running at http://localhost:" + sb.getLocalPort("port", 8090); } private String shutdownMessage() { diff --git a/source/net/yacy/http/Jetty9HttpServerImpl.java b/source/net/yacy/http/Jetty9HttpServerImpl.java index 4199dd92d..f0c918765 100644 --- a/source/net/yacy/http/Jetty9HttpServerImpl.java +++ b/source/net/yacy/http/Jetty9HttpServerImpl.java @@ -82,7 +82,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { final SSLContext sslContext = initSslContext(sb); if (sslContext != null) { - int sslport = sb.getConfigInt("port.ssl", 8443); + int sslport = sb.getLocalPort("port.ssl", 8443); sslContextFactory.setSslContext(sslContext); // SSL HTTP Configuration @@ -286,8 +286,8 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { } try { // reconnect with new settings (instead to stop/start server, just manipulate connectors final Connector[] cons = server.getConnectors(); - final int port = Switchboard.getSwitchboard().getConfigInt("port", 8090); - final int sslport = Switchboard.getSwitchboard().getConfigInt("port.ssl", 8443); + final int port = Switchboard.getSwitchboard().getLocalPort("port", 8090); + final int sslport = Switchboard.getSwitchboard().getLocalPort("port.ssl", 8443); for (Connector con : cons) { // check http connector if (con.getName().startsWith("httpd") && ((ServerConnector)con).getPort() != port) { diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index 250d361d4..036dde2fb 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -1162,9 +1162,9 @@ public class Seed implements Cloneable, Comparable, Comparator final Seed newSeed = new Seed(hashs); // now calculate other information about the host - final long port = Switchboard.getSwitchboard().getConfigLong("port", 8090); //get port from config + final int port = Switchboard.getSwitchboard().getPublicPort("port", 8090); //get port from config newSeed.dna.put(Seed.NAME, defaultPeerName() ); - newSeed.dna.put(Seed.PORT, Long.toString(port)); + newSeed.dna.put(Seed.PORT, Integer.toString(port)); return newSeed; } diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 3773ac6f4..2852dadad 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -958,7 +958,7 @@ public final class SeedDB implements AlternativeDomainNames { if (this.mySeed == null) initMySeed(); if ((seed == this.mySeed) && (!(seed.isOnline()))) { // take local ip instead of external - return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getConfig("port", "8090") + ((subdom == null) ? "" : ("/" + subdom)); + return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getLocalPort("port", 8090) + ((subdom == null) ? "" : ("/" + subdom)); } return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom)); } else { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index c809727c6..8c24242e3 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -294,7 +294,7 @@ public final class Switchboard extends serverSwitch { super(dataPath, appPath, initPath, configPath); sb = this; // check if port is already occupied - final int port = getConfigInt("port", 8090); + final int port = getLocalPort("port", 8090); if (TimeoutRequest.ping(Domains.LOCALHOST, port, 500)) { throw new RuntimeException( "a server is already running on the YaCy port " @@ -2453,7 +2453,7 @@ public final class Switchboard extends serverSwitch { startupAction = false; // execute api calls - final Map callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); + final Map callResult = this.tables.execAPICalls("localhost", getLocalPort("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); for ( final Map.Entry call : callResult.entrySet() ) { this.log.info("Scheduler executed api call, response " + call.getValue() + ": " + call.getKey()); } @@ -3675,7 +3675,7 @@ public final class Switchboard extends serverSwitch { private static long indeSizeCache = 0; private static long indexSizeTime = 0; public void updateMySeed() { - this.peers.mySeed().put(Seed.PORT, getConfig("port", "8090")); + this.peers.mySeed().put(Seed.PORT, Integer.toString(getPublicPort("port", 8090))); //the speed of indexing (pages/minute) of the peer final long uptime = (System.currentTimeMillis() - this.startupTime) / 1000; diff --git a/source/net/yacy/server/http/HTTPDemon.java b/source/net/yacy/server/http/HTTPDemon.java index 31da170e4..5891bd063 100644 --- a/source/net/yacy/server/http/HTTPDemon.java +++ b/source/net/yacy/server/http/HTTPDemon.java @@ -162,13 +162,13 @@ public final class HTTPDemon { final InetAddress hostAddress = Domains.dnsResolve(clientIP); if (hostAddress == null) { tp.put("host", Domains.myPublicLocalIP().getHostAddress()); - tp.put("port", switchboard.getConfig("port", "8090")); + tp.put("port", switchboard.getLocalPort("port", 8090)); } else if (hostAddress.isSiteLocalAddress() || hostAddress.isLoopbackAddress()) { tp.put("host", Domains.myPublicLocalIP().getHostAddress()); - tp.put("port", switchboard.getConfig("port", "8090")); + tp.put("port", switchboard.getLocalPort("port", 8090)); } else { tp.put("host", switchboard.myPublicIP()); - tp.put("port", switchboard.getConfig("port", "8090")); + tp.put("port", switchboard.getPublicPort("port", 8090)); } tp.put("peerName", (switchboard.peers == null) ? "" : switchboard.peers.myName()); diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index ea81d3892..272309077 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -56,529 +56,646 @@ import net.yacy.kelondro.workflow.WorkflowThread; import net.yacy.peers.Seed; import net.yacy.search.SwitchboardConstants; -public class serverSwitch -{ - - // configuration management - private final File configFile; - private final String configComment; - public final File dataPath; - public final File appPath; - protected boolean firstInit; - public ConcurrentLog log; - protected int serverJobs; - private ConcurrentMap configProps; - private final ConcurrentMap configRemoved; - private final NavigableMap workerThreads; - private YaCyHttpServer httpserver; // implemented HttpServer - - public serverSwitch( - final File dataPath, - final File appPath, - final String initPath, - final String configPath) { - // we initialize the switchboard with a property file, - // but maintain these properties then later in a new 'config' file - // to reset all changed configs, the config file must - // be deleted, but not the init file - // the only attribute that will always be read from the init is the - // file name of the config file - this.dataPath = dataPath; - this.appPath = appPath; - this.configComment = "This is an automatically generated file, updated by serverAbstractSwitch and initialized by " + initPath; - final File initFile = new File(appPath, initPath); - this.configFile = new File(dataPath, configPath); // propertiesFile(config); - this.firstInit = !this.configFile.exists(); // this is true if the application was started for the first time - new File(this.configFile.getParent()).mkdir(); - - // predefine init's - final ConcurrentMap initProps; - if ( initFile.exists() ) { - initProps = FileUtils.loadMap(initFile); - } else { - initProps = new ConcurrentHashMap(); - } - - // load config's from last save - if ( this.configFile.exists() ) { - this.configProps = FileUtils.loadMap(this.configFile); - } else { - this.configProps = new ConcurrentHashMap(); - } - - // remove all values from config that do not appear in init - this.configRemoved = new ConcurrentHashMap(); - synchronized ( this.configProps ) { - Iterator i = this.configProps.keySet().iterator(); - String key; - while ( i.hasNext() ) { - key = i.next(); - if ( !(initProps.containsKey(key)) ) { - this.configRemoved.put(key, this.configProps.get(key)); - i.remove(); - } - } - - // merge new props from init to config - // this is necessary for migration, when new properties are attached - initProps.putAll(this.configProps); - this.configProps = initProps; - - // save result; this may initially create a config file after initialization - saveConfig(); - } - - // init thread control - this.workerThreads = new TreeMap(); - - // init busy state control - //this.serverJobs = 0; - - // init server tracking - serverAccessTracker.init( - getConfigLong("server.maxTrackingTime", 60 * 60 * 1000), - (int) getConfigLong("server.maxTrackingCount", 1000), - (int) getConfigLong("server.maxTrackingHostCount", 100)); - } - - /** - * get my public IP, either set statically or figure out dynamic - * This method is deprecated because there may be more than one public IPs of this peer, - * i.e. one IPv4 and one IPv6. Please use myPublicIPs() instead - * @return the public IP of this peer, if known - */ - @Deprecated - public String myPublicIP() { - // if a static IP was configured, we have to return it here ... - final String staticIP = getConfig("staticIP", ""); - if ( staticIP.length() > 0 ) return staticIP; - - // otherwise we return the real IP address of this host - final InetAddress pLIP = Domains.myPublicLocalIP(); - if ( pLIP != null ) return pLIP.getHostAddress(); - return null; - } - - /** - * Get all my public IPs. If there was a static IP assignment, only one, that IP is returned. - * @return a set of IPs which are supposed to be my own public IPs - */ - public Set myPublicIPs() { - // if a static IP was configured, we have to return it here ... - final String staticIP = getConfig("staticIP", ""); - if ( staticIP.length() > 0 ) { - HashSet h = new HashSet<>(); - h.add(staticIP); - return h; - } - - Set h = new LinkedHashSet<>(); - for (InetAddress i: Domains.myPublicIPv6()) { - String s = i.getHostAddress(); - if (Seed.isProperIP(s)) h.add(s); - } - for (InetAddress i: Domains.myPublicIPv4()) { - String s = i.getHostAddress(); - if (Seed.isProperIP(s)) h.add(s); - } - return h; - } - - // a logger for this switchboard - public void setLog(final ConcurrentLog log) { - this.log = log; - } - - public ConcurrentLog getLog() { - return this.log; - } - - /** - * add whole map of key-value pairs to config - * @param otherConfigs - */ - public void setConfig(final Map otherConfigs) { - final Iterator> i = otherConfigs.entrySet().iterator(); - Map.Entry entry; - while ( i.hasNext() ) { - entry = i.next(); - setConfig(entry.getKey(), entry.getValue()); - } - } - - public void setConfig(final String key, final boolean value) { - setConfig(key, (value) ? "true" : "false"); - } - - public void setConfig(final String key, final long value) { - setConfig(key, Long.toString(value)); - } - - public void setConfig(final String key, final float value) { - setConfig(key, Float.toString(value)); - } - - public void setConfig(final String key, final double value) { - setConfig(key, Double.toString(value)); - } - - public void setConfig(final String key, final String value) { - // set the value - final String oldValue = this.configProps.put(key, value); - if ( oldValue == null || !value.equals(oldValue) ) { - saveConfig(); - } - } - - public void removeConfig(final String key) { - this.configProps.remove(key); - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key name of the configuration parameter - * @param dflt default value which will be used in case parameter can not be found or if it is invalid - * @return value if the parameter or default value - */ - public String getConfig(final String key, final String dflt) { - // get the value - final String s = this.configProps.get(key); - - // return value - if ( s == null ) { - return dflt; - } - return s; - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key name of the configuration parameter - * @param dflt default value which will be used in case parameter can not be found or if it is invalid - * @return value if the parameter or default value - */ - public long getConfigLong(final String key, final long dflt) { - try { - return Long.parseLong(getConfig(key, Long.toString(dflt))); - } catch (final NumberFormatException e ) { - return dflt; - } - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key name of the configuration parameter - * @param dflt default value which will be used in case parameter can not be found or if it is invalid - * @return value if the parameter or default value - */ - public float getConfigFloat(final String key, final float dflt) { - try { - return Float.parseFloat(getConfig(key, Float.toString(dflt))); - } catch (final NumberFormatException e ) { - return dflt; - } - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key name of the configuration parameter - * @param dflt default value which will be used in case parameter can not be found or if it is invalid - * @return value if the parameter or default value - */ - public int getConfigInt(final String key, final int dflt) { - try { - return Integer.parseInt(getConfig(key, Integer.toString(dflt))); - } catch (final NumberFormatException e ) { - return dflt; - } - } - - /** - * Gets a configuration parameter from the properties. - * - * @param key name of the configuration parameter - * @param dflt default value which will be used in case parameter can not be found or if it is invalid - * @return value if the parameter or default value - */ - public boolean getConfigBool(final String key, final boolean dflt) { - return Boolean.parseBoolean(getConfig(key, Boolean.toString(dflt))); - } - - /** - * Create a File instance for a configuration setting specifying a path. - * - * @param key config key - * @param dflt default path value, that is used when there is no value key in the - * configuration. - * @return if the value of the setting is an absolute path String, then the returned File is derived from - * this setting only. Otherwise the path's file is constructed from the applications root path + - * the relative path setting. - */ - public File getDataPath(final String key, final String dflt) { - return getFileByPath(key, dflt, this.dataPath); - } - - /** - * return file at path from config entry "key", or fallback to default dflt - * @param key - * @param dflt - * @return - */ - public File getAppPath(final String key, final String dflt) { - return getFileByPath(key, dflt, this.appPath); - } - - private File getFileByPath(String key, String dflt, File prefix) { - final String path = getConfig(key, dflt).replace('\\', '/'); - final File f = new File(path); - return (f.isAbsolute() ? new File(f.getAbsolutePath()) : new File(prefix, path)); - } - - public Iterator configKeys() { - return this.configProps.keySet().iterator(); - } - - /** - * write the changes to permanent storage (File) - */ - private void saveConfig() { - ConcurrentMap configPropsCopy = new ConcurrentHashMap(); - configPropsCopy.putAll(this.configProps); // avoid concurrency problems - FileUtils.saveMap(this.configFile, configPropsCopy, this.configComment); - } - - /** - * Gets configuration parameters which have been removed during initialization. - * - * @return contains parameter name as key and parameter value as value - */ - public ConcurrentMap getRemoved() { - return this.configRemoved; - } - - public void deployThread( - final String threadName, - final String threadShortDescription, - final String threadLongDescription, - final String threadMonitorURL, - final BusyThread newThread, - final long startupDelay) { - deployThread( - threadName, - threadShortDescription, - threadLongDescription, - threadMonitorURL, - newThread, - startupDelay, - Long.parseLong(getConfig(threadName + "_idlesleep", "100")), - Long.parseLong(getConfig(threadName + "_busysleep", "1000")), - Long.parseLong(getConfig(threadName + "_memprereq", "1000000")), - Double.parseDouble(getConfig(threadName + "_loadprereq", "9.0"))); - } - - public void deployThread( - final String threadName, - final String threadShortDescription, - final String threadLongDescription, - final String threadMonitorURL, - final BusyThread newThread, - final long startupDelay, - final long initialIdleSleep, - final long initialBusySleep, - final long initialMemoryPreRequisite, - final double initialLoadPreRequisite) { - if ( newThread.isAlive() ) { - throw new RuntimeException( - "undeployed threads must not live; they are started as part of the deployment"); - } - newThread.setStartupSleep(startupDelay); - long x; - try { - x = Long.parseLong(getConfig(threadName + "_idlesleep", "novalue")); - newThread.setIdleSleep(x); - } catch (final NumberFormatException e ) { - newThread.setIdleSleep(initialIdleSleep); - setConfig(threadName + "_idlesleep", initialIdleSleep); - } - try { - x = Long.parseLong(getConfig(threadName + "_busysleep", "novalue")); - newThread.setBusySleep(x); - } catch (final NumberFormatException e ) { - newThread.setBusySleep(initialBusySleep); - setConfig(threadName + "_busysleep", initialBusySleep); - } - try { - x = Long.parseLong(getConfig(threadName + "_memprereq", "novalue")); - newThread.setMemPreReqisite(x); - } catch (final NumberFormatException e ) { - newThread.setMemPreReqisite(initialMemoryPreRequisite); - setConfig(threadName + "_memprereq", initialMemoryPreRequisite); - } - try { - final double load = Double.parseDouble(getConfig(threadName + "_loadprereq", "novalue")); - newThread.setLoadPreReqisite(load); - } catch (final NumberFormatException e ) { - newThread.setLoadPreReqisite(initialLoadPreRequisite); - setConfig(threadName + "_loadprereq", (float)initialLoadPreRequisite); - } - newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL); - this.workerThreads.put(threadName, newThread); - // start the thread - if ( this.workerThreads.containsKey(threadName) ) { - newThread.start(); - } - } - - public BusyThread getThread(final String threadName) { - return this.workerThreads.get(threadName); - } - - public void setThreadPerformance( - final String threadName, - final long idleMillis, - final long busyMillis, - final long memprereqBytes, - final double loadprereq) { - final BusyThread thread = this.workerThreads.get(threadName); - if ( thread != null ) { - setConfig(threadName + "_idlesleep", thread.setIdleSleep(idleMillis)); - setConfig(threadName + "_busysleep", thread.setBusySleep(busyMillis)); - setConfig(threadName + "_memprereq", memprereqBytes); - thread.setMemPreReqisite(memprereqBytes); - setConfig(threadName + "_loadprereq", (float)loadprereq); - thread.setLoadPreReqisite(loadprereq); - } - } - - public synchronized void terminateThread(final String threadName, final boolean waitFor) { - if ( this.workerThreads.containsKey(threadName) ) { - ((WorkflowThread) this.workerThreads.get(threadName)).terminate(waitFor); - this.workerThreads.remove(threadName); - } - } - - public void intermissionAllThreads(final long pause) { - final Iterator e = this.workerThreads.keySet().iterator(); - while ( e.hasNext() ) { - this.workerThreads.get(e.next()).intermission(pause); - } - } - - public synchronized void terminateAllThreads(final boolean waitFor) { - Iterator e = this.workerThreads.keySet().iterator(); - while ( e.hasNext() ) { - ((WorkflowThread) this.workerThreads.get(e.next())).terminate(false); - } - if ( waitFor ) { - e = this.workerThreads.keySet().iterator(); - while ( e.hasNext() ) { - ((WorkflowThread) this.workerThreads.get(e.next())).terminate(true); - e.remove(); - } - } - } - - public Iterator /*of serverThread-Names (String)*/threadNames() { - return this.workerThreads.keySet().iterator(); - } - - public File getDataPath() { - return this.dataPath; - } - - public File getAppPath() { - return this.appPath; - } - - @Override - public String toString() { - return this.configProps.toString(); - } - - public void handleBusyState(final int jobs) { - this.serverJobs = jobs; - } - - /** - * Retrieve text data (e. g. config file) from file file may be an url or a filename with path relative to - * rootPath parameter - * - * @param file url or filename - * @param rootPath searchpath for file - * @param file file to use when remote fetching fails (null if unused) - */ - public Reader getConfigFileFromWebOrLocally(final String uri, final String rootPath, final File file) - throws IOException, - FileNotFoundException { - if ( uri.startsWith("http://") || uri.startsWith("https://") ) { - final String[] uris = uri.split(","); - for ( String netdef : uris ) { - netdef = netdef.trim(); - try { - final RequestHeader reqHeader = new RequestHeader(); - reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent); - final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); - client.setHeader(reqHeader.entrySet()); - byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), false); - if ( data == null || data.length == 0 ) { - continue; - } - // save locally in case next fetch fails - if ( file != null ) { - FileOutputStream f = new FileOutputStream(file); - f.write(data); - f.close(); - } - return new InputStreamReader(new BufferedInputStream(new ByteArrayInputStream(data))); - } catch (final Exception e ) { - continue; - } - } - if ( file != null && file.exists() ) { - return new FileReader(file); - } - throw new FileNotFoundException(); - } - final File f = (uri.length() > 0 && uri.startsWith("/")) ? new File(uri) : new File(rootPath, uri); - if (f.exists()) return new FileReader(f); - throw new FileNotFoundException(f.toString()); - } - - private static Random pwGenerator = new Random(); - - /** - * Generates a random password. - * - * @return random password which is 20 characters long. - */ - public String genRandomPassword() { - return genRandomPassword(20); - } - - /** - * Generates a random password of a given length. - * - * @param length length o password - * @return password of given length - */ - public String genRandomPassword(final int length) { - byte[] bytes = new byte[length]; - pwGenerator.nextBytes(bytes); - return Digest.encodeMD5Hex(bytes); - } - /** - * set/remember jetty server - * @param jettyserver - */ - public void setHttpServer(YaCyHttpServer jettyserver) { - this.httpserver = jettyserver; - } - public YaCyHttpServer getHttpServer() { - return httpserver; - } +public class serverSwitch { + + // configuration management + private final File configFile; + private final String configComment; + public final File dataPath; + public final File appPath; + protected boolean firstInit; + public ConcurrentLog log; + protected int serverJobs; + private ConcurrentMap configProps; + private final ConcurrentMap configRemoved; + private final NavigableMap workerThreads; + private YaCyHttpServer httpserver; // implemented HttpServer + private ConcurrentMap upnpPortMap = new ConcurrentHashMap<>(); + private boolean isConnectedViaUpnp; + + public serverSwitch(final File dataPath, final File appPath, + final String initPath, final String configPath) { + // we initialize the switchboard with a property file, + // but maintain these properties then later in a new 'config' file + // to reset all changed configs, the config file must + // be deleted, but not the init file + // the only attribute that will always be read from the init is the + // file name of the config file + this.dataPath = dataPath; + this.appPath = appPath; + this.configComment = "This is an automatically generated file, updated by serverAbstractSwitch and initialized by " + + initPath; + final File initFile = new File(appPath, initPath); + this.configFile = new File(dataPath, configPath); // propertiesFile(config); + this.firstInit = !this.configFile.exists(); // this is true if the + // application was started + // for the first time + new File(this.configFile.getParent()).mkdir(); + + // predefine init's + final ConcurrentMap initProps; + if (initFile.exists()) { + initProps = FileUtils.loadMap(initFile); + } else { + initProps = new ConcurrentHashMap(); + } + + // load config's from last save + if (this.configFile.exists()) { + this.configProps = FileUtils.loadMap(this.configFile); + } else { + this.configProps = new ConcurrentHashMap(); + } + + // remove all values from config that do not appear in init + this.configRemoved = new ConcurrentHashMap(); + synchronized (this.configProps) { + Iterator i = this.configProps.keySet().iterator(); + String key; + while (i.hasNext()) { + key = i.next(); + if (!(initProps.containsKey(key))) { + this.configRemoved.put(key, this.configProps.get(key)); + i.remove(); + } + } + + // merge new props from init to config + // this is necessary for migration, when new properties are attached + initProps.putAll(this.configProps); + this.configProps = initProps; + + // save result; this may initially create a config file after + // initialization + saveConfig(); + } + + // init thread control + this.workerThreads = new TreeMap(); + + // init busy state control + // this.serverJobs = 0; + + // init server tracking + serverAccessTracker.init( + getConfigLong("server.maxTrackingTime", 60 * 60 * 1000), + (int) getConfigLong("server.maxTrackingCount", 1000), + (int) getConfigLong("server.maxTrackingHostCount", 100)); + } + + /** + * get my public IP, either set statically or figure out dynamic This method + * is deprecated because there may be more than one public IPs of this peer, + * i.e. one IPv4 and one IPv6. Please use myPublicIPs() instead + * + * @return the public IP of this peer, if known + */ + @Deprecated + public String myPublicIP() { + // if a static IP was configured, we have to return it here ... + final String staticIP = getConfig("staticIP", ""); + if (staticIP.length() > 0) + return staticIP; + + // otherwise we return the real IP address of this host + final InetAddress pLIP = Domains.myPublicLocalIP(); + if (pLIP != null) + return pLIP.getHostAddress(); + return null; + } + + /** + * Get all my public IPs. If there was a static IP assignment, only one, + * that IP is returned. + * + * @return a set of IPs which are supposed to be my own public IPs + */ + public Set myPublicIPs() { + // if a static IP was configured, we have to return it here ... + final String staticIP = getConfig("staticIP", ""); + if (staticIP.length() > 0) { + HashSet h = new HashSet<>(); + h.add(staticIP); + return h; + } + + Set h = new LinkedHashSet<>(); + for (InetAddress i : Domains.myPublicIPv6()) { + String s = i.getHostAddress(); + if (Seed.isProperIP(s)) + h.add(s); + } + for (InetAddress i : Domains.myPublicIPv4()) { + String s = i.getHostAddress(); + if (Seed.isProperIP(s)) + h.add(s); + } + return h; + } + + /** + * Gets public port. May differ from local port due to NATting. This method + * will eventually removed once nobody used IPv4 anymore, but until then we + * have to live with it. + * + * @param key + * original key from config (for example "port" or "port.ssl") + * @param dflt + * default value which will be used if no value is found + * @return the public port of this system on its IPv4 address + * + * @see #getLocalPort(String, int) + */ + public int getPublicPort(final String key, final int dflt) { + + if (isConnectedViaUpnp && upnpPortMap.containsKey(key)) { + return upnpPortMap.get(key).intValue(); + } + + // TODO: add way of setting and retrieving port for manual NAT + + return getConfigInt(key, dflt); + } + + /** + * Wrapper for {@link #getConfigInt(String, int)} to have a more consistent + * API. + * + * @param key + * original key from config (for example "port" or "port.ssl") + * @param dflt + * default value which will be used if no value is found + * @return the local port of this system + * @see #getPublicPort(String, int) + */ + public int getLocalPort(final String key, final int dflt) { + + return getConfigInt(key, dflt); + } + + // a logger for this switchboard + public void setLog(final ConcurrentLog log) { + this.log = log; + } + + public ConcurrentLog getLog() { + return this.log; + } + + /** + * add whole map of key-value pairs to config + * + * @param otherConfigs + */ + public void setConfig(final Map otherConfigs) { + final Iterator> i = otherConfigs.entrySet() + .iterator(); + Map.Entry entry; + while (i.hasNext()) { + entry = i.next(); + setConfig(entry.getKey(), entry.getValue()); + } + } + + public void setConfig(final String key, final boolean value) { + setConfig(key, (value) ? "true" : "false"); + } + + public void setConfig(final String key, final long value) { + setConfig(key, Long.toString(value)); + } + + public void setConfig(final String key, final float value) { + setConfig(key, Float.toString(value)); + } + + public void setConfig(final String key, final double value) { + setConfig(key, Double.toString(value)); + } + + public void setConfig(final String key, final String value) { + // set the value + final String oldValue = this.configProps.put(key, value); + if (oldValue == null || !value.equals(oldValue)) { + saveConfig(); + } + } + + public void removeConfig(final String key) { + this.configProps.remove(key); + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public String getConfig(final String key, final String dflt) { + // get the value + final String s = this.configProps.get(key); + + // return value + if (s == null) { + return dflt; + } + return s; + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public long getConfigLong(final String key, final long dflt) { + try { + return Long.parseLong(getConfig(key, Long.toString(dflt))); + } catch (final NumberFormatException e) { + return dflt; + } + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public float getConfigFloat(final String key, final float dflt) { + try { + return Float.parseFloat(getConfig(key, Float.toString(dflt))); + } catch (final NumberFormatException e) { + return dflt; + } + } + + public boolean isConnectedViaUpnp() { + + return isConnectedViaUpnp; + } + + public void setConnectedViaUpnp(final boolean isConnectedViaUpnp) { + + this.isConnectedViaUpnp = isConnectedViaUpnp; + + if (!isConnectedViaUpnp) { + upnpPortMap.clear(); + } + } + + public void setUpnpPorts(final String key, final int port) { + + upnpPortMap.put(key, Integer.valueOf(port)); + } + + public void removeUpnpPort(final String key) { + upnpPortMap.remove(key); + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public int getConfigInt(final String key, final int dflt) { + try { + + return Integer.parseInt(getConfig(key, Integer.toString(dflt))); + + } catch (final NumberFormatException e) { + return dflt; + } + } + + /** + * Gets a configuration parameter from the properties. + * + * @param key + * name of the configuration parameter + * @param dflt + * default value which will be used in case parameter can not be + * found or if it is invalid + * @return value if the parameter or default value + */ + public boolean getConfigBool(final String key, final boolean dflt) { + return Boolean.parseBoolean(getConfig(key, Boolean.toString(dflt))); + } + + /** + * Create a File instance for a configuration setting specifying a path. + * + * @param key + * config key + * @param dflt + * default path value, that is used when there is no value + * key in the configuration. + * @return if the value of the setting is an absolute path String, then the + * returned File is derived from this setting only. Otherwise the + * path's file is constructed from the applications root path + the + * relative path setting. + */ + public File getDataPath(final String key, final String dflt) { + return getFileByPath(key, dflt, this.dataPath); + } + + /** + * return file at path from config entry "key", or fallback to default dflt + * + * @param key + * @param dflt + * @return + */ + public File getAppPath(final String key, final String dflt) { + return getFileByPath(key, dflt, this.appPath); + } + + private File getFileByPath(String key, String dflt, File prefix) { + final String path = getConfig(key, dflt).replace('\\', '/'); + final File f = new File(path); + return (f.isAbsolute() ? new File(f.getAbsolutePath()) : new File( + prefix, path)); + } + + public Iterator configKeys() { + return this.configProps.keySet().iterator(); + } + + /** + * write the changes to permanent storage (File) + */ + private void saveConfig() { + ConcurrentMap configPropsCopy = new ConcurrentHashMap(); + configPropsCopy.putAll(this.configProps); // avoid concurrency problems + FileUtils.saveMap(this.configFile, configPropsCopy, this.configComment); + } + + /** + * Gets configuration parameters which have been removed during + * initialization. + * + * @return contains parameter name as key and parameter value as value + */ + public ConcurrentMap getRemoved() { + return this.configRemoved; + } + + public void deployThread(final String threadName, + final String threadShortDescription, + final String threadLongDescription, final String threadMonitorURL, + final BusyThread newThread, final long startupDelay) { + deployThread( + threadName, + threadShortDescription, + threadLongDescription, + threadMonitorURL, + newThread, + startupDelay, + Long.parseLong(getConfig(threadName + "_idlesleep", "100")), + Long.parseLong(getConfig(threadName + "_busysleep", "1000")), + Long.parseLong(getConfig(threadName + "_memprereq", "1000000")), + Double.parseDouble(getConfig(threadName + "_loadprereq", "9.0"))); + } + + public void deployThread(final String threadName, + final String threadShortDescription, + final String threadLongDescription, final String threadMonitorURL, + final BusyThread newThread, final long startupDelay, + final long initialIdleSleep, final long initialBusySleep, + final long initialMemoryPreRequisite, + final double initialLoadPreRequisite) { + if (newThread.isAlive()) { + throw new RuntimeException( + "undeployed threads must not live; they are started as part of the deployment"); + } + newThread.setStartupSleep(startupDelay); + long x; + try { + x = Long.parseLong(getConfig(threadName + "_idlesleep", "novalue")); + newThread.setIdleSleep(x); + } catch (final NumberFormatException e) { + newThread.setIdleSleep(initialIdleSleep); + setConfig(threadName + "_idlesleep", initialIdleSleep); + } + try { + x = Long.parseLong(getConfig(threadName + "_busysleep", "novalue")); + newThread.setBusySleep(x); + } catch (final NumberFormatException e) { + newThread.setBusySleep(initialBusySleep); + setConfig(threadName + "_busysleep", initialBusySleep); + } + try { + x = Long.parseLong(getConfig(threadName + "_memprereq", "novalue")); + newThread.setMemPreReqisite(x); + } catch (final NumberFormatException e) { + newThread.setMemPreReqisite(initialMemoryPreRequisite); + setConfig(threadName + "_memprereq", initialMemoryPreRequisite); + } + try { + final double load = Double.parseDouble(getConfig(threadName + + "_loadprereq", "novalue")); + newThread.setLoadPreReqisite(load); + } catch (final NumberFormatException e) { + newThread.setLoadPreReqisite(initialLoadPreRequisite); + setConfig(threadName + "_loadprereq", + (float) initialLoadPreRequisite); + } + newThread.setDescription(threadShortDescription, threadLongDescription, + threadMonitorURL); + this.workerThreads.put(threadName, newThread); + // start the thread + if (this.workerThreads.containsKey(threadName)) { + newThread.start(); + } + } + + public BusyThread getThread(final String threadName) { + return this.workerThreads.get(threadName); + } + + public void setThreadPerformance(final String threadName, + final long idleMillis, final long busyMillis, + final long memprereqBytes, final double loadprereq) { + final BusyThread thread = this.workerThreads.get(threadName); + if (thread != null) { + setConfig(threadName + "_idlesleep", + thread.setIdleSleep(idleMillis)); + setConfig(threadName + "_busysleep", + thread.setBusySleep(busyMillis)); + setConfig(threadName + "_memprereq", memprereqBytes); + thread.setMemPreReqisite(memprereqBytes); + setConfig(threadName + "_loadprereq", (float) loadprereq); + thread.setLoadPreReqisite(loadprereq); + } + } + + public synchronized void terminateThread(final String threadName, + final boolean waitFor) { + if (this.workerThreads.containsKey(threadName)) { + ((WorkflowThread) this.workerThreads.get(threadName)) + .terminate(waitFor); + this.workerThreads.remove(threadName); + } + } + + public void intermissionAllThreads(final long pause) { + final Iterator e = this.workerThreads.keySet().iterator(); + while (e.hasNext()) { + this.workerThreads.get(e.next()).intermission(pause); + } + } + + public synchronized void terminateAllThreads(final boolean waitFor) { + Iterator e = this.workerThreads.keySet().iterator(); + while (e.hasNext()) { + ((WorkflowThread) this.workerThreads.get(e.next())) + .terminate(false); + } + if (waitFor) { + e = this.workerThreads.keySet().iterator(); + while (e.hasNext()) { + ((WorkflowThread) this.workerThreads.get(e.next())) + .terminate(true); + e.remove(); + } + } + } + + public Iterator /* of serverThread-Names (String) */threadNames() { + return this.workerThreads.keySet().iterator(); + } + + public File getDataPath() { + return this.dataPath; + } + + public File getAppPath() { + return this.appPath; + } + + @Override + public String toString() { + return this.configProps.toString(); + } + + public void handleBusyState(final int jobs) { + this.serverJobs = jobs; + } + + /** + * Retrieve text data (e. g. config file) from file file may be an url or a + * filename with path relative to rootPath parameter + * + * @param file + * url or filename + * @param rootPath + * searchpath for file + * @param file + * file to use when remote fetching fails (null if unused) + */ + public Reader getConfigFileFromWebOrLocally(final String uri, + final String rootPath, final File file) throws IOException, + FileNotFoundException { + if (uri.startsWith("http://") || uri.startsWith("https://")) { + final String[] uris = uri.split(","); + for (String netdef : uris) { + netdef = netdef.trim(); + try { + final RequestHeader reqHeader = new RequestHeader(); + reqHeader + .put(HeaderFramework.USER_AGENT, + ClientIdentification.yacyInternetCrawlerAgent.userAgent); + final HTTPClient client = new HTTPClient( + ClientIdentification.yacyInternetCrawlerAgent); + client.setHeader(reqHeader.entrySet()); + byte[] data = client + .GETbytes( + uri, + getConfig( + SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, + "admin"), + getConfig( + SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, + ""), false); + if (data == null || data.length == 0) { + continue; + } + // save locally in case next fetch fails + if (file != null) { + FileOutputStream f = new FileOutputStream(file); + f.write(data); + f.close(); + } + return new InputStreamReader(new BufferedInputStream( + new ByteArrayInputStream(data))); + } catch (final Exception e) { + continue; + } + } + if (file != null && file.exists()) { + return new FileReader(file); + } + throw new FileNotFoundException(); + } + final File f = (uri.length() > 0 && uri.startsWith("/")) ? new File(uri) + : new File(rootPath, uri); + if (f.exists()) + return new FileReader(f); + throw new FileNotFoundException(f.toString()); + } + + private static Random pwGenerator = new Random(); + + /** + * Generates a random password. + * + * @return random password which is 20 characters long. + */ + public String genRandomPassword() { + return genRandomPassword(20); + } + + /** + * Generates a random password of a given length. + * + * @param length + * length o password + * @return password of given length + */ + public String genRandomPassword(final int length) { + byte[] bytes = new byte[length]; + pwGenerator.nextBytes(bytes); + return Digest.encodeMD5Hex(bytes); + } + + /** + * set/remember jetty server + * + * @param jettyserver + */ + public void setHttpServer(YaCyHttpServer jettyserver) { + this.httpserver = jettyserver; + } + + public YaCyHttpServer getHttpServer() { + return httpserver; + } } diff --git a/source/net/yacy/utils/upnp/UPnP.java b/source/net/yacy/utils/upnp/UPnP.java index dcd9574f8..cadf4a9c6 100644 --- a/source/net/yacy/utils/upnp/UPnP.java +++ b/source/net/yacy/utils/upnp/UPnP.java @@ -35,6 +35,7 @@ import net.yacy.search.Switchboard; import org.bitlet.weupnp.GatewayDevice; import org.bitlet.weupnp.GatewayDiscover; +import org.bitlet.weupnp.PortMappingEntry; import org.xml.sax.SAXException; public class UPnP { @@ -42,7 +43,7 @@ public class UPnP { private static final ConcurrentLog LOG = new ConcurrentLog("UPNP"); private static final Switchboard SB = Switchboard.getSwitchboard(); - private static Map GATEWAY_DEVICES = null; + private static GatewayDevice gatewayDevice; private static final Map MAPPINGS = new EnumMap<>( UPnPMappingType.class); @@ -53,20 +54,23 @@ public class UPnP { "server.https", "TCP", "YaCy HTTPS")); } + private static final int MIN_CANDIDATE_PORT = 49152; + private static final int MAX_CANDIDATE_PORT = 65535; + private static boolean init() { boolean init = true; try { - if (GATEWAY_DEVICES == null) { - GATEWAY_DEVICES = new GatewayDiscover().discover(); + if (gatewayDevice == null || !gatewayDevice.isConnected()) { + final GatewayDiscover discover = new GatewayDiscover(); + discover.discover(); + gatewayDevice = discover.getValidGateway(); } } catch (IOException | SAXException | ParserConfigurationException e) { init = false; } - if (GATEWAY_DEVICES != null) { - for (final GatewayDevice gatewayDevice : GATEWAY_DEVICES.values()) { - LOG.info("found device: " + gatewayDevice.getFriendlyName()); - } + if (gatewayDevice != null) { + LOG.info("found device: " + gatewayDevice.getFriendlyName()); } else { LOG.info("no device found"); init = false; @@ -91,6 +95,8 @@ public class UPnP { addPortMapping(entry.getKey(), mapping, SB.getConfigInt(mapping.getConfigPortKey(), 0)); } + + SB.setConnectedViaUpnp(true); } /** @@ -102,6 +108,8 @@ public class UPnP { return; } + SB.setConnectedViaUpnp(false); + UPnPMapping mapping; for (final Entry entry : MAPPINGS .entrySet()) { @@ -123,7 +131,7 @@ public class UPnP { * @param port * port number to map */ - public static void addPortMapping(final UPnPMappingType type, + private static void addPortMapping(final UPnPMappingType type, final UPnPMapping mapping, final int port) { if (port < 1) { @@ -137,32 +145,49 @@ public class UPnP { if ((mapping.isConfigEnabledKeyEmpty() || SB.getConfigBool( mapping.getConfigEnabledKey(), false)) && mapping.getPort() == 0 - && ((GATEWAY_DEVICES != null) || init())) { + && ((gatewayDevice != null) || init())) { String localHostIP; boolean mapped; String msg; - for (final GatewayDevice gatewayDevice : GATEWAY_DEVICES.values()) { - try { - localHostIP = toString(gatewayDevice.getLocalAddress()); + try { + localHostIP = toString(gatewayDevice.getLocalAddress()); + + int portCandidate = port; + while (isInUse(portCandidate) && portCandidate > 0) { + LOG.info(portCandidate + " in use, trying different port."); + portCandidate = getNewPortCandidate(portCandidate); + } + + if (portCandidate > 0) { - mapped = gatewayDevice.addPortMapping(port, port, + mapped = gatewayDevice.addPortMapping(portCandidate, port, localHostIP, mapping.getProtocol(), mapping.getDescription()); - msg = "port " + port + " on device " - + gatewayDevice.getFriendlyName(); - - if (mapped) { - LOG.info("mapped " + msg); - mapping.setPort(port); - } else { - LOG.warn("could not map " + msg); - } - } catch (IOException | SAXException e) { - LOG.severe("mapping error: " + e.getMessage()); + msg = "mapped port " + port + " to port " + portCandidate + + " on device " + gatewayDevice.getFriendlyName() + + ", external IP is " + + gatewayDevice.getExternalIPAddress(); + + } else { + + mapped = false; + + msg = "no free port found"; } + + if (mapped) { + LOG.info("mapped " + msg); + mapping.setPort(portCandidate); + + SB.setUpnpPorts(mapping.getConfigPortKey(), portCandidate); + } else { + LOG.warn("could not map " + msg); + } + } catch (IOException | SAXException e) { + LOG.severe("mapping error: " + e.getMessage()); } } } @@ -173,33 +198,31 @@ public class UPnP { * @param mapping * to delete */ - public static void deletePortMapping(final UPnPMapping mapping) { - if (mapping.getPort() > 0 && GATEWAY_DEVICES != null) { + private static void deletePortMapping(final UPnPMapping mapping) { + if (mapping.getPort() > 0 && gatewayDevice != null) { boolean unmapped; String msg; - for (final GatewayDevice gatewayDevice : GATEWAY_DEVICES.values()) { - - try { - unmapped = gatewayDevice.deletePortMapping( - mapping.getPort(), mapping.getProtocol()); - msg = "port " + mapping.getPort() + " on device " - + gatewayDevice.getFriendlyName(); + try { + unmapped = gatewayDevice.deletePortMapping(mapping.getPort(), + mapping.getProtocol()); - if (unmapped) { - LOG.info("unmapped " + msg); - } else { - LOG.warn("could not unmap " + msg); - } + msg = "port " + mapping.getPort() + " on device " + + gatewayDevice.getFriendlyName(); - } catch (SAXException | IOException e) { - LOG.severe("unmapping error: " + e.getMessage()); + if (unmapped) { + LOG.info("unmapped " + msg); + } else { + LOG.warn("could not unmap " + msg); } - } - mapping.setPort(0); // reset mapped port + } catch (SAXException | IOException e) { + LOG.severe("unmapping error: " + e.getMessage()); + } } + + mapping.setPort(0); // reset mapped port } /** @@ -219,6 +242,30 @@ public class UPnP { return MAPPINGS.get(type).getPort(); } + public static int getNewPortCandidate(final int oldCandidate) { + + int newPortCandidate = Math.min( + Math.max(MIN_CANDIDATE_PORT, oldCandidate + 1), + MAX_CANDIDATE_PORT); + + if (newPortCandidate == MAX_CANDIDATE_PORT) { + newPortCandidate = -1; + } + + return newPortCandidate; + } + + private static boolean isInUse(final int port) { + + try { + return gatewayDevice != null + && gatewayDevice.getSpecificPortMappingEntry(port, "TCP", + new PortMappingEntry()); + } catch (IOException | SAXException e) { + return false; + } + } + private static String toString(final InetAddress inetAddress) { final String localHostIP; diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index c64559c4a..5a7939f8c 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -202,7 +202,7 @@ public final class yacy { sb.setConfig("memoryTotalAfterStartup", startupMemTotal); // start gui if wanted - if (gui) YaCyApp.start("localhost", (int) sb.getConfigLong("port", 8090)); + if (gui) YaCyApp.start("localhost", sb.getLocalPort("port", 8090)); // hardcoded, forced, temporary value-migration sb.setConfig("htTemplatePath", "htroot/env/templates"); @@ -277,7 +277,7 @@ public final class yacy { HTTPClient.setDefaultUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent); // start main threads - final int port = sb.getConfigInt("port", 8090); + final int port = sb.getLocalPort("port", 8090); try { // start http server YaCyHttpServer httpServer;