added network scanner process to discover all YaCy peers in the intranet

this will be used to wire YaCy peers in a kubernetes cluster

htroot/api/localpeers.java

@@ -0,0 +1,41 @@
+// localpeers.java
+// ------------
+// (C) 2021 by Michael Peter Christen; mc@yacy.net
+// first published 23.01.2021 on http://yacy.net
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.search.Switchboard;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
+import net.yacy.server.servletProperties;
+
+public class localpeers {
+
+    public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
+        final servletProperties prop = new servletProperties();
+        int c = 0;
+        for (String urlstub: Switchboard.getSwitchboard().localpeers) {
+            prop.putJSON("peers_" + c + "_urlstub", urlstub);
+            c++;
+        }
+        prop.put("peers", c);
+
+        // return rewrite properties
+        return prop;
+    }
+
+}

htroot/api/localpeers.json

@@ -0,0 +1,5 @@
+{
+"peers" : [#{peers|,}#
+{"urlstub":"#[urlstub]#"}
+#{/peers}#]
+}

source/net/yacy/cora/protocol/Domains.java

@@ -106,6 +106,7 @@ public class Domains {
     private static Set<InetAddress> publicIPv4HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
     private static Set<InetAddress> publicIPv6HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
     private static Set<String> localHostNames = new HashSet<String>(); // subset of myHostNames
+    private static Thread domaininit = null;
     static {
         localHostNames.add(LOCALHOST);
         try {
@@ -121,7 +122,7 @@ public class Domains {
         // if such a lookup blocks, it can cause that the static initiatializer does not finish fast
         // therefore we start the host name lookup as concurrent thread
         // meanwhile the host name is "127.0.0.1" which is not completely wrong
-        new Thread("Domains: init") {
+        domaininit = new Thread("Domains: init") {
             @Override
             public void run() {
                 // try to get local addresses from interfaces
@@ -188,7 +189,8 @@ public class Domains {
                     }
                 }
             }
-        }.start();
+        };
+        domaininit.start();
     }
     
     /**
@@ -1103,9 +1105,21 @@ public class Domains {
      * @return list of all intranet addresses
      */
     public static Set<InetAddress> myIntranetIPs() {
-        if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {}
+        while (domaininit == null || domaininit.isAlive()) try {Thread.sleep(1000);} catch (final InterruptedException e) {}
         return localHostAddresses;
     }
+    
+    public static Set<InetAddress> myIPv4IntranetIPs() {
+        Set<InetAddress> in = new HashSet<>();
+        for (InetAddress a: myIntranetIPs()) if (a instanceof Inet4Address) in.add(a);
+        return in;
+    }
+    
+    public static Set<InetAddress> myIPv4IntranetNonLocalhostIPs() {
+        Set<InetAddress> in = new HashSet<>();
+        for (InetAddress a: myIPv4IntranetIPs()) if (((Inet4Address) a).getAddress()[0] != 127) in.add(a);
+        return in;
+    }
 
     /**
      * this method is deprecated in some way because it is not applicable on IPv6

source/net/yacy/cora/protocol/Scanner.java

@@ -25,6 +25,7 @@
 package net.yacy.cora.protocol;
 
 import java.io.IOException;
+import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.UnknownHostException;
@@ -32,9 +33,11 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -53,10 +56,14 @@ import net.yacy.cora.protocol.http.HTTPClient;
 public class Scanner {
 
     public static enum Access {unknown, empty, granted, denied;}
-    public static enum Protocol {http(80), https(443), ftp(21), smb(445);
+    public static enum Protocol {http(80, true), https(443, true), ftp(21, true), smb(445, true);
         public int port;
-        private Protocol(final int port) {this.port = port;}
-        public Protocol setPort(int port) {this.port = port; return this;}
+        public boolean standard;
+        private Protocol(final int port, boolean standard) {this.port = port; this.standard = standard;}
+        public Protocol setPort(int port, boolean standard) {this.port = port; this.standard = standard; return this;}
+        public String portSuffix() {
+            return this.standard ? "" : ":" + this.port;
+        }
     }
     public class Service implements Runnable {
         public Protocol protocol;
@@ -106,7 +113,7 @@ public class Scanner {
         @Override
         public String toString() {
             try {
-                return new MultiProtocolURL(this.protocol.name() + "://" + this.inetAddress.getHostAddress() + "/").toNormalform(true);
+                return new MultiProtocolURL(this.protocol.name() + "://" + this.inetAddress.getHostAddress() + this.protocol.portSuffix() + "/").toNormalform(true);
             } catch (final MalformedURLException e) {
                 return "";
             }
@@ -230,13 +237,13 @@ public class Scanner {
     }
 
     public void addProtocols(final List<InetAddress> addresses, boolean http, int httpPort, boolean https, int httpsPort, boolean ftp, boolean smb) {
-        if (http) addProtocol(Protocol.http.setPort(httpPort), addresses);
-        if (https) addProtocol(Protocol.https.setPort(httpsPort), addresses);
+        if (http) addProtocol(Protocol.http.setPort(httpPort, httpPort == Protocol.http.port), addresses);
+        if (https) addProtocol(Protocol.https.setPort(httpsPort, httpsPort == Protocol.https.port), addresses);
         if (ftp) addProtocol(Protocol.ftp, addresses);
         if (smb) addProtocol(Protocol.smb, addresses);
     }
 
-    private void addProtocol(final Protocol protocol, final List<InetAddress> addresses) {
+    public void addProtocol(final Protocol protocol, final List<InetAddress> addresses) {
         for (final InetAddress i: addresses) {
             threadPool.execute(new Service(protocol, i));
         }
@@ -291,18 +298,50 @@ public class Scanner {
         return null;
     }
 
+    public static Set<String> scanForOtherYaCyInIntranet() {
+        final Set<InetAddress> in = Domains.myIPv4IntranetNonLocalhostIPs();
+        final List<InetAddress> myaddresses = genlist(in, 20);
+        myaddresses.removeAll(in);
+        final Scanner scanner = new Scanner(100, 1000);
+        scanner.addProtocol(Protocol.http.setPort(8090, false), myaddresses);
+        while (scanner.pending() > 0) try {Thread.sleep(1000);} catch (InterruptedException e1) {}
+        scanner.terminate();
+        final Set<String> urls = Collections.newSetFromMap(new ConcurrentHashMap<>());
+        scanner.services().keySet().forEach(service -> {
+            String urlstub = service.toString();
+            boolean notMy = true;
+            for (InetAddress a: in) if (urlstub.indexOf(a.getHostAddress()) >= 0) notMy = false;
+            if (notMy) urls.add(urlstub);
+        });
+        return urls;
+    }
+
     public static void main(final String[] args) {
+        // boolean test = TimeoutRequest.ping("192.168.1.40", 8090, 10000);
+
         //try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (final MalformedURLException e) {}
-        final Scanner scanner = new Scanner(100, 10);
-        List<InetAddress> addresses = genlist(Domains.myIntranetIPs(), 20);
-        scanner.addProtocols(addresses, true, 80, true, 443, true, true);
+        System.out.println("collecting addresses");
+        Set<InetAddress> in = Domains.myIPv4IntranetNonLocalhostIPs();
+        System.out.println("found " + in.size() + " intranet addresses");
+        for (InetAddress a: in) System.out.println("intranet address:" + a.getHostAddress());
+        List<InetAddress> addresses = genlist(in, 20);
+        //for (InetAddress a: addresses) System.out.print(a.getHostAddress() + " ");
+        System.out.println();
+        System.out.println("scanner start with " + addresses.size() + " addresses");
+        long time = System.currentTimeMillis();
+        final Scanner scanner = new Scanner(100, 1000);
+        scanner.addProtocol(Protocol.http.setPort(8090, false), addresses);
+        while (scanner.pending() > 0) try {Thread.sleep(1000);} catch (InterruptedException e1) {}
         scanner.terminate();
         for (final Service service: scanner.services().keySet()) {
-            System.out.println(service.toString());
+            System.out.println("PING successful: " + service.toString());
         }
         try {
             HTTPClient.closeConnectionManager();
         } catch (final InterruptedException e) {
         }
+        time = System.currentTimeMillis() - time;
+        System.out.println("scanner terminated after " + time/1000 + " seconds, " + 1000 * addresses.size() / time + " addresses per second");
+        System.exit(0);
     }
 }

source/net/yacy/search/Switchboard.java

@@ -63,6 +63,7 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -133,6 +134,7 @@ import net.yacy.cora.protocol.ClientIdentification;
 import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.cora.protocol.Domains;
 import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.protocol.Scanner;
 import net.yacy.cora.protocol.TimeoutRequest;
 import net.yacy.cora.protocol.http.HTTPClient;
 import net.yacy.cora.protocol.http.ProxySettings;
@@ -302,6 +304,7 @@ public final class Switchboard extends serverSwitch {
     private Dispatcher dhtDispatcher;
     public LinkedBlockingQueue<String> trail; // connect infos from cytag servlet
     public SeedDB peers;
+    public Set<String> localpeers;
     public WorkTables tables;
     public Tray tray;
     private long lastStats = 0; // time when the last row was written to the stats table
@@ -626,6 +629,14 @@ public final class Switchboard extends serverSwitch {
                 partitionExponent,
                 false,
                 this.exceed134217727);
+        this.localpeers = Collections.newSetFromMap(new ConcurrentHashMap<>());
+        new OneTimeBusyThread("Switchboard.scanForOtherYaCyInIntranet") {
+            @Override
+            public boolean jobImpl() throws Exception {
+                Switchboard.this.localpeers.addAll(Scanner.scanForOtherYaCyInIntranet());
+                return true;
+            }
+        }.start();
 
         // load domainList
         try {

source/net/yacy/yacy.java

@@ -62,7 +62,6 @@ import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.cora.protocol.HeaderFramework;
 import net.yacy.cora.protocol.TimeoutRequest;
 import net.yacy.cora.protocol.http.HTTPClient;
-import net.yacy.cora.sorting.Array;
 import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.data.TransactionManager;
 import net.yacy.data.Translator;