diff --git a/htroot/api/localpeers.java b/htroot/api/localpeers.java new file mode 100644 index 000000000..ba933d125 --- /dev/null +++ b/htroot/api/localpeers.java @@ -0,0 +1,41 @@ +// localpeers.java +// ------------ +// (C) 2021 by Michael Peter Christen; mc@yacy.net +// first published 23.01.2021 on http://yacy.net +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.search.Switchboard; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; +import net.yacy.server.servletProperties; + +public class localpeers { + + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { + final servletProperties prop = new servletProperties(); + int c = 0; + for (String urlstub: Switchboard.getSwitchboard().localpeers) { + prop.putJSON("peers_" + c + "_urlstub", urlstub); + c++; + } + prop.put("peers", c); + + // return rewrite properties + return prop; + } + +} diff --git a/htroot/api/localpeers.json b/htroot/api/localpeers.json new file mode 100644 index 000000000..e2be9012d --- /dev/null +++ b/htroot/api/localpeers.json @@ -0,0 +1,5 @@ +{ +"peers" : [#{peers|,}# +{"urlstub":"#[urlstub]#"} +#{/peers}#] +} \ No newline at end of file diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 6e71bf8a5..9a1c9836f 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -106,6 +106,7 @@ public class Domains { private static Set publicIPv4HostAddresses = new HashSet(); // subset of myHostAddresses private static Set publicIPv6HostAddresses = new HashSet(); // subset of myHostAddresses private static Set localHostNames = new HashSet(); // subset of myHostNames + private static Thread domaininit = null; static { localHostNames.add(LOCALHOST); try { @@ -121,7 +122,7 @@ public class Domains { // if such a lookup blocks, it can cause that the static initiatializer does not finish fast // therefore we start the host name lookup as concurrent thread // meanwhile the host name is "127.0.0.1" which is not completely wrong - new Thread("Domains: init") { + domaininit = new Thread("Domains: init") { @Override public void run() { // try to get local addresses from interfaces @@ -188,7 +189,8 @@ public class Domains { } } } - }.start(); + }; + domaininit.start(); } /** @@ -1103,9 +1105,21 @@ public class Domains { * @return list of all intranet addresses */ public static Set myIntranetIPs() { - if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {} + while (domaininit == null || domaininit.isAlive()) try {Thread.sleep(1000);} catch (final InterruptedException e) {} return localHostAddresses; } + + public static Set myIPv4IntranetIPs() { + Set in = new HashSet<>(); + for (InetAddress a: myIntranetIPs()) if (a instanceof Inet4Address) in.add(a); + return in; + } + + public static Set myIPv4IntranetNonLocalhostIPs() { + Set in = new HashSet<>(); + for (InetAddress a: myIPv4IntranetIPs()) if (((Inet4Address) a).getAddress()[0] != 127) in.add(a); + return in; + } /** * this method is deprecated in some way because it is not applicable on IPv6 diff --git a/source/net/yacy/cora/protocol/Scanner.java b/source/net/yacy/cora/protocol/Scanner.java index 2d4895158..cd1bb8132 100644 --- a/source/net/yacy/cora/protocol/Scanner.java +++ b/source/net/yacy/cora/protocol/Scanner.java @@ -25,6 +25,7 @@ package net.yacy.cora.protocol; import java.io.IOException; +import java.net.Inet4Address; import java.net.InetAddress; import java.net.MalformedURLException; import java.net.UnknownHostException; @@ -32,9 +33,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; @@ -53,10 +56,14 @@ import net.yacy.cora.protocol.http.HTTPClient; public class Scanner { public static enum Access {unknown, empty, granted, denied;} - public static enum Protocol {http(80), https(443), ftp(21), smb(445); + public static enum Protocol {http(80, true), https(443, true), ftp(21, true), smb(445, true); public int port; - private Protocol(final int port) {this.port = port;} - public Protocol setPort(int port) {this.port = port; return this;} + public boolean standard; + private Protocol(final int port, boolean standard) {this.port = port; this.standard = standard;} + public Protocol setPort(int port, boolean standard) {this.port = port; this.standard = standard; return this;} + public String portSuffix() { + return this.standard ? "" : ":" + this.port; + } } public class Service implements Runnable { public Protocol protocol; @@ -106,7 +113,7 @@ public class Scanner { @Override public String toString() { try { - return new MultiProtocolURL(this.protocol.name() + "://" + this.inetAddress.getHostAddress() + "/").toNormalform(true); + return new MultiProtocolURL(this.protocol.name() + "://" + this.inetAddress.getHostAddress() + this.protocol.portSuffix() + "/").toNormalform(true); } catch (final MalformedURLException e) { return ""; } @@ -230,13 +237,13 @@ public class Scanner { } public void addProtocols(final List addresses, boolean http, int httpPort, boolean https, int httpsPort, boolean ftp, boolean smb) { - if (http) addProtocol(Protocol.http.setPort(httpPort), addresses); - if (https) addProtocol(Protocol.https.setPort(httpsPort), addresses); + if (http) addProtocol(Protocol.http.setPort(httpPort, httpPort == Protocol.http.port), addresses); + if (https) addProtocol(Protocol.https.setPort(httpsPort, httpsPort == Protocol.https.port), addresses); if (ftp) addProtocol(Protocol.ftp, addresses); if (smb) addProtocol(Protocol.smb, addresses); } - private void addProtocol(final Protocol protocol, final List addresses) { + public void addProtocol(final Protocol protocol, final List addresses) { for (final InetAddress i: addresses) { threadPool.execute(new Service(protocol, i)); } @@ -291,18 +298,50 @@ public class Scanner { return null; } + public static Set scanForOtherYaCyInIntranet() { + final Set in = Domains.myIPv4IntranetNonLocalhostIPs(); + final List myaddresses = genlist(in, 20); + myaddresses.removeAll(in); + final Scanner scanner = new Scanner(100, 1000); + scanner.addProtocol(Protocol.http.setPort(8090, false), myaddresses); + while (scanner.pending() > 0) try {Thread.sleep(1000);} catch (InterruptedException e1) {} + scanner.terminate(); + final Set urls = Collections.newSetFromMap(new ConcurrentHashMap<>()); + scanner.services().keySet().forEach(service -> { + String urlstub = service.toString(); + boolean notMy = true; + for (InetAddress a: in) if (urlstub.indexOf(a.getHostAddress()) >= 0) notMy = false; + if (notMy) urls.add(urlstub); + }); + return urls; + } + public static void main(final String[] args) { + // boolean test = TimeoutRequest.ping("192.168.1.40", 8090, 10000); + //try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (final MalformedURLException e) {} - final Scanner scanner = new Scanner(100, 10); - List addresses = genlist(Domains.myIntranetIPs(), 20); - scanner.addProtocols(addresses, true, 80, true, 443, true, true); + System.out.println("collecting addresses"); + Set in = Domains.myIPv4IntranetNonLocalhostIPs(); + System.out.println("found " + in.size() + " intranet addresses"); + for (InetAddress a: in) System.out.println("intranet address:" + a.getHostAddress()); + List addresses = genlist(in, 20); + //for (InetAddress a: addresses) System.out.print(a.getHostAddress() + " "); + System.out.println(); + System.out.println("scanner start with " + addresses.size() + " addresses"); + long time = System.currentTimeMillis(); + final Scanner scanner = new Scanner(100, 1000); + scanner.addProtocol(Protocol.http.setPort(8090, false), addresses); + while (scanner.pending() > 0) try {Thread.sleep(1000);} catch (InterruptedException e1) {} scanner.terminate(); for (final Service service: scanner.services().keySet()) { - System.out.println(service.toString()); + System.out.println("PING successful: " + service.toString()); } try { HTTPClient.closeConnectionManager(); } catch (final InterruptedException e) { } + time = System.currentTimeMillis() - time; + System.out.println("scanner terminated after " + time/1000 + " seconds, " + 1000 * addresses.size() / time + " addresses per second"); + System.exit(0); } } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index eae4b82d0..352548bfe 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -63,6 +63,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -133,6 +134,7 @@ import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.protocol.Scanner; import net.yacy.cora.protocol.TimeoutRequest; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.protocol.http.ProxySettings; @@ -302,6 +304,7 @@ public final class Switchboard extends serverSwitch { private Dispatcher dhtDispatcher; public LinkedBlockingQueue trail; // connect infos from cytag servlet public SeedDB peers; + public Set localpeers; public WorkTables tables; public Tray tray; private long lastStats = 0; // time when the last row was written to the stats table @@ -626,6 +629,14 @@ public final class Switchboard extends serverSwitch { partitionExponent, false, this.exceed134217727); + this.localpeers = Collections.newSetFromMap(new ConcurrentHashMap<>()); + new OneTimeBusyThread("Switchboard.scanForOtherYaCyInIntranet") { + @Override + public boolean jobImpl() throws Exception { + Switchboard.this.localpeers.addAll(Scanner.scanForOtherYaCyInIntranet()); + return true; + } + }.start(); // load domainList try { diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 9296ef2f9..d114fef92 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -62,7 +62,6 @@ import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.TimeoutRequest; import net.yacy.cora.protocol.http.HTTPClient; -import net.yacy.cora.sorting.Array; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.TransactionManager; import net.yacy.data.Translator;