From 5a7f12a9c17594d45997fc7dbfecb87ceb968185 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 11 Jan 2021 00:28:24 +0100 Subject: [PATCH] allow network scans for non-standard http/https ports --- htroot/CrawlStartScanner_p.html | 8 ++--- htroot/CrawlStartScanner_p.java | 39 +++++++++++----------- source/net/yacy/cora/protocol/Scanner.java | 11 +++--- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/htroot/CrawlStartScanner_p.html b/htroot/CrawlStartScanner_p.html index 1d1219957..c22434790 100644 --- a/htroot/CrawlStartScanner_p.html +++ b/htroot/CrawlStartScanner_p.html @@ -60,10 +60,10 @@
Service Type
- ftp - smb - http - https + ftp  + smb  + http   + https 
Scheduler
diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 30a995853..ed4e050ba 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -49,8 +49,7 @@ import net.yacy.search.schema.CollectionSchema; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; -public class CrawlStartScanner_p -{ +public class CrawlStartScanner_p { private final static int CONCURRENT_RUNNER = 200; @@ -83,7 +82,7 @@ public class CrawlStartScanner_p String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(",")); Set hostSet = new LinkedHashSet(); for (String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s); - + final Set ips = Domains.myIntranetIPs(); prop.put("intranethosts", ips.toString()); prop.put("intranetHint", sb.isIntranetMode() ? 0 : 1); @@ -99,22 +98,22 @@ public class CrawlStartScanner_p hostSet.add(ip.getHostAddress()); } } else { - final Set myPublicIPs = new HashSet(); - myPublicIPs.addAll(Domains.myPublicIPv4()); - myPublicIPs.addAll(Domains.myPublicIPv6()); - for(final InetAddress myPublicIP: myPublicIPs) { + final Set myPublicIPs = new HashSet(); + myPublicIPs.addAll(Domains.myPublicIPv4()); + myPublicIPs.addAll(Domains.myPublicIPv6()); + for(final InetAddress myPublicIP: myPublicIPs) { if (Domains.isThisHostIP(myPublicIP)) { - final Set myIPs = sb.peers.mySeed().getIPs(); - for(final String myIP: myIPs) { + final Set myIPs = sb.peers.mySeed().getIPs(); + for(final String myIP: myIPs) { ip = Domains.dnsResolve(myIP); if(ip != null) { - hostSet.add(ip.getHostAddress()); + hostSet.add(ip.getHostAddress()); } - } + } } else { hostSet.add(myPublicIP.getHostAddress()); } - } + } } } String hos = ""; for (String s: hostSet) hos += s + "\n"; @@ -130,7 +129,7 @@ public class CrawlStartScanner_p for (String s: hostscore) hostSet.add(s); } } - + // parse post requests if ( post != null ) { int repeat_time = 0; @@ -149,12 +148,14 @@ public class CrawlStartScanner_p // start a scanner final Scanner scanner = new Scanner(CONCURRENT_RUNNER, timeout); - + boolean scanhttp = "on".equals(post.get("scanhttp", "")); + int scanhttpport = post.getInt("scanhttpport", 80); boolean scanhttps = "on".equals(post.get("scanhttps", "")); + int scanhttpsport = post.getInt("scanhttpsport", 443); boolean scanftp = "on".equals(post.get("scanftp", "")); boolean scansmb = "on".equals(post.get("scansmb", "")); - + // select host base to scan if ("hosts".equals(post.get("source", ""))) { for (String host: hostSet) { @@ -187,13 +188,13 @@ public class CrawlStartScanner_p if (ip != null) scanbase.add(ip); } } - scanner.addProtocols(Scanner.genlist(scanbase, subnet), scanhttp, scanhttps, scanftp, scansmb); + scanner.addProtocols(Scanner.genlist(scanbase, subnet), scanhttp, scanhttpport, scanhttps, scanhttpsport, scanftp, scansmb); } } if ("intranet".equals(post.get("source", ""))) { - scanner.addProtocols(Scanner.genlist(Domains.myIntranetIPs(), subnet), scanhttp, scanhttps, scanftp, scansmb); + scanner.addProtocols(Scanner.genlist(Domains.myIntranetIPs(), subnet), scanhttp, scanhttpport, scanhttps, scanhttpsport, scanftp, scansmb); } - + scanner.terminate(); if ("on".equals(post.get("accumulatescancache", "")) && !"scheduler".equals(post.get("rescan", ""))) { Scanner.scancacheExtend(scanner); @@ -295,5 +296,5 @@ public class CrawlStartScanner_p return prop; } - + } diff --git a/source/net/yacy/cora/protocol/Scanner.java b/source/net/yacy/cora/protocol/Scanner.java index 8c84931a9..2d4895158 100644 --- a/source/net/yacy/cora/protocol/Scanner.java +++ b/source/net/yacy/cora/protocol/Scanner.java @@ -56,6 +56,7 @@ public class Scanner { public static enum Protocol {http(80), https(443), ftp(21), smb(445); public int port; private Protocol(final int port) {this.port = port;} + public Protocol setPort(int port) {this.port = port; return this;} } public class Service implements Runnable { public Protocol protocol; @@ -228,13 +229,13 @@ public class Scanner { this.threadPool.shutdown(); } - public void addProtocols(final List addresses, boolean http, boolean https, boolean ftp, boolean smb) { - if (http) addProtocol(Protocol.http, addresses); - if (https) addProtocol(Protocol.https, addresses); + public void addProtocols(final List addresses, boolean http, int httpPort, boolean https, int httpsPort, boolean ftp, boolean smb) { + if (http) addProtocol(Protocol.http.setPort(httpPort), addresses); + if (https) addProtocol(Protocol.https.setPort(httpsPort), addresses); if (ftp) addProtocol(Protocol.ftp, addresses); if (smb) addProtocol(Protocol.smb, addresses); } - + private void addProtocol(final Protocol protocol, final List addresses) { for (final InetAddress i: addresses) { threadPool.execute(new Service(protocol, i)); @@ -294,7 +295,7 @@ public class Scanner { //try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (final MalformedURLException e) {} final Scanner scanner = new Scanner(100, 10); List addresses = genlist(Domains.myIntranetIPs(), 20); - scanner.addProtocols(addresses, true, true, true, true); + scanner.addProtocols(addresses, true, 80, true, 443, true, true); scanner.terminate(); for (final Service service: scanner.services().keySet()) { System.out.println(service.toString());