boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

allow network scans for non-standard http/https ports

Browse Source
pull/402/head
Michael Peter Christen 4 years ago
parent 022fb15670
commit 5a7f12a9c1
3 changed files with 30 additions and 28 deletions
Show Stats Download Patch File Download Diff File

8
htroot/CrawlStartScanner_p.html
Unescape View File

@ -60,10 +60,10 @@
</dd>
<dt>Service Type</dt>
<dd>
<input type="checkbox" name="scanftp" checked="checked" />ftp
<input type="checkbox" name="scansmb" checked="checked" />smb
<input type="checkbox" name="scanhttp" />http
<input type="checkbox" name="scanhttps" />https
<input type="checkbox" name="scanftp" checked="checked" />ftp&nbsp;
<input type="checkbox" name="scansmb" checked="checked" />smb&nbsp;
<input type="checkbox" name="scanhttp" />http&nbsp;<input type="number" name="scanhttpport" value="80" size="5" min="0" max="65535" />&nbsp;
<input type="checkbox" name="scanhttps" />https&nbsp;<input type="number" name="scanhttpsport" value="443" size="5" min="0" max="65535" />
</dd>
<dt>Scheduler</dt>

39
htroot/CrawlStartScanner_p.java
Unescape View File

@ -49,8 +49,7 @@ import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public class CrawlStartScanner_p
{
public class CrawlStartScanner_p {
private final static int CONCURRENT_RUNNER = 200;
@ -83,7 +82,7 @@ public class CrawlStartScanner_p
String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(","));
Set<String> hostSet = new LinkedHashSet<String>();
for (String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s);
final Set<InetAddress> ips = Domains.myIntranetIPs();
prop.put("intranethosts", ips.toString());
prop.put("intranetHint", sb.isIntranetMode() ? 0 : 1);
@ -99,22 +98,22 @@ public class CrawlStartScanner_p
hostSet.add(ip.getHostAddress());
}
} else {
final Set<InetAddress> myPublicIPs = new HashSet<InetAddress>();
myPublicIPs.addAll(Domains.myPublicIPv4());
myPublicIPs.addAll(Domains.myPublicIPv6());
for(final InetAddress myPublicIP: myPublicIPs) {
final Set<InetAddress> myPublicIPs = new HashSet<InetAddress>();
myPublicIPs.addAll(Domains.myPublicIPv4());
myPublicIPs.addAll(Domains.myPublicIPv6());
for(final InetAddress myPublicIP: myPublicIPs) {
if (Domains.isThisHostIP(myPublicIP)) {
final Set<String> myIPs = sb.peers.mySeed().getIPs();
for(final String myIP: myIPs) {
final Set<String> myIPs = sb.peers.mySeed().getIPs();
for(final String myIP: myIPs) {
ip = Domains.dnsResolve(myIP);
if(ip != null) {
hostSet.add(ip.getHostAddress());
hostSet.add(ip.getHostAddress());
}
}
}
} else {
hostSet.add(myPublicIP.getHostAddress());
}
}
}
}
}
String hos = ""; for (String s: hostSet) hos += s + "\n";
@ -130,7 +129,7 @@ public class CrawlStartScanner_p
for (String s: hostscore) hostSet.add(s);
}
}
// parse post requests
if ( post != null ) {
int repeat_time = 0;
@ -149,12 +148,14 @@ public class CrawlStartScanner_p
// start a scanner
final Scanner scanner = new Scanner(CONCURRENT_RUNNER, timeout);
boolean scanhttp = "on".equals(post.get("scanhttp", ""));
int scanhttpport = post.getInt("scanhttpport", 80);
boolean scanhttps = "on".equals(post.get("scanhttps", ""));
int scanhttpsport = post.getInt("scanhttpsport", 443);
boolean scanftp = "on".equals(post.get("scanftp", ""));
boolean scansmb = "on".equals(post.get("scansmb", ""));
// select host base to scan
if ("hosts".equals(post.get("source", ""))) {
for (String host: hostSet) {
@ -187,13 +188,13 @@ public class CrawlStartScanner_p
if (ip != null) scanbase.add(ip);
}
}
scanner.addProtocols(Scanner.genlist(scanbase, subnet), scanhttp, scanhttps, scanftp, scansmb);
scanner.addProtocols(Scanner.genlist(scanbase, subnet), scanhttp, scanhttpport, scanhttps, scanhttpsport, scanftp, scansmb);
}
}
if ("intranet".equals(post.get("source", ""))) {
scanner.addProtocols(Scanner.genlist(Domains.myIntranetIPs(), subnet), scanhttp, scanhttps, scanftp, scansmb);
scanner.addProtocols(Scanner.genlist(Domains.myIntranetIPs(), subnet), scanhttp, scanhttpport, scanhttps, scanhttpsport, scanftp, scansmb);
}
scanner.terminate();
if ("on".equals(post.get("accumulatescancache", "")) && !"scheduler".equals(post.get("rescan", ""))) {
Scanner.scancacheExtend(scanner);
@ -295,5 +296,5 @@ public class CrawlStartScanner_p
return prop;
}
}

11
source/net/yacy/cora/protocol/Scanner.java
Unescape View File

@ -56,6 +56,7 @@ public class Scanner {
public static enum Protocol {http(80), https(443), ftp(21), smb(445);
public int port;
private Protocol(final int port) {this.port = port;}
public Protocol setPort(int port) {this.port = port; return this;}
}
public class Service implements Runnable {
public Protocol protocol;
@ -228,13 +229,13 @@ public class Scanner {
this.threadPool.shutdown();
}
public void addProtocols(final List<InetAddress> addresses, boolean http, boolean https, boolean ftp, boolean smb) {
if (http) addProtocol(Protocol.http, addresses);
if (https) addProtocol(Protocol.https, addresses);
public void addProtocols(final List<InetAddress> addresses, boolean http, int httpPort, boolean https, int httpsPort, boolean ftp, boolean smb) {
if (http) addProtocol(Protocol.http.setPort(httpPort), addresses);
if (https) addProtocol(Protocol.https.setPort(httpsPort), addresses);
if (ftp) addProtocol(Protocol.ftp, addresses);
if (smb) addProtocol(Protocol.smb, addresses);
}
private void addProtocol(final Protocol protocol, final List<InetAddress> addresses) {
for (final InetAddress i: addresses) {
threadPool.execute(new Service(protocol, i));
@ -294,7 +295,7 @@ public class Scanner {
//try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (final MalformedURLException e) {}
final Scanner scanner = new Scanner(100, 10);
List<InetAddress> addresses = genlist(Domains.myIntranetIPs(), 20);
scanner.addProtocols(addresses, true, true, true, true);
scanner.addProtocols(addresses, true, 80, true, 443, true, true);
scanner.terminate();
for (final Service service: scanner.services().keySet()) {
System.out.println(service.toString());

Write Preview
Loading…
Cancel
Save