boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced network scanner

Browse Source 
- textarea input field can now be used to paste in a large list of hosts
- /31er subnet is possible (only one host)
- auto-detect subdomains for ftp and www subdomains
pull/1/head
orbiter 12 years ago
parent d8354a389c
commit 9f0cc9b401
3 changed files with 99 additions and 59 deletions
Show Stats Download Patch File Download Diff File

13
htroot/CrawlStartScanner_p.html
Unescape View File

@ -41,20 +41,19 @@
</legend>
<dl>
<dt>Scan Range</dt>
<dd>
<dd align="top">
<input type="radio" name="source" id="sourcehost" value="hosts"#(intranet.checked)# checked="checked"::#(/intranet.checked)# />Scan sub-range with given host
<input type="text" name="scanhosts" value="#[scanhosts]#" size="60" maxlength="400" /><br/><br/>
<textarea name="scanhosts" id="scanhosts" value="#[scanhosts]#" cols="64" rows="3" size="41"></textarea><br/><br/>
<input type="radio" name="source" id="sourcenet" value="intranet"#(intranet.checked)#:: checked="checked"#(/intranet.checked)# />Full Intranet Scan: #[intranethosts]#<br/>
#(intranetHint)#::<div class="info">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Do not use intranet scan results, you are not in an intranet environment!</div>#(/intranetHint)#
</dd>
<dt>Time-Out</dt>
<dt>Subnet</dt>
<dd>
<input type="text" name="timeout" value ="100" size="4"/> ms
<input type="radio" name="subnet" value="31"/>/31 (only the given host(s)) <input type="radio" name="subnet" value="24" checked="checked"/>/24 (254 addresses) <input type="radio" name="subnet" value="20"/>/20 (4064 addresses) <input type="radio" name="subnet" value="16"/>/16 (65024 adresses)
</dd>
<dt>Subnet</dt>
<dt>Time-Out</dt>
<dd>
<input type="radio" name="subnet" value="24" checked="checked"/>/24 (254 addresses) <input type="radio" name="subnet" value="20"/>/20 (4064 addresses) <input type="radio" name="subnet" value="16"/>/16 (65024 adresses)
<input type="text" name="timeout" value ="500" size="4"/> ms
</dd>
<dt>Scan Cache</dt>
<dd>

70
htroot/CrawlStartScanner_p.java
Unescape View File

@ -18,27 +18,33 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
import java.io.IOException;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.util.ConcurrentModificationException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.Scanner;
import net.yacy.cora.protocol.Scanner.Access;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.data.WorkTables;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@ -66,11 +72,20 @@ public class CrawlStartScanner_p
timeout = post == null ? timeout : post.getInt("timeout", timeout);
// make a scanhosts entry
String hosts = post == null ? "" : post.get("scanhosts", "");
String hostt = post == null ? "" : post.get("scanhosts", "").trim();
boolean listall = false;
if (hostt.equals("*")) {
hostt = "";
listall = true;
}
String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(","));
Set<String> hostSet = new LinkedHashSet<String>();
for (String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s);
final Set<InetAddress> ips = Domains.myIntranetIPs();
prop.put("intranethosts", ips.toString());
prop.put("intranetHint", sb.isIntranetMode() ? 0 : 1);
if ( hosts.isEmpty() ) {
if ( hostSet.isEmpty() ) {
InetAddress ip;
if ( sb.isIntranetMode() ) {
if ( !ips.isEmpty() ) {
@ -85,11 +100,23 @@ public class CrawlStartScanner_p
}
}
if ( ip != null ) {
hosts = ip.getHostAddress();
hostSet.add(ip.getHostAddress());
}
}
prop.put("scanhosts", hosts);
String hos = ""; for (String s: hostSet) hos += s + "\n";
prop.put("scanhosts", hos.trim());
if (listall) {
// get a list of all hosts in the index
ReversibleScoreMap<String> hostscore = null;
try {
hostscore = sb.index.fulltext().getDefaultConnector().getFacets(AbstractSolrConnector.CATCHALL_TERM, 1000, CollectionSchema.host_s.getSolrFieldName()).get(CollectionSchema.host_s.getSolrFieldName());
} catch (IOException e) {}
if (hostscore != null) {
for (String s: hostscore) hostSet.add(s);
}
}
// parse post requests
if ( post != null ) {
int repeat_time = 0;
@ -105,18 +132,35 @@ public class CrawlStartScanner_p
// scan a range of ips
if (post.containsKey("scan")) {
boolean scanftp = "on".equals(post.get("scanftp", ""));
boolean scanhttp = "on".equals(post.get("scanhttp", ""));
boolean scanhttps = "on".equals(post.get("scanhttps", ""));
boolean scansmb = "on".equals(post.get("scansmb", ""));
final Set<InetAddress> scanbase = new HashSet<InetAddress>();
// select host base to scan
if ("hosts".equals(post.get("source", ""))) {
for (String host: hosts.split(",")) {
for (String host: hostSet) {
if (host.startsWith("http://")) host = host.substring(7);
if (host.startsWith("https://")) host = host.substring(8);
if (host.startsWith("ftp://")) host = host.substring(6);
if (host.startsWith("smb://")) host = host.substring(6);
final int p = host.indexOf('/', 0);
if (p >= 0) host = host.substring(0, p);
if (host.length() > 0) scanbase.add(Domains.dnsResolve(host));
InetAddress ip;
if (host.length() > 0) {
ip = Domains.dnsResolve(host); if (ip != null) scanbase.add(ip);
if (scanftp && !hostSet.contains("ftp." + host)) {
ip = Domains.dnsResolve("ftp." + host);
if (ip != null) scanbase.add(ip);
}
if ((scanhttp || scanhttps) && !hostSet.contains("www." + host)) {
ip = Domains.dnsResolve("www." + host);
if (ip != null) scanbase.add(ip);
}
}
}
}
if ("intranet".equals(post.get("source", ""))) {
@ -124,12 +168,12 @@ public class CrawlStartScanner_p
}
// start a scanner
final Scanner scanner = new Scanner(scanbase, CONCURRENT_RUNNER, timeout);
List<InetAddress> addresses = scanner.genlist(subnet);
if ("on".equals(post.get("scanftp", ""))) scanner.addFTP(addresses);
if ("on".equals(post.get("scanhttp", ""))) scanner.addHTTP(addresses);
if ("on".equals(post.get("scanhttps", ""))) scanner.addHTTPS(addresses);
if ("on".equals(post.get("scansmb", ""))) scanner.addSMB(addresses);
final Scanner scanner = new Scanner(CONCURRENT_RUNNER, timeout);
List<InetAddress> addresses = Scanner.genlist(scanbase, subnet);
if (scanftp) scanner.addFTP(addresses);
if (scanhttp) scanner.addHTTP(addresses);
if (scanhttps) scanner.addHTTPS(addresses);
if (scansmb) scanner.addSMB(addresses);
scanner.start();
scanner.terminate();
if ("on".equals(post.get("accumulatescancache", "")) && !"scheduler".equals(post.get("rescan", ""))) {
@ -183,7 +227,7 @@ public class CrawlStartScanner_p
post,
"CrawlStartScanner_p.html",
WorkTables.TABLE_API_TYPE_CRAWLER,
"network scanner for hosts: " + hosts,
"network scanner for hosts: " + hostSet.toString(),
repeat_time,
repeat_unit.substring(3));
}

75
source/net/yacy/cora/protocol/Scanner.java
Unescape View File

@ -29,13 +29,12 @@ import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@ -121,9 +120,6 @@ public class Scanner extends Thread {
}
private final static Map<Service, Access> scancache = new ConcurrentHashMap<Service, Access>();
//private static long scancacheUpdateTime = 0;
//private static long scancacheValidUntilTime = Long.MAX_VALUE;
private static Set<InetAddress> scancacheScanrange = new HashSet<InetAddress>();
public static int scancacheSize() {
return scancache.size();
@ -132,9 +128,6 @@ public class Scanner extends Thread {
public static void scancacheReplace(final Scanner newScanner) {
scancache.clear();
scancache.putAll(newScanner.services());
//scancacheUpdateTime = System.currentTimeMillis();
//scancacheValidUntilTime = validTime == Long.MAX_VALUE ? Long.MAX_VALUE : scancacheUpdateTime + validTime;
scancacheScanrange = newScanner.scanrange;
}
public static void scancacheExtend(final Scanner newScanner) {
@ -145,9 +138,6 @@ public class Scanner extends Thread {
if (entry.getValue() != Access.granted) i.remove();
}
scancache.putAll(newScanner.services());
//scancacheUpdateTime = System.currentTimeMillis();
//scancacheValidUntilTime = validTime == Long.MAX_VALUE ? Long.MAX_VALUE : scancacheUpdateTime + validTime;
scancacheScanrange = newScanner.scanrange;
}
public static Iterator<Map.Entry<Service, Scanner.Access>> scancacheEntries() {
@ -163,18 +153,17 @@ public class Scanner extends Thread {
*/
public static boolean acceptURL(final MultiProtocolURI url) {
// if the scan range is empty, then all urls are accepted
if (scancacheScanrange == null || scancacheScanrange.isEmpty()) return true;
if (scancache == null || scancache.isEmpty()) return true;
//if (System.currentTimeMillis() > scancacheValidUntilTime) return true;
final InetAddress a = url.getInetAddress(); // try to avoid that!
if (a == null) return true;
final InetAddress n = normalize(a);
if (!scancacheScanrange.contains(n)) return true;
final Access access = scancache.get(new Service(url.getProtocol(), a));
if (access == null) return false;
return access == Access.granted;
}
/*
private static InetAddress normalize(final InetAddress a) {
if (a == null) return null;
final byte[] b = a.getAddress();
@ -186,28 +175,22 @@ public class Scanner extends Thread {
return a;
}
}
*/
private final int runnerCount;
private final Set<InetAddress> scanrange;
private final BlockingQueue<Service> scanqueue;
private final Map<Service, Access> services;
private final Map<Runner, Object> runner;
private final int timeout;
public Scanner(final Set<InetAddress> scanrange, final int concurrentRunner, final int timeout) {
public Scanner(final int concurrentRunner, final int timeout) {
this.runnerCount = concurrentRunner;
this.scanrange = new HashSet<InetAddress>();
for (final InetAddress a: scanrange) this.scanrange.add(normalize(a));
this.scanqueue = new LinkedBlockingQueue<Service>();
this.services = Collections.synchronizedMap(new HashMap<Service, Access>());
this.runner = new ConcurrentHashMap<Runner, Object>();
this.timeout = timeout;
}
public Scanner(final int concurrentRunner, final int timeout) {
this(Domains.myIntranetIPs(), concurrentRunner, timeout);
}
@Override
public void run() {
Service uri;
@ -332,23 +315,37 @@ public class Scanner extends Thread {
* @param subnet the subnet: 24 will generate 254 addresses, 16 will generate 256 * 254; must be >= 16 and <= 24
* @return
*/
public final List<InetAddress> genlist(final int subnet) {
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(10);
for (final InetAddress i: this.scanrange) {
int ul = subnet >= 24 ? i.getAddress()[2] : (1 << (24 - subnet)) - 1;
for (int br = subnet >= 24 ? i.getAddress()[2] : 0; br <= ul; br++) {
for (int j = 1; j < 255; j++) {
final byte[] address = i.getAddress();
address[2] = (byte) br;
address[3] = (byte) j;
try {
c.add(InetAddress.getByAddress(address));
} catch (final UnknownHostException e) {
public static final List<InetAddress> genlist(Collection<InetAddress> base, final int subnet) {
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(1);
for (final InetAddress i: base) {
genlist(c, i, subnet);
}
return c;
}
public static final List<InetAddress> genlist(InetAddress base, final int subnet) {
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(1);
genlist(c, base, subnet);
return c;
}
private static final void genlist(ArrayList<InetAddress> c, InetAddress base, final int subnet) {
if (subnet == 31) {
try {
c.add(InetAddress.getByAddress(base.getAddress()));
} catch (UnknownHostException e) {}
} else {
int ul = subnet >= 24 ? base.getAddress()[2] : (1 << (24 - subnet)) - 1;
for (int br = subnet >= 24 ? base.getAddress()[2] : 0; br <= ul; br++) {
for (int j = 1; j < 255; j++) {
final byte[] address = base.getAddress();
address[2] = (byte) br;
address[3] = (byte) j;
try {
c.add(InetAddress.getByAddress(address));
} catch (final UnknownHostException e) {
}
}
}
}
}
return c;
}
public Map<Service, Access> services() {
@ -365,7 +362,7 @@ public class Scanner extends Thread {
public static void main(final String[] args) {
//try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (MalformedURLException e) {}
final Scanner scanner = new Scanner(100, 10);
List<InetAddress> addresses = scanner.genlist(20);
List<InetAddress> addresses = genlist(Domains.myIntranetIPs(), 20);
scanner.addFTP(addresses);
scanner.addHTTP(addresses);
scanner.addHTTPS(addresses);

Write Preview
Loading…
Cancel
Save