From c79f687110df9c5bc3361e7b2d503ce71fd83515 Mon Sep 17 00:00:00 2001
From: orbiter <mc@yacy.net>
Date: Tue, 9 Jul 2013 11:42:13 +0200
Subject: [PATCH] enhanced the network scanner: find more hosts automatically
 by removal of common subdomains before application of protocol-specific
 prefix

---
 htroot/CrawlStartScanner_p.html |  8 ++++----
 htroot/CrawlStartScanner_p.java | 10 ++++++++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/htroot/CrawlStartScanner_p.html b/htroot/CrawlStartScanner_p.html
index 501a7e07b..ba8b6e191 100644
--- a/htroot/CrawlStartScanner_p.html
+++ b/htroot/CrawlStartScanner_p.html
@@ -77,15 +77,15 @@
 		<select name="repeat_time">
            <option value="1">1</option><option value="2">2</option><option value="3">3</option>
            <option value="4">4</option><option value="5">5</option><option value="6">6</option>
-           <option value="7">7</option><option value="8">8</option>
-           <option value="9">9</option><option value="10" selected="selected">10</option>
+           <option value="7" selected="selected">7</option><option value="8">8</option>
+           <option value="9">9</option><option value="10">10</option>
            <option value="12">12</option><option value="14">14</option><option value="15">15</option>
            <option value="21">21</option><option value="28">28</option><option value="30">30</option>
 		</select>
 		<select name="repeat_unit">
-           <option value="selminutes" selected="selected">minutes</option>
+           <option value="selminutes">minutes</option>
            <option value="selhours">hours</option>
-           <option value="seldays">days</option>
+           <option value="seldays" selected="selected">days</option>
         </select> again and add new sites automatically to indexer.<br />
         &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Sites that do not appear during a scheduled scan period will be excluded from search results.
     </dd>
diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java
index bbddf487a..22a15b001 100644
--- a/htroot/CrawlStartScanner_p.java
+++ b/htroot/CrawlStartScanner_p.java
@@ -155,6 +155,16 @@ public class CrawlStartScanner_p
                         Collection<InetAddress> scanbase = new ArrayList<InetAddress>();
                         if (host.length() > 0) {
                             ip = Domains.dnsResolve(host); if (ip != null) scanbase.add(ip);
+                            if (host.startsWith("ftp.") || host.startsWith("www.") ||
+                                host.startsWith("blog.") || host.startsWith("wiki.") ||
+                                host.startsWith("forum.") || host.startsWith("forums.") ||
+                                host.startsWith("events.") || host.startsWith("bugs.")) {
+                                host = host.substring(host.indexOf('.') + 1);
+                                if (!hostSet.contains(host)) {
+                                    ip = Domains.dnsResolve(host);
+                                    if (ip != null) scanbase.add(ip);
+                                }
+                            }
                             if (scanftp && !hostSet.contains("ftp." + host)) {
                                 ip = Domains.dnsResolve("ftp." + host);
                                 if (ip != null) scanbase.add(ip);