From 358feeeb391e0452a90fc488a16e71763deeec99 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 14 Dec 2010 12:12:13 +0000 Subject: [PATCH] enhanced speed and usability of network scanner servlet git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7373 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/CrawlStartScanner_p.html | 23 ++++++++----- htroot/CrawlStartScanner_p.java | 58 +++++++++++++++++++++------------ skins/generic_pd.css | 4 --- skins/pdblue.css | 4 --- 4 files changed, 52 insertions(+), 37 deletions(-) diff --git a/htroot/CrawlStartScanner_p.html b/htroot/CrawlStartScanner_p.html index 239615401..4550b535a 100644 --- a/htroot/CrawlStartScanner_p.html +++ b/htroot/CrawlStartScanner_p.html @@ -13,6 +13,11 @@ selectForm.elements["item_" + i].checked = !selectForm.elements["item_" + i].checked; } } + function disableSubmit() { + document.getElementById('scanipSubmit').disabled=true; + document.getElementById('scanhostSubmit').disabled=true; + document.getElementById('scanintranetSubmit').disabled=true; + } --> @@ -38,25 +43,27 @@ #(enterrange)#::
- - -
+

. . .[1-254] -

+ +

+

- -

+ +

+
- #(intranetHint)#::
Do not use intranet scan results, you are not in an intranet environment!
#(/intranetHint)# - #[intranethosts]# 
+ #(intranetHint)#::
Do not use intranet scan results, you are not in an intranet environment!
#(/intranetHint)# + + #[intranethosts]# 
#(/enterrange)# diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 61b54ac93..686bdf9ed 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -54,9 +54,13 @@ public class CrawlStartScanner_p { prop.put("noserverdetected", 0); prop.put("enterrange", 0); prop.put("servertable", 0); - + prop.put("enterrange_host", ""); + + // make a comment cache + Map apiCommentCache = commentCache(sb); + addSelectIPRange(sb, prop); - addScantable(sb, prop); + addScantable(apiCommentCache, prop); // case: no query part of the request; ask for input if (post == null) { @@ -71,7 +75,15 @@ public class CrawlStartScanner_p { if (post.containsKey("scanip")) { ia = InetAddress.getByAddress(new byte[]{(byte) post.getInt("ip4-0", 0), (byte) post.getInt("ip4-1", 0), (byte) post.getInt("ip4-2", 0), (byte) post.getInt("ip4-3", 0)}); } else { - ia = InetAddress.getByName(post.get("host", "")); + String host = post.get("scanhost", ""); + if (host.startsWith("http://")) host = host.substring(7); + if (host.startsWith("https://")) host = host.substring(8); + if (host.startsWith("ftp://")) host = host.substring(6); + if (host.startsWith("smb://")) host = host.substring(6); + int p = host.indexOf('/'); + if (p >= 0) host = host.substring(0, p); + ia = InetAddress.getByName(host); + prop.put("enterrange_host", host); } addSelectIPRange(ia, prop); Scanner scanner = new Scanner(ia, 100, sb.isIntranetMode() ? 100 : 3000); @@ -81,8 +93,8 @@ public class CrawlStartScanner_p { scanner.addSMB(false); scanner.start(); scanner.terminate(); - enlargeScancache(sb, scanner); - addScantable(sb, prop); + enlargeScancache(apiCommentCache, scanner); + addScantable(apiCommentCache, prop); } catch (UnknownHostException e) {} } @@ -94,8 +106,8 @@ public class CrawlStartScanner_p { scanner.addSMB(false); scanner.start(); scanner.terminate(); - enlargeScancache(sb, scanner); - addScantable(sb, prop); + enlargeScancache(apiCommentCache, scanner); + addScantable(apiCommentCache, prop); } // check crawl request @@ -145,13 +157,12 @@ public class CrawlStartScanner_p { private static void addSelectIPRange(InetAddress ip, serverObjects prop) { prop.put("enterrange", 1); byte[] address = ip.getAddress(); - prop.put("enterrange_host", ""); prop.put("enterrange_ip4-0", 0xff & address[0]); prop.put("enterrange_ip4-1", 0xff & address[1]); prop.put("enterrange_ip4-2", 0xff & address[2]); } - private static void addScantable(Switchboard sb, serverObjects prop) { + private static void addScantable(Map commentCache, serverObjects prop) { if (Scanner.scancache.size() > 0) { // show scancache table prop.put("servertable", 1); @@ -172,8 +183,8 @@ public class CrawlStartScanner_p { prop.put("servertable_list_" + i + "_accessEmpty", host.getValue() == Access.empty ? 1 : 0); prop.put("servertable_list_" + i + "_accessGranted", host.getValue() == Access.granted ? 1 : 0); prop.put("servertable_list_" + i + "_accessDenied", host.getValue() == Access.denied ? 1 : 0); - prop.put("servertable_list_" + i + "_process", inIndex(sb, urlString) == null ? 0 : 1); - prop.put("servertable_list_" + i + "_preselected", interesting(sb, u, host.getValue()) ? 1 : 0); + prop.put("servertable_list_" + i + "_process", inIndex(commentCache, urlString) == null ? 0 : 1); + prop.put("servertable_list_" + i + "_preselected", interesting(commentCache, u, host.getValue()) ? 1 : 0); i++; } prop.put("servertable_list", i); @@ -186,7 +197,7 @@ public class CrawlStartScanner_p { } } - private static void enlargeScancache(Switchboard sb, Scanner scanner) { + private static void enlargeScancache(Map commentCache, Scanner scanner) { if (Scanner.scancache == null) { Scanner.scancache = scanner.services(); return; @@ -195,31 +206,36 @@ public class CrawlStartScanner_p { Map.Entry entry; while (i.hasNext()) { entry = i.next(); - if (!interesting(sb, entry.getKey(), entry.getValue())) i.remove(); + if (!interesting(commentCache, entry.getKey(), entry.getValue())) i.remove(); } Scanner.scancache.putAll(scanner.services()); } - private static boolean interesting(Switchboard sb, MultiProtocolURI uri, Access access) { - return inIndex(sb, uri.toNormalform(true, false)) == null && access == Access.granted && (uri.getProtocol().equals("smb") || uri.getProtocol().equals("ftp")); + private static boolean interesting(Map commentCache, MultiProtocolURI uri, Access access) { + return inIndex(commentCache, uri.toNormalform(true, false)) == null && access == Access.granted && (uri.getProtocol().equals("smb") || uri.getProtocol().equals("ftp")); + } + + private static byte[] inIndex(Map commentCache, String url) { + for (Map.Entry comment: commentCache.entrySet()) { + if (comment.getValue().contains(url)) return comment.getKey(); + } + return null; } - private static byte[] inIndex(Switchboard sb, String url) { + private static Map commentCache(Switchboard sb) { + Map comments = new TreeMap(Base64Order.enhancedCoder); Iterator i; try { i = sb.tables.iterator(WorkTables.TABLE_API_NAME); Tables.Row row; - String comment; while (i.hasNext()) { row = i.next(); - comment = new String(row.get(WorkTables.TABLE_API_COL_COMMENT)); - if (comment.contains(url)) return row.getPK(); + comments.put(row.getPK(), new String(row.get(WorkTables.TABLE_API_COL_COMMENT))); } - return null; } catch (IOException e) { Log.logException(e); - return null; } + return comments; } } diff --git a/skins/generic_pd.css b/skins/generic_pd.css index 839c7aa00..c5417d849 100644 --- a/skins/generic_pd.css +++ b/skins/generic_pd.css @@ -190,10 +190,6 @@ div.ProgressBarFill { /* Copyright info */ -.info { - border-top:1px solid gray; -} - div#api { position:absolute; top:3px; diff --git a/skins/pdblue.css b/skins/pdblue.css index aaca85e23..b743b581b 100644 --- a/skins/pdblue.css +++ b/skins/pdblue.css @@ -172,10 +172,6 @@ div.ProgressBarFill { /* Copyright info */ -.info { - border-top:1px solid gray; -} - div#api { position:absolute; top:3px;