boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced speed and usability of network scanner servlet

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7373 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 99a7fe87f9
commit 358feeeb39
4 changed files with 52 additions and 37 deletions
Show Stats Download Patch File Download Diff File

23
htroot/CrawlStartScanner_p.html
Unescape View File

@ -13,6 +13,11 @@
selectForm.elements["item_" + i].checked = !selectForm.elements["item_" + i].checked; selectForm.elements["item_" + i].checked = !selectForm.elements["item_" + i].checked;
} }
} }
function disableSubmit() {
document.getElementById('scanipSubmit').disabled=true;
document.getElementById('scanhostSubmit').disabled=true;
document.getElementById('scanintranetSubmit').disabled=true;
}
--> -->
</script> </script>
<script type="text/javascript" src="/js/sorttable.js"></script> <script type="text/javascript" src="/js/sorttable.js"></script>
@ -38,25 +43,27 @@
#(enterrange)#:: #(enterrange)#::
<div><table border="0"><tr valign="top"> <div><table border="0"><tr valign="top">
<td width="270"><form onSubmit="document.getElementById('scanipinfo').innerHTML='Please wait...'" action="CrawlStartScanner_p.html" method="get"> <td width="270"><form onSubmit="document.getElementById('scanipinfo').innerHTML='Please wait...'; disableSubmit();" action="CrawlStartScanner_p.html" method="get">
<fieldset height="30"> <fieldset height="30">
<legend><label for="servertable">Scan with given IP range</label></legend><br> <legend><label for="servertable">Scan with given IP range</label></legend><br>
<input type="text" name="ip4-0" value="#[ip4-0]#" size="3" maxlength="3" />. <input type="text" name="ip4-0" value="#[ip4-0]#" size="3" maxlength="3" />.
<input type="text" name="ip4-1" value="#[ip4-1]#" size="3" maxlength="3" />. <input type="text" name="ip4-1" value="#[ip4-1]#" size="3" maxlength="3" />.
<input type="text" name="ip4-2" value="#[ip4-2]#" size="3" maxlength="3" />.[1-254] <input type="text" name="ip4-2" value="#[ip4-2]#" size="3" maxlength="3" />.[1-254]
<input type="submit" name="scanip" value="Scan"/><br><br><div class="error" id="scanipinfo"></div> <input type="hidden" name="scanip" value=""/>
<input type="submit" id="scanipSubmit" name="scanipSubmit" value="Scan"/><br><br><div class="info" id="scanipinfo" style="text-decoration: blink;"></div>
</fieldset></form></td> </fieldset></form></td>
<td width="270"><form onSubmit="document.getElementById('scanhostinfo').innerHTML='Please wait...'" action="CrawlStartScanner_p.html" method="get"> <td width="270"><form onSubmit="document.getElementById('scanhostinfo').innerHTML='Please wait...'; disableSubmit();" action="CrawlStartScanner_p.html" method="get">
<fieldset> <fieldset>
<legend><label for="servertable">Scan range with given host name</label></legend><br> <legend><label for="servertable">Scan range with given host name</label></legend><br>
<input type="text" name="host" value="#[host]#" size="28" maxlength="60" /> <input type="text" name="scanhost" value="#[host]#" size="28" maxlength="60" />
<input type="submit" name="scanhost" value="Scan" /><br><br><div class="error" id="scanhostinfo"></div> <input type="submit" id="scanhostSubmit" name="scanhostSubmit" value="Scan" /><br><br><div class="info" id="scanhostinfo" style="text-decoration: blink;"></div>
</fieldset></form></td> </fieldset></form></td>
<td><form onSubmit="document.getElementById('scanintranetinfo').innerHTML='Please wait...'" action="CrawlStartScanner_p.html" method="get"> <td><form onSubmit="document.getElementById('scanintranetinfo').innerHTML='Please wait...'; disableSubmit();" action="CrawlStartScanner_p.html" method="get">
<fieldset> <fieldset>
<legend><label for="servertable">Full Intranet Scan</label></legend> <legend><label for="servertable">Full Intranet Scan</label></legend>
#(intranetHint)#::<div class="warning">Do not use intranet scan results, you are not in an intranet environment!</div>#(/intranetHint)# #(intranetHint)#::<div class="info">Do not use intranet scan results, you are not in an intranet environment!</div>#(/intranetHint)#
#[intranethosts]#&nbsp;<input type="submit" name="scanintranet" value="Scan" /><div class="error" id="scanintranetinfo"></div> <input type="hidden" name="scanintranet" value=""/>
#[intranethosts]#&nbsp;<input type="submit" id="scanintranetSubmit" name="scanintranetSubmit" value="Scan" /><div class="info" id="scanintranetinfo" style="text-decoration: blink;"></div>
</fieldset></form></td> </fieldset></form></td>
</tr></table></div> </tr></table></div>
#(/enterrange)# #(/enterrange)#

56
htroot/CrawlStartScanner_p.java
Unescape View File

@ -54,9 +54,13 @@ public class CrawlStartScanner_p {
prop.put("noserverdetected", 0); prop.put("noserverdetected", 0);
prop.put("enterrange", 0); prop.put("enterrange", 0);
prop.put("servertable", 0); prop.put("servertable", 0);
prop.put("enterrange_host", "");
// make a comment cache
Map<byte[], String> apiCommentCache = commentCache(sb);
addSelectIPRange(sb, prop); addSelectIPRange(sb, prop);
addScantable(sb, prop); addScantable(apiCommentCache, prop);
// case: no query part of the request; ask for input // case: no query part of the request; ask for input
if (post == null) { if (post == null) {
@ -71,7 +75,15 @@ public class CrawlStartScanner_p {
if (post.containsKey("scanip")) { if (post.containsKey("scanip")) {
ia = InetAddress.getByAddress(new byte[]{(byte) post.getInt("ip4-0", 0), (byte) post.getInt("ip4-1", 0), (byte) post.getInt("ip4-2", 0), (byte) post.getInt("ip4-3", 0)}); ia = InetAddress.getByAddress(new byte[]{(byte) post.getInt("ip4-0", 0), (byte) post.getInt("ip4-1", 0), (byte) post.getInt("ip4-2", 0), (byte) post.getInt("ip4-3", 0)});
} else { } else {
ia = InetAddress.getByName(post.get("host", "")); String host = post.get("scanhost", "");
if (host.startsWith("http://")) host = host.substring(7);
if (host.startsWith("https://")) host = host.substring(8);
if (host.startsWith("ftp://")) host = host.substring(6);
if (host.startsWith("smb://")) host = host.substring(6);
int p = host.indexOf('/');
if (p >= 0) host = host.substring(0, p);
ia = InetAddress.getByName(host);
prop.put("enterrange_host", host);
} }
addSelectIPRange(ia, prop); addSelectIPRange(ia, prop);
Scanner scanner = new Scanner(ia, 100, sb.isIntranetMode() ? 100 : 3000); Scanner scanner = new Scanner(ia, 100, sb.isIntranetMode() ? 100 : 3000);
@ -81,8 +93,8 @@ public class CrawlStartScanner_p {
scanner.addSMB(false); scanner.addSMB(false);
scanner.start(); scanner.start();
scanner.terminate(); scanner.terminate();
enlargeScancache(sb, scanner); enlargeScancache(apiCommentCache, scanner);
addScantable(sb, prop); addScantable(apiCommentCache, prop);
} catch (UnknownHostException e) {} } catch (UnknownHostException e) {}
} }
@ -94,8 +106,8 @@ public class CrawlStartScanner_p {
scanner.addSMB(false); scanner.addSMB(false);
scanner.start(); scanner.start();
scanner.terminate(); scanner.terminate();
enlargeScancache(sb, scanner); enlargeScancache(apiCommentCache, scanner);
addScantable(sb, prop); addScantable(apiCommentCache, prop);
} }
// check crawl request // check crawl request
@ -145,13 +157,12 @@ public class CrawlStartScanner_p {
private static void addSelectIPRange(InetAddress ip, serverObjects prop) { private static void addSelectIPRange(InetAddress ip, serverObjects prop) {
prop.put("enterrange", 1); prop.put("enterrange", 1);
byte[] address = ip.getAddress(); byte[] address = ip.getAddress();
prop.put("enterrange_host", "");
prop.put("enterrange_ip4-0", 0xff & address[0]); prop.put("enterrange_ip4-0", 0xff & address[0]);
prop.put("enterrange_ip4-1", 0xff & address[1]); prop.put("enterrange_ip4-1", 0xff & address[1]);
prop.put("enterrange_ip4-2", 0xff & address[2]); prop.put("enterrange_ip4-2", 0xff & address[2]);
} }
private static void addScantable(Switchboard sb, serverObjects prop) { private static void addScantable(Map<byte[], String> commentCache, serverObjects prop) {
if (Scanner.scancache.size() > 0) { if (Scanner.scancache.size() > 0) {
// show scancache table // show scancache table
prop.put("servertable", 1); prop.put("servertable", 1);
@ -172,8 +183,8 @@ public class CrawlStartScanner_p {
prop.put("servertable_list_" + i + "_accessEmpty", host.getValue() == Access.empty ? 1 : 0); prop.put("servertable_list_" + i + "_accessEmpty", host.getValue() == Access.empty ? 1 : 0);
prop.put("servertable_list_" + i + "_accessGranted", host.getValue() == Access.granted ? 1 : 0); prop.put("servertable_list_" + i + "_accessGranted", host.getValue() == Access.granted ? 1 : 0);
prop.put("servertable_list_" + i + "_accessDenied", host.getValue() == Access.denied ? 1 : 0); prop.put("servertable_list_" + i + "_accessDenied", host.getValue() == Access.denied ? 1 : 0);
prop.put("servertable_list_" + i + "_process", inIndex(sb, urlString) == null ? 0 : 1); prop.put("servertable_list_" + i + "_process", inIndex(commentCache, urlString) == null ? 0 : 1);
prop.put("servertable_list_" + i + "_preselected", interesting(sb, u, host.getValue()) ? 1 : 0); prop.put("servertable_list_" + i + "_preselected", interesting(commentCache, u, host.getValue()) ? 1 : 0);
i++; i++;
} }
prop.put("servertable_list", i); prop.put("servertable_list", i);
@ -186,7 +197,7 @@ public class CrawlStartScanner_p {
} }
} }
private static void enlargeScancache(Switchboard sb, Scanner scanner) { private static void enlargeScancache(Map<byte[], String> commentCache, Scanner scanner) {
if (Scanner.scancache == null) { if (Scanner.scancache == null) {
Scanner.scancache = scanner.services(); Scanner.scancache = scanner.services();
return; return;
@ -195,31 +206,36 @@ public class CrawlStartScanner_p {
Map.Entry<MultiProtocolURI, Access> entry; Map.Entry<MultiProtocolURI, Access> entry;
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
if (!interesting(sb, entry.getKey(), entry.getValue())) i.remove(); if (!interesting(commentCache, entry.getKey(), entry.getValue())) i.remove();
} }
Scanner.scancache.putAll(scanner.services()); Scanner.scancache.putAll(scanner.services());
} }
private static boolean interesting(Switchboard sb, MultiProtocolURI uri, Access access) { private static boolean interesting(Map<byte[], String> commentCache, MultiProtocolURI uri, Access access) {
return inIndex(sb, uri.toNormalform(true, false)) == null && access == Access.granted && (uri.getProtocol().equals("smb") || uri.getProtocol().equals("ftp")); return inIndex(commentCache, uri.toNormalform(true, false)) == null && access == Access.granted && (uri.getProtocol().equals("smb") || uri.getProtocol().equals("ftp"));
} }
private static byte[] inIndex(Switchboard sb, String url) { private static byte[] inIndex(Map<byte[], String> commentCache, String url) {
for (Map.Entry<byte[], String> comment: commentCache.entrySet()) {
if (comment.getValue().contains(url)) return comment.getKey();
}
return null;
}
private static Map<byte[], String> commentCache(Switchboard sb) {
Map<byte[], String> comments = new TreeMap<byte[], String>(Base64Order.enhancedCoder);
Iterator<Tables.Row> i; Iterator<Tables.Row> i;
try { try {
i = sb.tables.iterator(WorkTables.TABLE_API_NAME); i = sb.tables.iterator(WorkTables.TABLE_API_NAME);
Tables.Row row; Tables.Row row;
String comment;
while (i.hasNext()) { while (i.hasNext()) {
row = i.next(); row = i.next();
comment = new String(row.get(WorkTables.TABLE_API_COL_COMMENT)); comments.put(row.getPK(), new String(row.get(WorkTables.TABLE_API_COL_COMMENT)));
if (comment.contains(url)) return row.getPK();
} }
return null;
} catch (IOException e) { } catch (IOException e) {
Log.logException(e); Log.logException(e);
return null;
} }
return comments;
} }
} }

4
skins/generic_pd.css
Unescape View File

@ -190,10 +190,6 @@ div.ProgressBarFill {
/* Copyright info */ /* Copyright info */
.info {
border-top:1px solid gray;
}
div#api { div#api {
position:absolute; position:absolute;
top:3px; top:3px;

4
skins/pdblue.css
Unescape View File

@ -172,10 +172,6 @@ div.ProgressBarFill {
/* Copyright info */ /* Copyright info */
.info {
border-top:1px solid gray;
}
div#api { div#api {
position:absolute; position:absolute;
top:3px; top:3px;

Write Preview
Loading…
Cancel
Save