- added information about granted access - enhanced servlet design - added submit-feedback (because it is a long-running task) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7372 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
14 years ago
parent
acab6801d9
commit
99a7fe87f9
@ -1,67 +0,0 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Intranet Crawl Start</title>
|
||||
#%env/templates/metas.template%#
|
||||
<script type="text/javascript">
|
||||
<!--
|
||||
function setall(name) {
|
||||
var selectForm = document.forms.namedItem(name);
|
||||
var count = selectForm.elements["num"].value;
|
||||
for (i = 0; i < count; i++) {
|
||||
if (selectForm.elements["item_" + i] == null) continue;
|
||||
selectForm.elements["item_" + i].checked = !selectForm.elements["item_" + i].checked;
|
||||
}
|
||||
}
|
||||
-->
|
||||
</script>
|
||||
<script type="text/javascript" src="/js/sorttable.js"></script>
|
||||
</head>
|
||||
<body id="IndexCreate">
|
||||
#%env/templates/header.template%#
|
||||
#%env/templates/submenuIndexCreate.template%#
|
||||
<h2>Intranet Crawl Start</h2>
|
||||
<p>
|
||||
When an index domain is configured to contain intranet links,
|
||||
the intranet may be scanned for available servers.
|
||||
Please select below the servers in your intranet that you want to fetch into the search index.
|
||||
</p>
|
||||
|
||||
#(notintranet)#::
|
||||
<p class="error">
|
||||
This network definition does not allow intranet links.
|
||||
A list of intranet servers is only available if you confiugure YaCy to index intranet targets.
|
||||
To do so, open the <a href="ConfigBasic.html">Basic Configuration</a> servlet and select the 'Intranet Indexing' use case.
|
||||
</p>
|
||||
#(/notintranet)#
|
||||
|
||||
#(servertable)#::
|
||||
<form id="servertable" name="servertable" action="CrawlStartIntranet_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8" ><fieldset>
|
||||
<legend><label for="servertable">Available Intranet Server</label></legend>
|
||||
<table class="sortable" border="0" cellpadding="2" cellspacing="1">
|
||||
<tr class="TableHeader" valign="bottom">
|
||||
<td><input type="checkbox" name="allswitch" onclick="setall(this.form.name)" /></td>
|
||||
<td>IP</td>
|
||||
<td>URL</td>
|
||||
<td>Process</td>
|
||||
</tr>
|
||||
#{list}#
|
||||
<tr class="TableCellLight">
|
||||
<td align="left"><input type="checkbox" name="item_#[count]#" value="mark_#[pk]#" /></td>
|
||||
<td><a href="#[url]#">#[ip]#</a></td>
|
||||
<td><a href="#[url]#">#[url]#</a></td>
|
||||
#(process)#<td class="error">not in index</td>::<td class="commit">indexed</td>#(/process)#
|
||||
</tr>
|
||||
#{/list}#
|
||||
</table>
|
||||
<p>
|
||||
<input type="hidden" name="num" value="#[num]#" />
|
||||
<input type="submit" name="crawl" value="Add Selected Servers to Crawler" />
|
||||
</p>
|
||||
</fieldset></form>
|
||||
#(/servertable)#
|
||||
|
||||
|
||||
#%env/templates/footer.template%#
|
||||
</body>
|
||||
</html>
|
||||
@ -1,122 +0,0 @@
|
||||
/**
|
||||
* CrawlStartIntranet_p
|
||||
* Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 28.10.2010 at http://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
import net.yacy.cora.protocol.Domains;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.protocol.Scanner;
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
|
||||
import de.anomic.data.WorkTables;
|
||||
import de.anomic.search.Switchboard;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
|
||||
public class CrawlStartIntranet_p {
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
final Switchboard sb = (Switchboard)env;
|
||||
|
||||
prop.put("notintranet", 0);
|
||||
prop.put("servertable", 0);
|
||||
|
||||
// check if there is a intranet configuration
|
||||
if (!sb.isIntranetMode()) {
|
||||
prop.put("notintranet", 1);
|
||||
return prop;
|
||||
}
|
||||
|
||||
// if there are no intranet addresses known, scan the net
|
||||
if (Scanner.intranetURLs.size() == 0) {
|
||||
Scanner scanner = new Scanner(100, 10);
|
||||
scanner.addFTP(false);
|
||||
scanner.addHTTP(false);
|
||||
scanner.addHTTPS(false);
|
||||
scanner.addSMB(false);
|
||||
scanner.start();
|
||||
scanner.terminate();
|
||||
DigestURI url;
|
||||
for (MultiProtocolURI service: scanner.services()) {
|
||||
url = new DigestURI(service);
|
||||
Scanner.intranetURLs.put(url.hash(), url);
|
||||
}
|
||||
}
|
||||
|
||||
// check crawl request
|
||||
if (post != null && post.containsKey("crawl")) {
|
||||
for (Map.Entry<String, String> entry: post.entrySet()) {
|
||||
if (entry.getValue().startsWith("mark_")) {
|
||||
byte [] pk = entry.getValue().substring(5).getBytes();
|
||||
DigestURI url = Scanner.intranetURLs.get(pk);
|
||||
if (url != null) {
|
||||
String path = "/Crawler_p.html?createBookmark=off&xsstopw=off&crawlingDomMaxPages=10000&intention=&range=domain&indexMedia=on&recrawl=nodoubles&xdstopw=off&storeHTCache=on&sitemapURL=&repeat_time=7&crawlingQ=on&cachePolicy=iffresh&indexText=on&crawlingMode=url&mustnotmatch=&crawlingDomFilterDepth=1&crawlingDomFilterCheck=off&crawlingstart=Start%20New%20Crawl&xpstopw=off&repeat_unit=seldays&crawlingDepth=99";
|
||||
path += "&crawlingURL=" + url.toNormalform(true, false);
|
||||
WorkTables.execAPICall("localhost", (int) sb.getConfigLong("port", 8080), sb.getConfig("adminAccountBase64MD5", ""), path, pk);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// show server table
|
||||
prop.put("servertable", 1);
|
||||
int i = 0;
|
||||
String urlString;
|
||||
for (final DigestURI url: Scanner.intranetURLs.values()) {
|
||||
urlString = url.toNormalform(true, false);
|
||||
prop.put("servertable_list_" + i + "_pk", new String(url.hash()));
|
||||
prop.put("servertable_list_" + i + "_count", i);
|
||||
prop.putHTML("servertable_list_" + i + "_ip", Domains.dnsResolve(url.getHost()).getHostAddress());
|
||||
prop.putHTML("servertable_list_" + i + "_url", urlString);
|
||||
prop.put("servertable_list_" + i + "_process", inIndex(sb, urlString) == null ? 0 : 1);
|
||||
i++;
|
||||
}
|
||||
prop.put("servertable_list", i);
|
||||
prop.put("servertable_num", i);
|
||||
return prop;
|
||||
}
|
||||
|
||||
private static byte[] inIndex(Switchboard sb, String url) {
|
||||
Iterator<Tables.Row> i;
|
||||
try {
|
||||
i = sb.tables.iterator(WorkTables.TABLE_API_NAME);
|
||||
Tables.Row row;
|
||||
String comment;
|
||||
while (i.hasNext()) {
|
||||
row = i.next();
|
||||
comment = new String(row.get(WorkTables.TABLE_API_COL_COMMENT));
|
||||
if (comment.contains(url)) return row.getPK();
|
||||
}
|
||||
return null;
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue