/**
* CrawlStartIntranet_p
* Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 28.10.2010 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see .
*/
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.Scanner;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import de.anomic.data.WorkTables;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class CrawlStartIntranet_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard)env;
prop.put("notintranet", 0);
prop.put("servertable", 0);
// check if there is a intranet configuration
if (!sb.isIntranetMode()) {
prop.put("notintranet", 1);
return prop;
}
// if there are no intranet addresses known, scan the net
if (sb.intranetURLs.size() == 0) {
Scanner scanner = new Scanner(100, 10);
scanner.addFTP(false);
scanner.addHTTP(false);
scanner.addHTTPS(false);
scanner.addSMB(false);
scanner.start();
scanner.terminate();
DigestURI url;
for (MultiProtocolURI service: scanner.services()) {
url = new DigestURI(service);
sb.intranetURLs.put(url.hash(), url);
}
}
// check crawl request
if (post != null && post.containsKey("crawl")) {
for (Map.Entry entry: post.entrySet()) {
if (entry.getValue().startsWith("mark_")) {
byte [] pk = entry.getValue().substring(5).getBytes();
DigestURI url = sb.intranetURLs.get(pk);
if (url != null) {
String path = "/Crawler_p.html?createBookmark=off&xsstopw=off&crawlingDomMaxPages=10000&intention=&range=domain&indexMedia=on&recrawl=nodoubles&xdstopw=off&storeHTCache=on&sitemapURL=&repeat_time=7&crawlingQ=on&cachePolicy=iffresh&indexText=on&crawlingMode=url&mustnotmatch=&crawlingDomFilterDepth=1&crawlingDomFilterCheck=off&crawlingstart=Start%20New%20Crawl&xpstopw=off&repeat_unit=seldays&crawlingDepth=99";
path += "&crawlingURL=" + url.toNormalform(true, false);
WorkTables.execAPICall("localhost", (int) sb.getConfigLong("port", 8080), sb.getConfig("adminAccountBase64MD5", ""), path, pk);
}
}
}
}
// show server table
prop.put("servertable", 1);
int i = 0;
String urlString;
for (final DigestURI url: sb.intranetURLs.values()) {
urlString = url.toNormalform(true, false);
prop.put("servertable_list_" + i + "_pk", new String(url.hash()));
prop.put("servertable_list_" + i + "_count", i);
prop.putHTML("servertable_list_" + i + "_ip", Domains.dnsResolve(url.getHost()).getHostAddress());
prop.putHTML("servertable_list_" + i + "_url", urlString);
prop.put("servertable_list_" + i + "_process", inIndex(sb, urlString) == null ? 0 : 1);
i++;
}
prop.put("servertable_list", i);
prop.put("servertable_num", i);
return prop;
}
private static byte[] inIndex(Switchboard sb, String url) {
Iterator i;
try {
i = sb.tables.iterator(WorkTables.TABLE_API_NAME);
Tables.Row row;
String comment;
while (i.hasNext()) {
row = i.next();
comment = new String(row.get(WorkTables.TABLE_API_COL_COMMENT));
if (comment.contains(url)) return row.getPK();
}
return null;
} catch (IOException e) {
Log.logException(e);
return null;
}
}
}