boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added a servlet that has been removed in SVN 4881; this servlet is now splitted and will be used for a simple crawl start and a remote crawl monitor (not yet integrated into the interface)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6582 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 938e806182
commit 34354cf9b2
5 changed files with 330 additions and 1 deletions
Show Stats Download Patch File Download Diff File

59
htroot/CrawlMonitorRemoteStart.html
Unescape View File

@ -0,0 +1,59 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Monitor for remotely started global crawls</title>
#%env/templates/metas.template%#
<script type="text/javascript" src="/js/ajax.js"></script>
<script type="text/javascript" src="/js/IndexCreate.js"></script>
</head>
<body id="IndexCreate">
#%env/templates/header.template%#
#%env/templates/submenuIndexCreate.template%#
<h2>Recently started remote crawls in progress</h2>
<p><strong>Remote crawl start points, crawl is ongoing</strong></p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader">
<td><strong>Start Time</strong></td>
<td><strong>Peer Name</strong></td>
<td><strong>Start URL</strong></td>
<td><strong>Intention/Description</strong></td>
<td><strong>Depth</strong></td>
<td><strong>Accept '?' URLs</strong></td>
</tr>
#{otherCrawlStartInProgress}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#" >
<td>#[cre]#</td>
<td>#[peername]#</td>
<td><a href="#[startURL]#">#[startURL]#</a></td>
<td>#[intention]#</td>
<td>#[generalDepth]#</td>
<td>#(crawlingQ)#no::yes#(/crawlingQ)#</td>
</tr>
#{/otherCrawlStartInProgress}#
</table>
<p><strong>Remote crawl start points, finished:</strong></p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader">
<td><strong>Start Time</strong></td>
<td><strong>Peer Name</strong></td>
<td><strong>Start URL</strong></td>
<td><strong>Intention/Description</strong></td>
<td><strong>Depth</strong></td>
<td><strong>Accept '?' URLs</strong></td>
</tr>
#{otherCrawlStartFinished}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#" >
<td>#[cre]#</td>
<td>#[peername]#</td>
<td><a href="#[startURL]#">#[startURL]#</a></td>
<td>#[intention]#</td>
<td>#[generalDepth]#</td>
<td>#(crawlingQ)#no::yes#(/crawlingQ)#</td>
</tr>
#{/otherCrawlStartFinished}#
</table>
#%env/templates/footer.template%#
</body>
</html>

97
htroot/CrawlMonitorRemoteStart.java
Unescape View File

@ -0,0 +1,97 @@
// CrawlMonitorRemoteStart.java
// (C) 2004 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 02.12.2004 as IndexCreate_p.java on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.Iterator;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
public class CrawlMonitorRemoteStart {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
Switchboard sb = (Switchboard) env;
serverObjects prop = new serverObjects();
boolean dark = true;
// create other peer crawl table using YaCyNews
Iterator<yacyNewsRecord> recordIterator = sb.peers.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
int showedCrawl = 0;
yacyNewsRecord record;
yacySeed peer;
String peername;
while (recordIterator.hasNext()) {
record = recordIterator.next();
if (record == null) continue;
if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
peer = sb.peers.get(record.originator());
if (peer == null) peername = record.originator(); else peername = peer.getName();
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_dark", dark ? "1" : "0");
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_cre", record.created().toString());
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_peername", peername);
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_startURL", record.attributes().get("startURL").toString());
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_intention", record.attributes().get("intention").toString());
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_generalDepth", record.attributes().get("generalDepth"));
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_crawlingQ", (record.attributes().get("crawlingQ").equals("true")) ? "1" : "0");
showedCrawl++;
if (showedCrawl > 20) break;
}
}
prop.put("otherCrawlStartInProgress", showedCrawl);
// finished remote crawls
recordIterator = sb.peers.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true);
showedCrawl = 0;
while (recordIterator.hasNext()) {
record = (yacyNewsRecord) recordIterator.next();
if (record == null) continue;
if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
peer = sb.peers.get(record.originator());
if (peer == null) peername = record.originator(); else peername = peer.getName();
prop.put("otherCrawlStartFinished_" + showedCrawl + "_dark", dark ? "1" : "0");
prop.put("otherCrawlStartFinished_" + showedCrawl + "_cre", record.created().toString());
prop.putHTML("otherCrawlStartFinished_" + showedCrawl + "_peername", peername);
prop.putHTML("otherCrawlStartFinished_" + showedCrawl + "_startURL", record.attributes().get("startURL").toString());
prop.put("otherCrawlStartFinished_" + showedCrawl + "_intention", record.attributes().get("intention").toString());
prop.put("otherCrawlStartFinished_" + showedCrawl + "_generalDepth", record.attributes().get("generalDepth"));
prop.put("otherCrawlStartFinished_" + showedCrawl + "_crawlingQ", (record.attributes().get("crawlingQ").equals("true")) ? "1" : "0");
showedCrawl++;
if (showedCrawl > 20) break;
}
}
prop.put("otherCrawlStartFinished", showedCrawl);
// return rewrite properties
return prop;
}
}

2
htroot/CrawlProfileEditor_p.java
Unescape View File

@ -138,7 +138,7 @@ public class CrawlProfileEditor_p {
} }
prop.put("profiles", count); prop.put("profiles", count);
selentry = sb.crawler.profilesActiveCrawls.getEntry(handle); selentry = sb.crawler.profilesActiveCrawls.getEntry(handle);
assert selentry.handle() != null; assert selentry == null || selentry.handle() != null;
// read post for change submit // read post for change submit
if ((post != null) && (selentry != null)) { if ((post != null) && (selentry != null)) {
if (post.containsKey("submit")) { if (post.containsKey("submit")) {

71
htroot/IndexCreateDomainCrawl_p.html
Unescape View File

@ -0,0 +1,71 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Index Creation with a Web Crawl for a Single Domain</title>
#%env/templates/metas.template%#
<script type="text/javascript" src="/js/ajax.js"></script>
<script type="text/javascript" src="/js/IndexCreate.js"></script>
</head>
<body id="IndexCreate">
#%env/templates/header.template%#
#%env/templates/submenuIndexCreate.template%#
<h2>Easy Crawl Start</h2>
<p id="startCrawling">
<strong>Start Crawling Job:</strong>&nbsp;
You can define URLs as start points for Web page crawling and start crawling here.
"Crawling" means that YaCy will download the given web-site, extract all links in it
and then download the content behind these links.
This is repeated as long as specified under "Crawling Depth".
</p>
<form action="Crawler_p.html" method="post" enctype="multipart/form-data">
<input type="hidden" name="crawlingFilter" value=".*" />
<input type="hidden" name="crawlingIfOlderCheck" value="off" />
<input type="hidden" name="crawlingDomFilterCheck" value="off" />
<input type="hidden" name="crawlingDomMaxCheck" value="off" />
<input type="hidden" name="crawlingQ" value="off" />
<input type="hidden" name="storeHTCache" value="on" />
<input type="hidden" name="indexText" value="on" />
<input type="hidden" name="indexMedia" value="on" />
<input type="hidden" name="crawlOrder" value="on" />
<input type="hidden" name="intention" value="simple web crawl" />
<input type="hidden" name="xsstopw" value="off" />
<table border="0" cellpadding="5" cellspacing="1">
<tr class="TableHeader">
<td><strong>Attribut</strong></td>
<td><strong>Value</strong></td>
<td><strong>Description</strong></td>
</tr>
<tr valign="top" class="TableCellSummary">
<td>Starting Point:</td>
<td>
<input name="crawlingURL" type="text" size="41" maxlength="256" value="http://" onkeypress="changed()" />
<span id="robotsOK"></span><br />
<span id="title"><br/></span>
<img src="/env/grafics/empty.gif" name="ajax" alt="empty" />
</td>
<td>
Enter here the start url of the web crawl.
</td>
</tr>
<tr valign="top" class="TableCellLight">
<td><label for="crawlingDepth">Crawling Range</label>:</td>
<td>
<input type="radio" name="range" value="wide" checked="checked" />Wide: depth <input name="crawlingDepth" id="crawlingDepth" type="text" size="2" maxlength="2" value="#[crawlingDepth]#" />&nbsp;&nbsp;|&nbsp;&nbsp;
<input type="radio" name="range" value="domain" />Complete Domain
</td>
<td>
The range defines if the crawl shall consider a complete domain, or a wide crawl up to a specific depth.
</td>
</tr>
<tr valign="top" class="TableCellLight">
<td colspan="3"><input type="submit" name="crawlingstart" value="Start New Distributed Crawl (will be visible at other peers)" /></td>
</tr>
</table>
</form>
#%env/templates/footer.template%#
</body>
</html>

102
htroot/IndexCreateDomainCrawl_p.java
Unescape View File

@ -0,0 +1,102 @@
// IndexCreateDomainCrawl_p
// (C) 2004 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 02.12.2004 as IndexCreate_p.java on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.Iterator;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
public class IndexCreateDomainCrawl_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
Switchboard sb = (Switchboard) env;
serverObjects prop = new serverObjects();
// define visible variables
prop.put("proxyPrefetchDepth", env.getConfig("proxyPrefetchDepth", "0"));
prop.put("crawlingDepth", Math.min(3, env.getConfigLong("crawlingDepth", 0)));
prop.put("crawlingFilter", env.getConfig("crawlingFilter", "0"));
int crawlingIfOlder = (int) env.getConfigLong("crawlingIfOlder", -1);
prop.put("crawlingIfOlderCheck", (crawlingIfOlder == -1) ? "0" : "1");
prop.put("crawlingIfOlderUnitYearCheck", "0");
prop.put("crawlingIfOlderUnitMonthCheck", "0");
prop.put("crawlingIfOlderUnitDayCheck", "0");
prop.put("crawlingIfOlderUnitHourCheck", "0");
prop.put("crawlingIfOlderUnitMinuteCheck", "0");
if ((crawlingIfOlder == -1) || (crawlingIfOlder == Integer.MAX_VALUE)) {
prop.put("crawlingIfOlderNumber", "-1");
prop.put("crawlingIfOlderUnitYearCheck", "1");
} else if (crawlingIfOlder >= 60*24*365) {
prop.put("crawlingIfOlderNumber", Math.round((float)crawlingIfOlder / (float)(60*24*365)));
prop.put("crawlingIfOlderUnitYearCheck", "1");
} else if (crawlingIfOlder >= 60*24*30) {
prop.put("crawlingIfOlderNumber", Math.round((float)crawlingIfOlder / (float)(60*24*30)));
prop.put("crawlingIfOlderUnitMonthCheck", "1");
} else if (crawlingIfOlder >= 60*24) {
prop.put("crawlingIfOlderNumber", Math.round((float)crawlingIfOlder / (float)(60*24)));
prop.put("crawlingIfOlderUnitDayCheck", "1");
} else if (crawlingIfOlder >= 60) {
prop.put("crawlingIfOlderNumber", Math.round(crawlingIfOlder / 60f));
prop.put("crawlingIfOlderUnitHourCheck", "1");
} else {
prop.put("crawlingIfOlderNumber", crawlingIfOlder);
prop.put("crawlingIfOlderUnitMinuteCheck", "1");
}
int crawlingDomFilterDepth = (int) env.getConfigLong("crawlingDomFilterDepth", -1);
prop.put("crawlingDomFilterCheck", (crawlingDomFilterDepth == -1) ? "0" : "1");
prop.put("crawlingDomFilterDepth", (crawlingDomFilterDepth == -1) ? 1 : crawlingDomFilterDepth);
int crawlingDomMaxPages = (int) env.getConfigLong("crawlingDomMaxPages", -1);
prop.put("crawlingDomMaxCheck", (crawlingDomMaxPages == -1) ? "0" : "1");
prop.put("crawlingDomMaxPages", (crawlingDomMaxPages == -1) ? 10000 : crawlingDomMaxPages);
prop.put("crawlingQChecked", env.getConfigBool("crawlingQ", false) ? "1" : "0");
prop.put("storeHTCacheChecked", env.getConfigBool("storeHTCache", false) ? "1" : "0");
prop.put("indexingTextChecked", env.getConfigBool("indexText", false) ? "1" : "0");
prop.put("indexingMediaChecked", env.getConfigBool("indexMedia", false) ? "1" : "0");
prop.put("crawlOrderChecked", env.getConfigBool("crawlOrder", false) ? "1" : "0");
long LCbusySleep = Integer.parseInt(env.getConfig(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP, "100"));
int LCppm = (LCbusySleep == 0) ? 1000 : (int) (60000L / LCbusySleep);
prop.put("crawlingSpeedMaxChecked", (LCppm >= 1000) ? "1" : "0");
prop.put("crawlingSpeedCustChecked", ((LCppm > 10) && (LCppm < 1000)) ? "1" : "0");
prop.put("crawlingSpeedMinChecked", (LCppm <= 10) ? "1" : "0");
prop.put("customPPMdefault", ((LCppm > 10) && (LCppm < 1000)) ? Integer.toString(LCppm) : "");
prop.put("xsstopwChecked", env.getConfigBool("xsstopw", false) ? "1" : "0");
prop.put("xdstopwChecked", env.getConfigBool("xdstopw", false) ? "1" : "0");
prop.put("xpstopwChecked", env.getConfigBool("xpstopw", false) ? "1" : "0");
// return rewrite properties
return prop;
}
}
Write Preview
Loading…
Cancel
Save