URL-Fetcher
- - - #(threadError)#:: - Error on stopping thread, it isn't alive anymore:: - Error on restarting thread, it isn't alive anymore#(/threadError)# - - #(runs)#:: - - #(/runs)# - #%env/templates/footer.template%# - - \ No newline at end of file diff --git a/htroot/CrawlURLFetch_p.java b/htroot/CrawlURLFetch_p.java deleted file mode 100644 index 5e19f2d02..000000000 --- a/htroot/CrawlURLFetch_p.java +++ /dev/null @@ -1,543 +0,0 @@ -// CrawlURLFetch_p.java -// ------------------------------------- -// part of YACY -// -// (C) 2007 by Franz Brausze -// -// last change: $LastChangedDate: $ by $LastChangedBy: $ -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// Using this software in any meaning (reading, learning, copying, compiling, -// running) means that you agree that the Author(s) is (are) not responsible -// for cost, loss of data or any harm that may be caused directly or indirectly -// by usage of this softare or this documentation. The usage of this software -// is on your own risk. The installation and usage (starting/running) of this -// software may allow other people or application to access your computer and -// any attached devices and is highly dependent on the configuration of the -// software which must be done by the user of the software; the author(s) is -// (are) also not responsible for proper configuration and usage of the -// software, even if provoked by documentation provided together with -// the software. -// -// Any changes to this file according to the GPL as documented in the file -// gpl.txt aside this file in the shipment you received can be done to the -// lines that follows this copyright notice here, but changes must not be -// done inside the copyright notive above. A re-distribution must contain -// the intact and unchanged copyright notice. -// Contributions and changes to the program code must be marked as such. - -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Random; -import java.util.TreeMap; - -import de.anomic.plasma.plasmaCrawlProfile; -import de.anomic.plasma.plasmaCrawlZURL; -import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.server.serverByteBuffer; -import de.anomic.server.serverSwitch; -import de.anomic.http.httpHeader; -import de.anomic.http.httpRemoteProxyConfig; -import de.anomic.http.httpc; -import de.anomic.server.serverObjects; -import de.anomic.server.logging.serverLog; -import de.anomic.yacy.yacyCore; -import de.anomic.yacy.yacySeed; -import de.anomic.yacy.yacyURL; -import de.anomic.yacy.yacyVersion; - -public class CrawlURLFetch_p { - - private static final long ERR_DATE = 1; - private static final long ERR_HOST_MALFORMED_URL = 1; - private static final long ERR_PEER_GENERAL_CONN = 1; - private static final long ERR_PEER_OFFLINE = 2; - private static final long ERR_THREAD_STOP = 1; - private static final long ERR_THREAD_RESUME = 2; - - private static final long STAT_THREAD_ALIVE = 0; - private static final long STAT_THREAD_STOPPED = 1; - private static final long STAT_THREAD_PAUSED = 2; - - private static URLFetcher fetcher = null; - private static plasmaCrawlProfile.entry profile = null; - private static ArrayList savedURLs = new ArrayList(); - - public static plasmaCrawlProfile.entry getCrawlProfile(serverSwitch env) { - if (profile == null) { - profile = ((plasmaSwitchboard)env).profilesActiveCrawls.newEntry( - "URLFetcher", // Name - null, // URL - ".*", ".*", // General / specific filter - 0, 0, // General / specific depth - -1, -1, -1, // Recrawl / Dom-filter depth / Dom-max-pages - true, // Crawl query - true, true, // Index text / media - false, true, // Store in HT- / TX-Cache - false, // Remote indexing - true, false, false); // Exclude static / dynamic / parent stopwords - } - return profile; - } - - public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { - serverObjects prop = new serverObjects(); - prop.put("host", ""); - - // List previously saved URLs for easy selection - listURLs(prop); - - // List known hosts - listPeers(prop, - post != null && post.containsKey("checkPeerURLCount"), - ((plasmaSwitchboard)env).remoteProxyConfig); - - if (post != null) { - if (post.containsKey("start")) { - long frequency = URLFetcher.DELAY_ONCE; - if (post.containsKey("reg")) { - if (post.get("reg", "").equals("self_det")) { - frequency = URLFetcher.DELAY_SELF_DET; - } else if (post.get("reg", "").equals("delay")) { - frequency = getDate(post.get("frequency", ""), post.get("freq_type", "")); - if (frequency == -1) - prop.put("freqError", ERR_DATE); - } - } - - int count = 50; - if (post.get("amount", "").matches("\\d+")) { - count = Integer.parseInt(post.get("amount", "")); - if (count > 999) count = 999; - } - - if (fetcher != null) fetcher.interrupt(); - fetcher = null; - if (post.get("source", "").equals("peer") && - post.get("peerhash", "").equals("random")) { - fetcher = new URLFetcher( - env, - getCrawlProfile(env), - count, - frequency); - } else { - yacyURL url = null; - if (post.get("source", "").equals("url")) { - try { - url = new yacyURL(post.get("host", null), null); - if (!savedURLs.contains(url.toNormalform(true, true))) - savedURLs.add(url.toNormalform(true, true)); - prop.put("host", post.get("host", url.toString())); - } catch (MalformedURLException e) { - prop.put("host", post.get("host", "")); - prop.put("hostError", ERR_HOST_MALFORMED_URL); - } - } else if (post.get("source", "").equals("savedURL")) { - try { - url = new yacyURL(post.get("saved", ""), null); - } catch (MalformedURLException e) { - /* should never appear, except for invalid input, see above */ - } - } else if (post.get("source", "").equals("peer")) { - yacySeed ys = null; - ys = yacyCore.seedDB.get(post.get("peerhash", null)); - if (ys != null) { - if ((url = URLFetcher.getListServletURL( - ys.getPublicAddress(), - URLFetcher.MODE_LIST, - count, - yacyCore.seedDB.mySeed().hash)) == null) { - prop.put("peerError", ERR_PEER_GENERAL_CONN); - prop.put("peerError_hash", post.get("peerhash", "")); - prop.put("peerError_name", ys.getName()); - } - } else { - prop.put("peerError", ERR_PEER_OFFLINE); - prop.put("peerError_hash", post.get("peerhash", "")); - } - } - - if (url != null) { - fetcher = new URLFetcher( - env, - getCrawlProfile(env), - url, - count, - frequency); - } - } - if (fetcher != null) fetcher.start(); - } - else if (post.containsKey("stop")) { - if (fetcher != null) { - fetcher.interrupt(); - } else { - prop.put("threadError", ERR_THREAD_STOP); - } - } - else if (post.containsKey("restart")) { - if (fetcher != null) { - fetcher.interrupt(); - if (fetcher.url == null) { - fetcher = new URLFetcher( - env, - getCrawlProfile(env), - fetcher.count, - fetcher.delay); - } else { - fetcher = new URLFetcher( - env, - getCrawlProfile(env), - fetcher.url, - fetcher.count, - fetcher.delay); - } - fetcher.start(); - } else { - prop.put("threadError", ERR_THREAD_RESUME); - } - } - else if (post.containsKey("resetDelay")) { - final long frequency = getDate(post.get("newDelay", ""), "minutes"); - if (frequency == -1) { - prop.put("freqError", ERR_DATE); - } else { - fetcher.delay = frequency; - } - } - prop.put("LOCATION", "/CrawlURLFetch_p.html"); - } - - if (fetcher != null) { - prop.put("runs", "1"); - prop.put("runs_status", - ((fetcher.paused && fetcher.isAlive()) ? STAT_THREAD_PAUSED : - (fetcher.isAlive()) ? STAT_THREAD_ALIVE : STAT_THREAD_STOPPED)); - prop.putNum("runs_totalRuns", URLFetcher.totalRuns); - prop.putNum("runs_totalFetchedURLs", URLFetcher.totalFetchedURLs); - prop.putNum("runs_totalFailedURLs", URLFetcher.totalFailed); - prop.putNum("runs_lastRun", fetcher.lastRun); - prop.putNum("runs_lastFetchedURLs", fetcher.lastFetchedURLs); - prop.put("runs_lastServerResponse", (fetcher.lastServerResponse == null) - ? "" : fetcher.lastServerResponse); - prop.putNum("runs_curDelay", (int)(fetcher.delay / 60000)); - - Iterator it = fetcher.failed.keySet().iterator(); - int i = 0; - Object key; - while (it.hasNext()) { - key = it.next(); - prop.put("runs_error_" + i + "_reason", fetcher.failed.get(key)); - prop.put("runs_error_" + i + "_url", (String)key); - i++; - } - prop.put("runs_error", i); - } - - return prop; - } - - private static int listURLs(serverObjects prop) { - if (savedURLs.size() == 0) return 0; - prop.put("saved", "1"); - for (int i=0; i#stop.gif::start.gif#(/remoteTriggeredCrawlPaused)#)
#stop.gif::start.gif#(/globalCrawlTriggerPaused)#)
-
\ No newline at end of file
diff --git a/htroot/js/WatchCrawler.js b/htroot/js/WatchCrawler.js
index 0bc7002d0..96f9d4691 100644
--- a/htroot/js/WatchCrawler.js
+++ b/htroot/js/WatchCrawler.js
@@ -150,9 +150,16 @@ function handleQueues(){
updateTable(localcrawlerqueue, "local crawler");
+ limitcrawlerqueue=getFirstChild(xml, "limitcrawlerqueue");
+ updateTable(limitcrawlerqueue, "limitCrawlerTable");
+ limitcrawlerqueue_size=getValue(getFirstChild(limitcrawlerqueue, "size"));
+ limitcrawlerqueue_state=getValue(getFirstChild(limitcrawlerqueue, "state"));
+ document.getElementById("limitcrawlerqueuesize").firstChild.nodeValue=limitcrawlerqueue_size;
+ putQueueState("limitcrawler", limitcrawlerqueue_state);
+ updateTable(limitcrawlerqueue, "limit crawler");
+
remotecrawlerqueue=getFirstChild(xml, "remotecrawlerqueue");
updateTable(remotecrawlerqueue, "remoteCrawlerTable");
-
remotecrawlerqueue_size=getValue(getFirstChild(remotecrawlerqueue, "size"));
remotecrawlerqueue_state=getValue(getFirstChild(remotecrawlerqueue, "state"));
document.getElementById("remotecrawlerqueuesize").firstChild.nodeValue=remotecrawlerqueue_size;
diff --git a/htroot/rct_p.html b/htroot/rct_p.html
new file mode 100644
index 000000000..af4a2dda4
--- /dev/null
+++ b/htroot/rct_p.html
@@ -0,0 +1,30 @@
+
+
+
+ URL Fetcher Menu
--
-
- URL Fetcher -
- URL Stack -
remote crawl fetch test
+ + + + #%env/templates/footer.template%# + + \ No newline at end of file diff --git a/htroot/rct_p.java b/htroot/rct_p.java new file mode 100644 index 000000000..c2c33a2f7 --- /dev/null +++ b/htroot/rct_p.java @@ -0,0 +1,124 @@ +// rct_p.java +// ----------------------- +// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 28.11.2007 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2007-11-14 01:15:28 +0000 (Mi, 14 Nov 2007) $ +// $LastChangedRevision: 4216 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import java.net.MalformedURLException; +import java.text.ParseException; +import java.util.Date; +import java.util.Iterator; + +import de.anomic.http.httpHeader; +import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.server.serverDate; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; +import de.anomic.xml.rssReader; +import de.anomic.yacy.yacyClient; +import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacySeed; +import de.anomic.yacy.yacyURL; + +public class rct_p { + + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { + // return variable that accumulates replacements + plasmaSwitchboard sb = (plasmaSwitchboard) env; + serverObjects prop = new serverObjects(); + + if (post != null) { + if (post.containsKey("retrieve")) { + String peerhash = post.get("peer", null); + yacySeed seed = (peerhash == null) ? null : yacyCore.seedDB.getConnected(peerhash); + rssReader reader = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(seed, 10); + if (reader != null) { + rssReader.Item item; + for (int i = 0; i < reader.items(); i++) { + item = reader.getItem(i); + //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate()); + + // put url on remote crawl stack + yacyURL url; + try { + url = new yacyURL(item.getLink(), null); + } catch (MalformedURLException e) { + url = null; + } + Date loaddate; + try { + loaddate = serverDate.parseShortSecondTime(item.getPubDate()); + } catch (ParseException e) { + loaddate = new Date(); + } + yacyURL referrer = null; // referrer needed! + if (sb.acceptURL(url)) { + // stack url + sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'"); + String reasonString = sb.crawlStacker.stackCrawl(url, referrer, peerhash, "REMOTE-CRAWLING", loaddate, 0, sb.defaultRemoteProfile); + + if (reasonString == null) { + // done + env.getLog().logInfo("crawlOrder: added remote crawl url: " + url.toNormalform(true, false)); + } else if (reasonString.startsWith("double")) { + // case where we have already the url loaded; + env.getLog().logInfo("crawlOrder: ignored double remote crawl url: " + url.toNormalform(true, false)); + } else { + env.getLog().logInfo("crawlOrder: ignored [" + reasonString + "] remote crawl url: " + url.toNormalform(true, false)); + } + } else { + env.getLog().logWarning("crawlOrder: Received URL outside of our domain: " + url.toNormalform(true, false)); + } + } + } + } + } + + listHosts(prop); + + // return rewrite properties + return prop; + } + + private static void listHosts(serverObjects prop) { + // list known hosts + yacySeed seed; + int hc = 0; + if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { + Iterator e = yacyCore.dhtAgent.getProvidesRemoteCrawlURLs(); + while (e.hasNext()) { + seed = (yacySeed) e.next(); + if (seed != null) { + prop.put("hosts_" + hc + "_hosthash", seed.hash); + prop.putHTML("hosts_" + hc + "_hostname", seed.hash + " " + seed.get(yacySeed.NAME, "nameless") + " (" + seed.getLong(yacySeed.RCOUNT, 0) + ")"); + hc++; + } + } + prop.put("hosts", hc); + } else { + prop.put("hosts", "0"); + } + } + +} diff --git a/htroot/xml/queues_p.java b/htroot/xml/queues_p.java index 82105b2ac..15e8e8321 100644 --- a/htroot/xml/queues_p.java +++ b/htroot/xml/queues_p.java @@ -163,11 +163,15 @@ public class queues_p { prop.put("localCrawlState", sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING); int stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE); addNTable(prop, "list-local", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize))); - - + //global crawl queue - prop.putNum("remoteCrawlSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount())); - prop.put("remoteCrawlState", sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER) ? STATE_PAUSED : STATE_RUNNING); + prop.putNum("limitCrawlSize", Integer.toString(sb.crawlQueues.limitCrawlJobSize())); + prop.put("limitCrawlState", STATE_RUNNING); + stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT); + + //global crawl queue + prop.putNum("remoteCrawlSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount())); + prop.put("remoteCrawlState", sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL) ? STATE_PAUSED : STATE_RUNNING); stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT); if (stackSize == 0) { diff --git a/htroot/xml/queues_p.xml b/htroot/xml/queues_p.xml index ed82dd877..23cfea8a0 100644 --- a/htroot/xml/queues_p.xml +++ b/htroot/xml/queues_p.xml @@ -49,6 +49,22 @@ #{/list-local}# +public static final String CRAWLJOB_GLOBAL_CRAWL_TRIGGER = "61_globalcrawltrigger"
Name of the global crawl trigger thread, popping one entry off it's queue and sending it to a non-busy peer to - * crawl it
+ *public static final String CRAWLJOB_REMOTE_CRAWL_LOADER = "60_remotecrawlloader"
Name of the remote crawl list loading thread
* - * @see plasmaSwitchboard#CRAWLJOB_REMOTE_TRIGGERED_CRAWL + * @see plasmaSwitchboard#CRAWLJOB_REMOTE_CRAWL_LOADER */ - public static final String CRAWLJOB_GLOBAL_CRAWL_TRIGGER = "61_globalcrawltrigger"; - public static final String CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_START = "limitCrawlTriggerJob"; - public static final String CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_JOBCOUNT = "limitCrawlTriggerJobSize"; - public static final String CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_FREEMEM = null; - public static final String CRAWLJOB_GLOBAL_CRAWL_TRIGGER_IDLESLEEP = "61_globalcrawltrigger_idlesleep"; - public static final String CRAWLJOB_GLOBAL_CRAWL_TRIGGER_BUSYSLEEP = "61_globalcrawltrigger_busysleep"; + public static final String CRAWLJOB_REMOTE_CRAWL_LOADER = "60_remotecrawlloader"; + public static final String CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START = "remoteCrawlLoaderJob"; + public static final String CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT = null; + public static final String CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM = null; + public static final String CRAWLJOB_REMOTE_CRAWL_LOADER_IDLESLEEP = "60_remotecrawlloader_idlesleep"; + public static final String CRAWLJOB_REMOTE_CRAWL_LOADER_BUSYSLEEP = "60_remotecrawlloader_busysleep"; // 62_remotetriggeredcrawl /** @@ -1208,9 +1207,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.crawlJobsStatus.put(CRAWLJOB_REMOTE_TRIGGERED_CRAWL, new Object[]{ new Object(), Boolean.valueOf(getConfig(CRAWLJOB_REMOTE_TRIGGERED_CRAWL + "_isPaused", "false"))}); - this.crawlJobsStatus.put(CRAWLJOB_GLOBAL_CRAWL_TRIGGER, new Object[]{ + this.crawlJobsStatus.put(CRAWLJOB_REMOTE_CRAWL_LOADER, new Object[]{ new Object(), - Boolean.valueOf(getConfig(CRAWLJOB_GLOBAL_CRAWL_TRIGGER + "_isPaused", "false"))}); + Boolean.valueOf(getConfig(CRAWLJOB_REMOTE_CRAWL_LOADER + "_isPaused", "false"))}); // init cookie-Monitor this.log.logConfig("Starting Cookie Monitor"); @@ -1340,8 +1339,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser new serverInstantThread(this, PROXY_CACHE_ENQUEUE_METHOD_START, PROXY_CACHE_ENQUEUE_METHOD_JOBCOUNT, PROXY_CACHE_ENQUEUE_METHOD_FREEMEM), 10000); deployThread(CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null, new serverInstantThread(crawlQueues, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000); - deployThread(CRAWLJOB_GLOBAL_CRAWL_TRIGGER, "Global Crawl Trigger", "thread that triggeres remote peers for crawling", "/IndexCreateWWWGlobalQueue_p.html", - new serverInstantThread(crawlQueues, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_START, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_JOBCOUNT, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_FREEMEM), 30000); // error here? + deployThread(CRAWLJOB_REMOTE_CRAWL_LOADER, "Remote Crawl URL Loader", "thread that loads remote crawl lists from other peers", "", + new serverInstantThread(crawlQueues, CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM), 30000); // error here? deployThread(CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html", new serverInstantThread(crawlQueues, CRAWLJOB_LOCAL_CRAWL_METHOD_START, CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000); deployThread(SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null, @@ -2639,18 +2638,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser thread.setIdleSleep(1000); } - thread = getThread(CRAWLJOB_GLOBAL_CRAWL_TRIGGER); - if (thread != null) { - setConfig(CRAWLJOB_GLOBAL_CRAWL_TRIGGER_BUSYSLEEP , thread.setBusySleep(Math.max(1000, newBusySleep * 3))); - thread.setIdleSleep(10000); - } - /* - thread = getThread(CRAWLJOB_REMOTE_TRIGGERED_CRAWL); - if (thread != null) { - setConfig(CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP , thread.setBusySleep(newBusySleep * 10)); - thread.setIdleSleep(10000); - } - */ thread = getThread(PROXY_CACHE_ENQUEUE); if (thread != null) { setConfig(PROXY_CACHE_ENQUEUE_BUSYSLEEP , thread.setBusySleep(0)); diff --git a/source/de/anomic/xml/rssReader.java b/source/de/anomic/xml/rssReader.java index 313f9408d..4616ec200 100644 --- a/source/de/anomic/xml/rssReader.java +++ b/source/de/anomic/xml/rssReader.java @@ -26,6 +26,8 @@ package de.anomic.xml; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; @@ -38,6 +40,9 @@ import org.xml.sax.Attributes; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; +import de.anomic.server.serverByteBuffer; +import de.anomic.server.logging.serverLog; + public class rssReader extends DefaultHandler { // statics for item generation and automatic categorization @@ -72,17 +77,7 @@ public class rssReader extends DefaultHandler { private HashMap items; // a guid:Item map - public rssReader(String path) { - init(); - parse(path); - } - - public rssReader(InputStream stream) { - init(); - parse(stream); - } - - private void init() { + public rssReader() { itemsGUID = new ArrayList(); items = new HashMap(); buffer = new StringBuffer(); @@ -93,7 +88,8 @@ public class rssReader extends DefaultHandler { parsingItem = false; } - private void parse(String path) { + public rssReader(String path) { + this(); try { SAXParserFactory factory = SAXParserFactory.newInstance(); SAXParser saxParser = factory.newSAXParser(); @@ -103,7 +99,8 @@ public class rssReader extends DefaultHandler { } } - private void parse(InputStream stream) { + public rssReader(InputStream stream) { + this(); try { SAXParserFactory factory = SAXParserFactory.newInstance(); SAXParser saxParser = factory.newSAXParser(); @@ -112,6 +109,42 @@ public class rssReader extends DefaultHandler { e.printStackTrace(); } } + + public static rssReader parse(byte[] a) { + + // check integrity of array + if ((a == null) || (a.length == 0)) { + serverLog.logWarning("rssReader", "response=null"); + return null; + } + if (a.length < 100) { + serverLog.logWarning("rssReader", "response=" + new String(a)); + return null; + } + if (!serverByteBuffer.equals(a, " 0) return s; + } + } catch (kelondroException e) { + System.out.println("DEBUG providesRemoteCrawlURLsEnum:" + e.getMessage()); + yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db."); + seedDB.resetActiveTable(); + return null; + } + return null; + } + + public Object next() { + yacySeed next = nextSeed; + nextSeed = nextInternal(); + return next; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + + } + public Iterator getAcceptRemoteIndexSeeds(String starthash) { // returns an enumeration of yacySeed-Objects // that have the AcceptRemoteIndex-Flag set diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index 4c1f67f91..792b0e04b 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -53,7 +53,7 @@ public final class yacyVersion implements Comparator, Comparable { public static final float YACY_SUPPORTS_GZIP_POST_REQUESTS = (float) 0.40300772; public static final float YACY_ACCEPTS_RANKING_TRANSMISSION = (float) 0.414; public static final float YACY_HANDLES_COLLECTION_INDEX = (float) 0.486; - public static final float YACY_PROVIDES_CRAWLS_VIA_LIST_HTML = (float) 0.50403367; + public static final float YACY_POVIDES_REMOTECRAWL_LISTS = (float) 0.550; // information about latest release, retrieved by other peers release version public static double latestRelease = 0.1; // this value is overwritten when a peer with later version appears diff --git a/yacy.init b/yacy.init index 55264ca16..6b536fb45 100644 --- a/yacy.init +++ b/yacy.init @@ -558,12 +558,12 @@ filterOutStopwordsFromTopwords=true 50_localcrawl_busysleep__pro=100 50_localcrawl_memprereq=4194304 50_localcrawl_isPaused=false -61_globalcrawltrigger_idlesleep=10000 -61_globalcrawltrigger_busysleep=500 -61_globalcrawltrigger_memprereq=2097152 -61_globalcrawltrigger_isPaused=false +60_remotecrawlloader_idlesleep=10000 +60_remotecrawlloader_busysleep=2000 +60_remotecrawlloader_memprereq=2097152 +60_remotecrawlloader_isPaused=false 62_remotetriggeredcrawl_idlesleep=10000 -62_remotetriggeredcrawl_busysleep=1000 +62_remotetriggeredcrawl_busysleep=500 62_remotetriggeredcrawl_memprereq=6291456 62_remotetriggeredcrawl_isPaused=false 70_cachemanager_idlesleep=1000