diff --git a/htroot/IndexControl_p.java b/htroot/IndexControl_p.java index 43b5966ed..39ae35e5b 100644 --- a/htroot/IndexControl_p.java +++ b/htroot/IndexControl_p.java @@ -333,6 +333,7 @@ public class IndexControl_p { prop.put("urlhashsimilar", 1); while (entryIt.hasNext() && i < 256) { entry = (indexURLEntry) entryIt.next(); + if (entry == null) break; prop.put("urlhashsimilar_rows_"+rows+"_cols_"+cols+"_urlHash", entry.hash()); cols++; if (cols==8) { diff --git a/htroot/WatchCrawler_p.java b/htroot/WatchCrawler_p.java index d837ff382..19c250fab 100644 --- a/htroot/WatchCrawler_p.java +++ b/htroot/WatchCrawler_p.java @@ -266,8 +266,7 @@ public class WatchCrawler_p { HashMap hyperlinks = (HashMap) scraper.getAnchors(); // creating a crawler profile - /* file.toURL() marked as deprecated per Sun JRE 6 */ - plasmaCrawlProfile.entry profile = switchboard.profiles.newEntry(fileName, file.toURL().toString(), newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); + plasmaCrawlProfile.entry profile = switchboard.profiles.newEntry(fileName, "file://" + file.toString(), newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); // loop through the contained links Iterator interator = hyperlinks.entrySet().iterator(); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 7367f93fa..442830ea8 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -127,6 +127,7 @@ public class yacysearch { prop.put("input_display", display); prop.putASIS("input_promoteSearchPageGreeting", promoteSearchPageGreeting); prop.put("input_former", ""); + prop.put("former", ""); prop.put("input_count", 10); prop.put("input_resource", "global"); prop.put("input_time", 6); @@ -502,6 +503,7 @@ public class yacysearch { prop.put("input_display", display); prop.putASIS("input_promoteSearchPageGreeting", promoteSearchPageGreeting); prop.put("input_former", post.get("search", "")); + prop.put("former", post.get("search", "")); prop.put("input_count", count); prop.put("input_resource", (global) ? "global" : "local"); prop.put("input_time", searchtime / 1000); diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 9738ef4cc..bcb25edae 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -362,7 +362,8 @@ public final class plasmaCrawlLURL { } public final boolean hasNext() { - if (this.error) { return false; } + if (this.error) return false; + if (this.iter == null) return false; return this.iter.hasNext(); } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index ceed8aaac..bebdfd7b6 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -1355,17 +1355,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser return network.indexOf(peer) >= 0; } else if (clustermode.equals("publiccluster")) { // check if we got the request from a peer in the public cluster - String network = getConfig("cluster.peers.yacydomain", ""); - // check for .yacyh hexhash-domain - String hexhash = yacySeed.b64Hash2hexHash(peer); - if (hexhash == null) return false; - if (network.indexOf(hexhash + ".yacyh") >= 0) return true; - // resolve seed - yacySeed seed = yacyCore.seedDB.get(peer); - if (seed == null) return false; - // check for .yacy (name) - Domain - if (network.indexOf(seed.getName() + ".yacy") >= 0) return true; - return false; + return this.clusterhashes.contains(peer); } else { return false; } @@ -1382,15 +1372,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser String network = getConfig("cluster.peers.ipport", ""); return network.indexOf(seed.getAddress()) >= 0; } else if (clustermode.equals("publiccluster")) { - // check if we got the request from a peer in the public cluster - String network = getConfig("cluster.peers.yacydomain", ""); - // check for .yacyh hexhash-domain - String hexhash = yacySeed.b64Hash2hexHash(seed.hash); - if (hexhash == null) return false; - if (network.indexOf(hexhash + ".yacyh") >= 0) return true; - // check for .yacy (name) - Domain - if (network.indexOf(seed.getName() + ".yacy") >= 0) return true; - return false; + // check if we got the request from a peer in the public cluster + return this.clusterhashes.contains(seed.hash); } else { return false; } @@ -3049,32 +3032,46 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser serverThread thread; thread = getThread(INDEX_DIST); - setConfig(INDEX_DIST_BUSYSLEEP , thread.setBusySleep(Math.max(2000, thread.setBusySleep(newBusySleep * 2)))); - thread.setIdleSleep(30000); + if (thread != null) { + setConfig(INDEX_DIST_BUSYSLEEP , thread.setBusySleep(Math.max(2000, thread.setBusySleep(newBusySleep * 2)))); + thread.setIdleSleep(30000); + } thread = getThread(CRAWLJOB_LOCAL_CRAWL); - setConfig(CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP , thread.setBusySleep(newBusySleep)); - thread.setIdleSleep(1000); + if (thread != null) { + setConfig(CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP , thread.setBusySleep(newBusySleep)); + thread.setIdleSleep(1000); + } thread = getThread(CRAWLJOB_GLOBAL_CRAWL_TRIGGER); - setConfig(CRAWLJOB_GLOBAL_CRAWL_TRIGGER_BUSYSLEEP , thread.setBusySleep(Math.max(1000, newBusySleep * 3))); - thread.setIdleSleep(10000); - - //thread = getThread(CRAWLJOB_REMOTE_TRIGGERED_CRAWL); - //setConfig(CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP , thread.setBusySleep(newBusySleep * 10)); - //thread.setIdleSleep(10000); - + if (thread != null) { + setConfig(CRAWLJOB_GLOBAL_CRAWL_TRIGGER_BUSYSLEEP , thread.setBusySleep(Math.max(1000, newBusySleep * 3))); + thread.setIdleSleep(10000); + } + /* + thread = getThread(CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + if (thread != null) { + setConfig(CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP , thread.setBusySleep(newBusySleep * 10)); + thread.setIdleSleep(10000); + } + */ thread = getThread(PROXY_CACHE_ENQUEUE); - setConfig(PROXY_CACHE_ENQUEUE_BUSYSLEEP , thread.setBusySleep(0)); - thread.setIdleSleep(1000); + if (thread != null) { + setConfig(PROXY_CACHE_ENQUEUE_BUSYSLEEP , thread.setBusySleep(0)); + thread.setIdleSleep(1000); + } thread = getThread(INDEXER); - setConfig(INDEXER_BUSYSLEEP , thread.setBusySleep(newBusySleep / 2)); - thread.setIdleSleep(1000); + if (thread != null) { + setConfig(INDEXER_BUSYSLEEP , thread.setBusySleep(newBusySleep / 2)); + thread.setIdleSleep(1000); + } thread = getThread(CRAWLSTACK); - setConfig(CRAWLSTACK_BUSYSLEEP , thread.setBusySleep(0)); - thread.setIdleSleep(5000); + if (thread != null) { + setConfig(CRAWLSTACK_BUSYSLEEP , thread.setBusySleep(0)); + thread.setIdleSleep(5000); + } }