From ddcf285499698b2f2e2ba7e0526e2b932e022c8f Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 16 Sep 2008 23:04:24 +0000 Subject: [PATCH] - fixed a bug in performance setting (did not work with german translation) - reduced maximum number of error url references to save some memory (this was actually a small memory leak) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5174 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/WatchCrawler_p.java | 16 ++++++++-------- source/de/anomic/crawler/Balancer.java | 2 +- source/de/anomic/crawler/ZURL.java | 2 ++ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/htroot/WatchCrawler_p.java b/htroot/WatchCrawler_p.java index 46bd0f54a..ea3a3651d 100644 --- a/htroot/WatchCrawler_p.java +++ b/htroot/WatchCrawler_p.java @@ -383,12 +383,12 @@ public class WatchCrawler_p { } // performance settings - final long LCbusySleep = Integer.parseInt(env.getConfig(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP, "100")); + final long LCbusySleep = Integer.parseInt(env.getConfig(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP, "1000")); final int LCppm = (int) (60000L / Math.max(1,LCbusySleep)); - prop.put("crawlingSpeedMaxChecked", (LCppm >= 1000) ? "1" : "0"); - prop.put("crawlingSpeedCustChecked", ((LCppm > 10) && (LCppm < 1000)) ? "1" : "0"); + prop.put("crawlingSpeedMaxChecked", (LCppm >= 6000) ? "1" : "0"); + prop.put("crawlingSpeedCustChecked", ((LCppm > 10) && (LCppm < 6000)) ? "1" : "0"); prop.put("crawlingSpeedMinChecked", (LCppm <= 10) ? "1" : "0"); - prop.put("customPPMdefault", ((LCppm > 10) && (LCppm < 1000)) ? Integer.toString(LCppm) : ""); + prop.put("customPPMdefault", Integer.toString(LCppm)); // return rewrite properties return prop; @@ -404,14 +404,14 @@ public class WatchCrawler_p { } private static void setPerformance(final plasmaSwitchboard sb, final serverObjects post) { - final String crawlingPerformance = post.get("crawlingPerformance","custom"); - final long LCbusySleep = Integer.parseInt(sb.getConfig(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP, "100")); + final String crawlingPerformance = post.get("crawlingPerformance", "custom"); + final long LCbusySleep = Integer.parseInt(sb.getConfig(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP, "1000")); int wantedPPM = (LCbusySleep == 0) ? 6000 : (int) (60000L / LCbusySleep); try { wantedPPM = Integer.parseInt(post.get("customPPM", Integer.toString(wantedPPM))); } catch (final NumberFormatException e) {} - if (crawlingPerformance.equals("minimum")) wantedPPM = 10; - if (crawlingPerformance.equals("maximum")) wantedPPM = 6000; + if (crawlingPerformance.toLowerCase().equals("minimum")) wantedPPM = 10; + if (crawlingPerformance.toLowerCase().equals("maximum")) wantedPPM = 6000; sb.setPerformance(wantedPPM); } diff --git a/source/de/anomic/crawler/Balancer.java b/source/de/anomic/crawler/Balancer.java index 1f5112f28..68b232d02 100644 --- a/source/de/anomic/crawler/Balancer.java +++ b/source/de/anomic/crawler/Balancer.java @@ -239,7 +239,7 @@ public class Balancer { return removedCounter; } - public synchronized boolean has(final String urlhash) { + public boolean has(final String urlhash) { return urlFileIndex.has(urlhash.getBytes()); } diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 26b6c93f1..4dfc97386 100755 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -44,6 +44,7 @@ import de.anomic.yacy.yacyURL; public class ZURL { private static final int EcoFSBufferSize = 200; + private static final int maxStackSize = 300; public final static kelondroRow rowdef = new kelondroRow( "String urlhash-" + yacySeedDB.commonHashLength + ", " + // the url's hash @@ -117,6 +118,7 @@ public class ZURL { public synchronized void push(final Entry e) { stack.add(e.hash()); + while (stack.size() > maxStackSize) stack.removeFirst(); } public Entry top(final int pos) {