From 321840fde3d2ffad15e37e9ff6a1d8e4285e61d3 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 2 Dec 2014 16:26:07 +0100 Subject: [PATCH] Replaced all fixed thread pools with cached thread pools. The cached thread pools will flush their cached (dead) threads after 60 seconds. This will cause that YaCy now runs constantly withl about 50 threads, about 100 at peak times. Previously, about 400 threads had been cached and kept in a hibernation state, which caused that the numproc counter in /proc/user_beancounters (exists only in VM-hosted linux) was as high as the cached number of threads. This caused that VM supervisors terminated whole VM sessions if a limit was reached. Many VM providers have limits of numproc=96 which made it virtually impossible to run YaCy on such machines. With this change, it will be possible to run many YaCy instances even on VM hosts. --- source/net/yacy/cora/protocol/Domains.java | 2 +- source/net/yacy/cora/protocol/http/HTTPClient.java | 2 +- source/net/yacy/document/importer/MediawikiImporter.java | 4 ++-- source/net/yacy/kelondro/blob/ArrayStack.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index f0f687840..10dc7b360 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -803,7 +803,7 @@ public class Domains { cacheHit_Insert++; } - final private static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newFixedThreadPool(20)); + final private static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newCachedThreadPool()); /** * strip off any parts of an url, address string (containing host/ip:port) or raw IPs/Hosts, diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index 4082cb486..7f665822d 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -117,7 +117,7 @@ public class HTTPClient { private long upbytes = 0L; private String host = null; private final long timeout; - private static ExecutorService executor = Executors.newFixedThreadPool(200); + private static ExecutorService executor = Executors.newCachedThreadPool(); public HTTPClient(final ClientIdentification.Agent agent) { super(); diff --git a/source/net/yacy/document/importer/MediawikiImporter.java b/source/net/yacy/document/importer/MediawikiImporter.java index d6f116638..1e9e7a6ef 100644 --- a/source/net/yacy/document/importer/MediawikiImporter.java +++ b/source/net/yacy/document/importer/MediawikiImporter.java @@ -175,7 +175,7 @@ public class MediawikiImporter extends Thread implements Importer { final int threads = Math.max(2, Runtime.getRuntime().availableProcessors() - 1); final BlockingQueue in = new ArrayBlockingQueue(threads * 10); final BlockingQueue out = new ArrayBlockingQueue(threads * 10); - final ExecutorService service = Executors.newFixedThreadPool(threads + 1); + final ExecutorService service = Executors.newCachedThreadPool(); final convertConsumer[] consumers = new convertConsumer[threads]; final Future[] consumerResults = (Future[]) Array.newInstance(Future.class, threads); for (int i = 0; i < threads; i++) { @@ -325,7 +325,7 @@ public class MediawikiImporter extends Thread implements Importer { final PositionAwareReader in = new PositionAwareReader(dumpFile); final indexProducer producer = new indexProducer(100, idxFromMediawikiXML(dumpFile)); final wikiConsumer consumer = new wikiConsumer(100, producer); - final ExecutorService service = Executors.newFixedThreadPool(2); + final ExecutorService service = Executors.newCachedThreadPool(); final Future producerResult = service.submit(consumer); final Future consumerResult = service.submit(producer); service.shutdown(); diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 2e3d8b3b5..bf2e1d781 100644 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -879,7 +879,7 @@ public class ArrayStack implements BLOB { assert mem() <= m : "m = " + m + ", mem() = " + mem(); } - private static final ExecutorService DELETE_EXECUTOR = Executors.newFixedThreadPool(128); + private static final ExecutorService DELETE_EXECUTOR = Executors.newCachedThreadPool(); /** * close the BLOB