diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index f4b142e96..c7793c4e1 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -52,6 +52,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -100,7 +101,8 @@ public class Domains { private static List nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList()); public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write - + private static AtomicLong dnsRequests = new AtomicLong(0); + private static Set myHostAddresses = new HashSet(); private static Set localHostAddresses = new HashSet(); // subset of myHostAddresses private static Set publicIPv4HostAddresses = new HashSet(); // subset of myHostAddresses @@ -993,18 +995,27 @@ public class Domains { } } Thread.currentThread().setName(oldName); - if (ip == null) try { - ip = timeLimiter.callWithTimeout(new Callable() { - @Override - public InetAddress call() throws Exception { - return InetAddress.getByName(host); - } - }, 3000L, TimeUnit.MILLISECONDS); - //ip = TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone - } catch (final InterruptedException | TimeoutException e) { - // in case of a timeout - maybe cause of massive requests - do not fill NAME_CACHE_MISS - LOOKUP_SYNC.remove(host); - return null; + if (ip == null) { + long activeRequests = dnsRequests.incrementAndGet(); + if (activeRequests > 50) { + // throttle requests to remote DNS + try {Thread.sleep(10 * (activeRequests - 50));} catch (InterruptedException e) {} + } + try { + ip = timeLimiter.callWithTimeout(new Callable() { + @Override + public InetAddress call() throws Exception { + return InetAddress.getByName(host); + } + }, 3000L, TimeUnit.MILLISECONDS); + //ip = TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone + } catch (final InterruptedException | TimeoutException e) { + // in case of a timeout - maybe cause of massive requests - do not fill NAME_CACHE_MISS + LOOKUP_SYNC.remove(host); + return null; + } finally { + dnsRequests.decrementAndGet(); + } } //.out.println("DNSLOOKUP-*LOOKUP* " + host + ", time = " + (System.currentTimeMillis() - t) + "ms"); } catch (final Throwable e) {