From 61b27217b991f2072e1e6f673358bc4ab6c661f7 Mon Sep 17 00:00:00 2001
From: Michael Christen <mc@yacy.net>
Date: Wed, 5 Oct 2022 22:59:09 +0200
Subject: [PATCH] throttle number of DNS requests: as soon as the number of
 requests is > 50, there is a forced delay of (10 * (requests - 50))
 milliseconds. That means that once the number of DNS requests reach 150,
 there is a one second delay to each request.

This shall prevent that a remote DNS is flooded with request and
possibly gets damaged.
This is also a fix/enhancement for
https://github.com/yacy/yacy_search_server/issues/513
---
 source/net/yacy/cora/protocol/Domains.java | 37 ++++++++++++++--------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java
index f4b142e96..c7793c4e1 100644
--- a/source/net/yacy/cora/protocol/Domains.java
+++ b/source/net/yacy/cora/protocol/Domains.java
@@ -52,6 +52,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
@@ -100,7 +101,8 @@ public class Domains {
     private static       List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>());
     public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write
     public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write
-
+    private static AtomicLong dnsRequests = new AtomicLong(0);
+    
     private static Set<InetAddress> myHostAddresses = new HashSet<InetAddress>();
     private static Set<InetAddress> localHostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
     private static Set<InetAddress> publicIPv4HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
@@ -993,18 +995,27 @@ public class Domains {
                     }
                 }
                 Thread.currentThread().setName(oldName);
-                if (ip == null) try {
-                    ip = timeLimiter.callWithTimeout(new Callable<InetAddress>() {
-                        @Override
-                        public InetAddress call() throws Exception {
-                            return InetAddress.getByName(host);
-                        }
-                    }, 3000L, TimeUnit.MILLISECONDS);
-                    //ip = TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone
-                } catch (final InterruptedException | TimeoutException e) {
-                	// in case of a timeout - maybe cause of massive requests - do not fill NAME_CACHE_MISS
-                	LOOKUP_SYNC.remove(host);
-                    return null;
+                if (ip == null) {
+                	long activeRequests = dnsRequests.incrementAndGet();
+                	if (activeRequests > 50) {
+                		// throttle requests to remote DNS
+                		try {Thread.sleep(10 * (activeRequests - 50));} catch (InterruptedException e) {}
+                	}
+                	try {
+	                    ip = timeLimiter.callWithTimeout(new Callable<InetAddress>() {
+	                        @Override
+	                        public InetAddress call() throws Exception {
+	                            return InetAddress.getByName(host);
+	                        }
+	                    }, 3000L, TimeUnit.MILLISECONDS);
+	                    //ip = TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone
+	                } catch (final InterruptedException | TimeoutException e) {
+	                	// in case of a timeout - maybe cause of massive requests - do not fill NAME_CACHE_MISS
+	                	LOOKUP_SYNC.remove(host);
+	                    return null;
+	                } finally {
+	                	dnsRequests.decrementAndGet();
+	                }
                 }
                 //.out.println("DNSLOOKUP-*LOOKUP* " + host + ", time = " + (System.currentTimeMillis() - t) + "ms");
             } catch (final Throwable e) {