From 9ef77d57f58048a469fb9395e93b112068e99ca7 Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 12 Jan 2009 10:55:48 +0000 Subject: [PATCH] added an access control to the search interface using white/blacklists: in the network configuration, you can configure a whiteliste and a blacklist - blacklistet clients cannot search - whitelistet client get never any search restrictions - for all other clients: apply DoS search restrictions Please see the example configuriation in yacy.network.freeworld.unit by default, all clients from localhosts get whitlistet. If you have your own YaCy network, please put all the IPs of your peers into the whitelist git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5475 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- build.properties | 2 +- defaults/yacy.network.freeworld.unit | 6 +- defaults/yacy.network.intranet.unit | 6 +- defaults/yacy.network.readme | 16 +++++ defaults/yacy.network.webportal.unit | 6 +- htroot/yacysearch.java | 33 ++++++--- .../de/anomic/plasma/plasmaSwitchboard.java | 15 ++-- .../plasma/plasmaSwitchboardConstants.java | 3 + source/de/anomic/server/serverDomains.java | 70 +++++++++---------- 9 files changed, 98 insertions(+), 59 deletions(-) diff --git a/build.properties b/build.properties index ff261ed84..9587330ed 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.5 javacTarget=1.5 # Release Configuration -releaseVersion=0.618 +releaseVersion=0.619 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index 96d17cc23..a1df98a0b 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -28,4 +28,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in network.unit.update.location3 = http://www.yacystats.de/yacybuild/ # properties for in-protocol response authentication: -network.unit.protocol.control = uncontrolled \ No newline at end of file +network.unit.protocol.control = uncontrolled + +# white/blacklists +network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost +network.unit.access.blacklist = \ No newline at end of file diff --git a/defaults/yacy.network.intranet.unit b/defaults/yacy.network.intranet.unit index 947c360e4..be7a80b2b 100644 --- a/defaults/yacy.network.intranet.unit +++ b/defaults/yacy.network.intranet.unit @@ -23,4 +23,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in network.unit.update.location3 = http://www.yacystats.de/yacybuild/ # properties for in-protocol response authentication: -network.unit.protocol.control = uncontrolled \ No newline at end of file +network.unit.protocol.control = uncontrolled + +# white/blacklists +network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost +network.unit.access.blacklist = \ No newline at end of file diff --git a/defaults/yacy.network.readme b/defaults/yacy.network.readme index dd97be776..5db8ad267 100644 --- a/defaults/yacy.network.readme +++ b/defaults/yacy.network.readme @@ -42,3 +42,19 @@ # network.unit.administration.manager = # network.unit.administration.request.authentication.method = 'salted-magic' # network.unit.administration.request.authentication.essentials = + +# two lists of wildcards may be used to control the limited or unlimited access to the +# peers and search functionality. By default, all clients from any IP may access the peers +# and use a limited amount of search requests. The number of requests is limited to apply a +# DoS functionality. The DoS check may be removed for a given number of IPs (separated by comma) +# or a number of IP patterns (patterns as defined by match operator in java); this IP-pattern list +# can be configured in the whitelist. +# In contrast also a list of IPs may be given to be not allowed to access the peers at all +# The YaCy-httpd will not respond on any request from IPs given in the blacklist +# By default both lists should be empty to provide a basic search functionality for everyone. +# When running a private YaCy network, all peers that are in the own networ should be configured +# in the whitelist. Othervise the access from whitelistes clients is allowed, but the executing +# search requests in the private network for a singe request from outside is also blocked. +# Put simply all IPs from you peers in the whitelist. +# network.unit.access.whitelist = +# network.unit.access.blacklist = diff --git a/defaults/yacy.network.webportal.unit b/defaults/yacy.network.webportal.unit index 943b2a652..adcc169d1 100644 --- a/defaults/yacy.network.webportal.unit +++ b/defaults/yacy.network.webportal.unit @@ -20,4 +20,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in network.unit.update.location3 = http://www.yacystats.de/yacybuild/ # properties for in-protocol response authentication: -network.unit.protocol.control = uncontrolled \ No newline at end of file +network.unit.protocol.control = uncontrolled + +# white/blacklists +network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost +network.unit.access.blacklist = \ No newline at end of file diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 6fa141fab..e36dd4e4a 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -45,6 +45,7 @@ import de.anomic.plasma.plasmaSnippetCache; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.server.serverCore; +import de.anomic.server.serverDomains; import de.anomic.server.serverMemory; import de.anomic.server.serverObjects; import de.anomic.server.serverProfiling; @@ -159,25 +160,37 @@ public class yacysearch { TreeSet trackerHandles = sb.localSearchTracker.get(client); if (trackerHandles == null) trackerHandles = new TreeSet(); boolean block = false; - if (global || fetchSnippets) { + if (serverDomains.matchesList(client, sb.networkBlacklist)) { + global = false; + fetchSnippets = false; + block = true; + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: BLACKLISTED CLIENT FROM " + client + " gets no permission to search"); + } else if (serverDomains.matchesList(client, sb.networkWhitelist)) { + serverLog.logInfo("LOCAL_SEARCH", "ACCECC CONTROL: WHITELISTED CLIENT FROM " + client + " gets no search restrictions"); + } else if (global || fetchSnippets) { // in case that we do a global search or we want to fetch snippets, we check for DoS cases - if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 1000)).size() > 2) { + int accInOneSecond = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 1000)).size(); + int accInThreeSeconds = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size(); + int accInOneMinute = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size(); + int accInTenMinutes = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size(); + if (accInTenMinutes > 600) { global = false; fetchSnippets = false; - } - if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size() > 1) { + block = true; + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInTenMinutes + " searches in ten minutes, fully blocked (no results generated)"); + } else if (accInOneMinute > 200) { global = false; fetchSnippets = false; - } - if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size() > 200) { + block = true; + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInOneMinute + " searches in one minute, fully blocked (no results generated)"); + } else if (accInThreeSeconds > 1) { global = false; fetchSnippets = false; - block = true; - } - if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size() > 600) { + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + " searches in three seconds, blocked global search and snippets"); + } else if (accInOneSecond > 2) { global = false; fetchSnippets = false; - block = true; + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInOneSecond + " searches in one second, blocked global search and snippets"); } } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 2f630ee1d..333835a63 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -99,6 +99,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Hashtable; import java.util.Iterator; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; @@ -107,6 +108,7 @@ import java.util.Timer; import java.util.TimerTask; import java.util.TreeMap; import java.util.TreeSet; +import java.util.regex.Pattern; import de.anomic.crawler.CrawlEntry; import de.anomic.crawler.CrawlProfile; @@ -236,6 +238,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used public URLLicense licensedURLs; public Timer moreMemory; + public List networkWhitelist, networkBlacklist; public serverProcessor indexingDocumentProcessor; public serverProcessor indexingCondensementProcessor; @@ -520,12 +523,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch nameCacheHit = new ConcurrentHashMap(); // a not-synchronized map resulted in deadlocks private static final Set nameCacheMiss = Collections.synchronizedSet(new HashSet()); private static final int maxNameCacheHitSize = 8000; private static final int maxNameCacheMissSize = 8000; - public static final List nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList()); + public static List nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList()); + public static final List localhostPatterns = makePatterns(localPatterns); private static final Set nameCacheNoCachingList = Collections.synchronizedSet(new HashSet()); - + /** * ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! ! * @@ -415,6 +419,26 @@ public class serverDomains { throw new UnknownHostException("host not in cache"); } + public static void setNoCachingPatterns(String patternList) { + nameCacheNoCachingPatterns = makePatterns(patternList); + } + + public static List makePatterns(String patternList) { + final String[] entries = patternList.split(","); + final List patterns = new ArrayList(entries.length); + for (int i = 0; i < entries.length; i++) { + patterns.add(Pattern.compile(entries[i].trim())); + } + return patterns; + } + + public static boolean matchesList(String obj, List patterns) { + for (Pattern nextPattern: patterns) { + if (nextPattern.matcher(obj).matches()) return true; + } + return false; + } + public static InetAddress dnsResolve(String host) { if ((host == null) || (host.length() == 0)) return null; host = host.toLowerCase().trim(); @@ -434,17 +458,10 @@ public class serverDomains { ) { doCaching = false; } else { - final Iterator noCachingPatternIter = nameCacheNoCachingPatterns.iterator(); - String nextPattern; - while (noCachingPatternIter.hasNext()) { - nextPattern = noCachingPatternIter.next(); - if (host.matches(nextPattern)) { - // disallow dns caching for this host - nameCacheNoCachingList.add(host); - doCaching = false; - break; - } - } + if (matchesList(host, nameCacheNoCachingPatterns)) { + nameCacheNoCachingList.add(host); + doCaching = false; + } } if (doCaching && ip != null) { @@ -533,29 +550,7 @@ public class serverDomains { // FIXME IPv4 only // check local ip addresses - if (host.equals("localhost") || host.startsWith("127") - || host.startsWith("192.168") - || host.startsWith("10.") - || host.startsWith("169.254") - || - // 172.16.0.0-172.31.255.255 (I think this is faster than a regex) - (host.startsWith("172.") && (host.startsWith("172.16.") - || host.startsWith("172.17.") - || host.startsWith("172.18.") - || host.startsWith("172.19.") - || host.startsWith("172.20.") - || host.startsWith("172.21.") - || host.startsWith("172.22.") - || host.startsWith("172.23.") - || host.startsWith("172.24.") - || host.startsWith("172.25.") - || host.startsWith("172.26.") - || host.startsWith("172.27.") - || host.startsWith("172.28.") - || host.startsWith("172.29.") - || host.startsWith("172.30.") - || host.startsWith("172.31.")))) - return true; + if (matchesList(host, localhostPatterns)) return true; // check the tld list final int p = host.lastIndexOf('.'); @@ -653,5 +648,4 @@ public class serverDomains { return null; } } - }