diff --git a/build.properties b/build.properties index ff261ed84..9587330ed 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.5 javacTarget=1.5 # Release Configuration -releaseVersion=0.618 +releaseVersion=0.619 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index 96d17cc23..a1df98a0b 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -28,4 +28,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in network.unit.update.location3 = http://www.yacystats.de/yacybuild/ # properties for in-protocol response authentication: -network.unit.protocol.control = uncontrolled \ No newline at end of file +network.unit.protocol.control = uncontrolled + +# white/blacklists +network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost +network.unit.access.blacklist = \ No newline at end of file diff --git a/defaults/yacy.network.intranet.unit b/defaults/yacy.network.intranet.unit index 947c360e4..be7a80b2b 100644 --- a/defaults/yacy.network.intranet.unit +++ b/defaults/yacy.network.intranet.unit @@ -23,4 +23,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in network.unit.update.location3 = http://www.yacystats.de/yacybuild/ # properties for in-protocol response authentication: -network.unit.protocol.control = uncontrolled \ No newline at end of file +network.unit.protocol.control = uncontrolled + +# white/blacklists +network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost +network.unit.access.blacklist = \ No newline at end of file diff --git a/defaults/yacy.network.readme b/defaults/yacy.network.readme index dd97be776..5db8ad267 100644 --- a/defaults/yacy.network.readme +++ b/defaults/yacy.network.readme @@ -42,3 +42,19 @@ # network.unit.administration.manager = # network.unit.administration.request.authentication.method = 'salted-magic' # network.unit.administration.request.authentication.essentials = + +# two lists of wildcards may be used to control the limited or unlimited access to the +# peers and search functionality. By default, all clients from any IP may access the peers +# and use a limited amount of search requests. The number of requests is limited to apply a +# DoS functionality. The DoS check may be removed for a given number of IPs (separated by comma) +# or a number of IP patterns (patterns as defined by match operator in java); this IP-pattern list +# can be configured in the whitelist. +# In contrast also a list of IPs may be given to be not allowed to access the peers at all +# The YaCy-httpd will not respond on any request from IPs given in the blacklist +# By default both lists should be empty to provide a basic search functionality for everyone. +# When running a private YaCy network, all peers that are in the own networ should be configured +# in the whitelist. Othervise the access from whitelistes clients is allowed, but the executing +# search requests in the private network for a singe request from outside is also blocked. +# Put simply all IPs from you peers in the whitelist. +# network.unit.access.whitelist = +# network.unit.access.blacklist = diff --git a/defaults/yacy.network.webportal.unit b/defaults/yacy.network.webportal.unit index 943b2a652..adcc169d1 100644 --- a/defaults/yacy.network.webportal.unit +++ b/defaults/yacy.network.webportal.unit @@ -20,4 +20,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in network.unit.update.location3 = http://www.yacystats.de/yacybuild/ # properties for in-protocol response authentication: -network.unit.protocol.control = uncontrolled \ No newline at end of file +network.unit.protocol.control = uncontrolled + +# white/blacklists +network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost +network.unit.access.blacklist = \ No newline at end of file diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 6fa141fab..e36dd4e4a 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -45,6 +45,7 @@ import de.anomic.plasma.plasmaSnippetCache; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.server.serverCore; +import de.anomic.server.serverDomains; import de.anomic.server.serverMemory; import de.anomic.server.serverObjects; import de.anomic.server.serverProfiling; @@ -159,25 +160,37 @@ public class yacysearch { TreeSet trackerHandles = sb.localSearchTracker.get(client); if (trackerHandles == null) trackerHandles = new TreeSet(); boolean block = false; - if (global || fetchSnippets) { + if (serverDomains.matchesList(client, sb.networkBlacklist)) { + global = false; + fetchSnippets = false; + block = true; + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: BLACKLISTED CLIENT FROM " + client + " gets no permission to search"); + } else if (serverDomains.matchesList(client, sb.networkWhitelist)) { + serverLog.logInfo("LOCAL_SEARCH", "ACCECC CONTROL: WHITELISTED CLIENT FROM " + client + " gets no search restrictions"); + } else if (global || fetchSnippets) { // in case that we do a global search or we want to fetch snippets, we check for DoS cases - if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 1000)).size() > 2) { + int accInOneSecond = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 1000)).size(); + int accInThreeSeconds = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size(); + int accInOneMinute = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size(); + int accInTenMinutes = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size(); + if (accInTenMinutes > 600) { global = false; fetchSnippets = false; - } - if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size() > 1) { + block = true; + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInTenMinutes + " searches in ten minutes, fully blocked (no results generated)"); + } else if (accInOneMinute > 200) { global = false; fetchSnippets = false; - } - if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size() > 200) { + block = true; + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInOneMinute + " searches in one minute, fully blocked (no results generated)"); + } else if (accInThreeSeconds > 1) { global = false; fetchSnippets = false; - block = true; - } - if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size() > 600) { + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + " searches in three seconds, blocked global search and snippets"); + } else if (accInOneSecond > 2) { global = false; fetchSnippets = false; - block = true; + serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInOneSecond + " searches in one second, blocked global search and snippets"); } } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 2f630ee1d..333835a63 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -99,6 +99,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Hashtable; import java.util.Iterator; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; @@ -107,6 +108,7 @@ import java.util.Timer; import java.util.TimerTask; import java.util.TreeMap; import java.util.TreeSet; +import java.util.regex.Pattern; import de.anomic.crawler.CrawlEntry; import de.anomic.crawler.CrawlProfile; @@ -236,6 +238,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used public URLLicense licensedURLs; public Timer moreMemory; + public List networkWhitelist, networkBlacklist; public serverProcessor indexingDocumentProcessor; public serverProcessor indexingCondensementProcessor; @@ -520,12 +523,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch nameCacheHit = new ConcurrentHashMap(); // a not-synchronized map resulted in deadlocks private static final Set nameCacheMiss = Collections.synchronizedSet(new HashSet()); private static final int maxNameCacheHitSize = 8000; private static final int maxNameCacheMissSize = 8000; - public static final List nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList()); + public static List nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList()); + public static final List localhostPatterns = makePatterns(localPatterns); private static final Set nameCacheNoCachingList = Collections.synchronizedSet(new HashSet()); - + /** * ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! ! * @@ -415,6 +419,26 @@ public class serverDomains { throw new UnknownHostException("host not in cache"); } + public static void setNoCachingPatterns(String patternList) { + nameCacheNoCachingPatterns = makePatterns(patternList); + } + + public static List makePatterns(String patternList) { + final String[] entries = patternList.split(","); + final List patterns = new ArrayList(entries.length); + for (int i = 0; i < entries.length; i++) { + patterns.add(Pattern.compile(entries[i].trim())); + } + return patterns; + } + + public static boolean matchesList(String obj, List patterns) { + for (Pattern nextPattern: patterns) { + if (nextPattern.matcher(obj).matches()) return true; + } + return false; + } + public static InetAddress dnsResolve(String host) { if ((host == null) || (host.length() == 0)) return null; host = host.toLowerCase().trim(); @@ -434,17 +458,10 @@ public class serverDomains { ) { doCaching = false; } else { - final Iterator noCachingPatternIter = nameCacheNoCachingPatterns.iterator(); - String nextPattern; - while (noCachingPatternIter.hasNext()) { - nextPattern = noCachingPatternIter.next(); - if (host.matches(nextPattern)) { - // disallow dns caching for this host - nameCacheNoCachingList.add(host); - doCaching = false; - break; - } - } + if (matchesList(host, nameCacheNoCachingPatterns)) { + nameCacheNoCachingList.add(host); + doCaching = false; + } } if (doCaching && ip != null) { @@ -533,29 +550,7 @@ public class serverDomains { // FIXME IPv4 only // check local ip addresses - if (host.equals("localhost") || host.startsWith("127") - || host.startsWith("192.168") - || host.startsWith("10.") - || host.startsWith("169.254") - || - // 172.16.0.0-172.31.255.255 (I think this is faster than a regex) - (host.startsWith("172.") && (host.startsWith("172.16.") - || host.startsWith("172.17.") - || host.startsWith("172.18.") - || host.startsWith("172.19.") - || host.startsWith("172.20.") - || host.startsWith("172.21.") - || host.startsWith("172.22.") - || host.startsWith("172.23.") - || host.startsWith("172.24.") - || host.startsWith("172.25.") - || host.startsWith("172.26.") - || host.startsWith("172.27.") - || host.startsWith("172.28.") - || host.startsWith("172.29.") - || host.startsWith("172.30.") - || host.startsWith("172.31.")))) - return true; + if (matchesList(host, localhostPatterns)) return true; // check the tld list final int p = host.lastIndexOf('.'); @@ -653,5 +648,4 @@ public class serverDomains { return null; } } - }