boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added an access control to the search interface using white/blacklists:

Browse Source 
in the network configuration, you can configure a whiteliste and a blacklist
- blacklistet clients cannot search
- whitelistet client get never any search restrictions
- for all other clients: apply DoS search restrictions
Please see the example configuriation in yacy.network.freeworld.unit
by default, all clients from localhosts get whitlistet.
If you have your own YaCy network, please put all the IPs of your peers into the whitelist

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5475 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent ac89e8e84d
commit 9ef77d57f5
9 changed files with 98 additions and 59 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5
# Release Configuration
releaseVersion=0.618
releaseVersion=0.619
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

6
defaults/yacy.network.freeworld.unit
Unescape View File

@ -28,4 +28,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in
network.unit.update.location3 = http://www.yacystats.de/yacybuild/
# properties for in-protocol response authentication:
network.unit.protocol.control = uncontrolled
network.unit.protocol.control = uncontrolled
# white/blacklists
network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost
network.unit.access.blacklist =

6
defaults/yacy.network.intranet.unit
Unescape View File

@ -23,4 +23,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in
network.unit.update.location3 = http://www.yacystats.de/yacybuild/
# properties for in-protocol response authentication:
network.unit.protocol.control = uncontrolled
network.unit.protocol.control = uncontrolled
# white/blacklists
network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost
network.unit.access.blacklist =

16
defaults/yacy.network.readme
Unescape View File

@ -42,3 +42,19 @@
# network.unit.administration.manager = <a .yacy or .yacyh - domain, naming the manager peer>
# network.unit.administration.request.authentication.method = 'salted-magic'
# network.unit.administration.request.authentication.essentials = <any string, a secret password>
# two lists of wildcards may be used to control the limited or unlimited access to the
# peers and search functionality. By default, all clients from any IP may access the peers
# and use a limited amount of search requests. The number of requests is limited to apply a
# DoS functionality. The DoS check may be removed for a given number of IPs (separated by comma)
# or a number of IP patterns (patterns as defined by match operator in java); this IP-pattern list
# can be configured in the whitelist.
# In contrast also a list of IPs may be given to be not allowed to access the peers at all
# The YaCy-httpd will not respond on any request from IPs given in the blacklist
# By default both lists should be empty to provide a basic search functionality for everyone.
# When running a private YaCy network, all peers that are in the own networ should be configured
# in the whitelist. Othervise the access from whitelistes clients is allowed, but the executing
# search requests in the private network for a singe request from outside is also blocked.
# Put simply all IPs from you peers in the whitelist.
# network.unit.access.whitelist =
# network.unit.access.blacklist =

6
defaults/yacy.network.webportal.unit
Unescape View File

@ -20,4 +20,8 @@ network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/in
network.unit.update.location3 = http://www.yacystats.de/yacybuild/
# properties for in-protocol response authentication:
network.unit.protocol.control = uncontrolled
network.unit.protocol.control = uncontrolled
# white/blacklists
network.unit.access.whitelist = 10.*,127.*,172.16.*,169.254.*,192.168.*,localhost
network.unit.access.blacklist =

33
htroot/yacysearch.java
Unescape View File

@ -45,6 +45,7 @@ import de.anomic.plasma.plasmaSnippetCache;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverCore;
import de.anomic.server.serverDomains;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverProfiling;
@ -159,25 +160,37 @@ public class yacysearch {
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (global || fetchSnippets) {
if (serverDomains.matchesList(client, sb.networkBlacklist)) {
global = false;
fetchSnippets = false;
block = true;
serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: BLACKLISTED CLIENT FROM " + client + " gets no permission to search");
} else if (serverDomains.matchesList(client, sb.networkWhitelist)) {
serverLog.logInfo("LOCAL_SEARCH", "ACCECC CONTROL: WHITELISTED CLIENT FROM " + client + " gets no search restrictions");
} else if (global || fetchSnippets) {
// in case that we do a global search or we want to fetch snippets, we check for DoS cases
if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 1000)).size() > 2) {
int accInOneSecond = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 1000)).size();
int accInThreeSeconds = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size();
int accInOneMinute = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size();
int accInTenMinutes = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size();
if (accInTenMinutes > 600) {
global = false;
fetchSnippets = false;
}
if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size() > 1) {
block = true;
serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInTenMinutes + " searches in ten minutes, fully blocked (no results generated)");
} else if (accInOneMinute > 200) {
global = false;
fetchSnippets = false;
}
if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size() > 200) {
block = true;
serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInOneMinute + " searches in one minute, fully blocked (no results generated)");
} else if (accInThreeSeconds > 1) {
global = false;
fetchSnippets = false;
block = true;
}
if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size() > 600) {
serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + " searches in three seconds, blocked global search and snippets");
} else if (accInOneSecond > 2) {
global = false;
fetchSnippets = false;
block = true;
serverLog.logWarning("LOCAL_SEARCH", "ACCECC CONTROL: CLIENT FROM " + client + ": " + accInOneSecond + " searches in one second, blocked global search and snippets");
}
}

15
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -99,6 +99,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
@ -107,6 +108,7 @@ import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern;
import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.CrawlProfile;
@ -236,6 +238,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
public TreeMap<String, String> clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used
public URLLicense licensedURLs;
public Timer moreMemory;
public List<Pattern> networkWhitelist, networkBlacklist;
public serverProcessor<indexingQueueEntry> indexingDocumentProcessor;
public serverProcessor<indexingQueueEntry> indexingCondensementProcessor;
@ -520,12 +523,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
rankingOtherDistribution = new plasmaRankingDistribution(log, webIndex.seedDB, new File(rankingPath, getConfig(plasmaSwitchboardConstants.RANKING_DIST_1_PATH, plasmaRankingDistribution.CR_OTHER)), (int) getConfigLong(plasmaSwitchboardConstants.RANKING_DIST_1_METHOD, plasmaRankingDistribution.METHOD_MIXEDSENIOR), (int) getConfigLong(plasmaSwitchboardConstants.RANKING_DIST_1_METHOD, 30), getConfig(plasmaSwitchboardConstants.RANKING_DIST_1_TARGET, "kaskelix.de:8080,yacy.dyndns.org:8000"));
// init nameCacheNoCachingList
final String noCachingList = getConfig(plasmaSwitchboardConstants.HTTPC_NAME_CACHE_CACHING_PATTERNS_NO,"");
final String[] noCachingEntries = noCachingList.split(",");
for (int i = 0; i < noCachingEntries.length; i++) {
final String entry = noCachingEntries[i].trim();
serverDomains.nameCacheNoCachingPatterns.add(entry);
}
serverDomains.setNoCachingPatterns(getConfig(plasmaSwitchboardConstants.HTTPC_NAME_CACHE_CACHING_PATTERNS_NO,""));
// generate snippets cache
log.logConfig("Initializing Snippet Cache");
@ -697,6 +695,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// initiate url license object
licensedURLs = new URLLicense(8);
// set white/blacklists
this.networkWhitelist = serverDomains.makePatterns(getConfig(plasmaSwitchboardConstants.NETWORK_WHITELIST, ""));
this.networkBlacklist = serverDomains.makePatterns(getConfig(plasmaSwitchboardConstants.NETWORK_BLACKLIST, ""));
/*
// in intranet and portal network set robinson mode
if (networkUnitDefinition.equals("defaults/yacy.network.webportal.unit") ||
@ -1203,7 +1205,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// get next queue entry and start a queue processing
final IndexingStack.QueueEntry queueEntry = deQueue();
assert queueEntry != null;
if (queueEntry == null) return true;
if (queueEntry.profile() == null) {
queueEntry.close();

3
source/de/anomic/plasma/plasmaSwitchboardConstants.java
Unescape View File

@ -522,4 +522,7 @@ public final class plasmaSwitchboardConstants {
*
*/
public static final String NETWORK_NAME = "network.unit.name";
public static final String NETWORK_WHITELIST = "network.unit.access.whitelist";
public static final String NETWORK_BLACKLIST = "network.unit.access.blacklist";
}

70
source/de/anomic/server/serverDomains.java
Unescape View File

@ -27,28 +27,32 @@ package de.anomic.server;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import de.anomic.plasma.plasmaSwitchboard;
public class serverDomains {
private static final String localPatterns = "10.*,127.*,172.16.*,169.254.*,192.168.*,localhost";
// a dns cache
private static final Map<String, InetAddress> nameCacheHit = new ConcurrentHashMap<String, InetAddress>(); // a not-synchronized map resulted in deadlocks
private static final Set<String> nameCacheMiss = Collections.synchronizedSet(new HashSet<String>());
private static final int maxNameCacheHitSize = 8000;
private static final int maxNameCacheMissSize = 8000;
public static final List<String> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<String>());
public static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>());
public static final List<Pattern> localhostPatterns = makePatterns(localPatterns);
private static final Set<String> nameCacheNoCachingList = Collections.synchronizedSet(new HashSet<String>());
/**
* ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! !
*
@ -415,6 +419,26 @@ public class serverDomains {
throw new UnknownHostException("host not in cache");
}
public static void setNoCachingPatterns(String patternList) {
nameCacheNoCachingPatterns = makePatterns(patternList);
}
public static List<Pattern> makePatterns(String patternList) {
final String[] entries = patternList.split(",");
final List<Pattern> patterns = new ArrayList<Pattern>(entries.length);
for (int i = 0; i < entries.length; i++) {
patterns.add(Pattern.compile(entries[i].trim()));
}
return patterns;
}
public static boolean matchesList(String obj, List<Pattern> patterns) {
for (Pattern nextPattern: patterns) {
if (nextPattern.matcher(obj).matches()) return true;
}
return false;
}
public static InetAddress dnsResolve(String host) {
if ((host == null) || (host.length() == 0)) return null;
host = host.toLowerCase().trim();
@ -434,17 +458,10 @@ public class serverDomains {
) {
doCaching = false;
} else {
final Iterator<String> noCachingPatternIter = nameCacheNoCachingPatterns.iterator();
String nextPattern;
while (noCachingPatternIter.hasNext()) {
nextPattern = noCachingPatternIter.next();
if (host.matches(nextPattern)) {
// disallow dns caching for this host
nameCacheNoCachingList.add(host);
doCaching = false;
break;
}
}
if (matchesList(host, nameCacheNoCachingPatterns)) {
nameCacheNoCachingList.add(host);
doCaching = false;
}
}
if (doCaching && ip != null) {
@ -533,29 +550,7 @@ public class serverDomains {
// FIXME IPv4 only
// check local ip addresses
if (host.equals("localhost") || host.startsWith("127")
|| host.startsWith("192.168")
|| host.startsWith("10.")
|| host.startsWith("169.254")
||
// 172.16.0.0-172.31.255.255 (I think this is faster than a regex)
(host.startsWith("172.") && (host.startsWith("172.16.")
|| host.startsWith("172.17.")
|| host.startsWith("172.18.")
|| host.startsWith("172.19.")
|| host.startsWith("172.20.")
|| host.startsWith("172.21.")
|| host.startsWith("172.22.")
|| host.startsWith("172.23.")
|| host.startsWith("172.24.")
|| host.startsWith("172.25.")
|| host.startsWith("172.26.")
|| host.startsWith("172.27.")
|| host.startsWith("172.28.")
|| host.startsWith("172.29.")
|| host.startsWith("172.30.")
|| host.startsWith("172.31."))))
return true;
if (matchesList(host, localhostPatterns)) return true;
// check the tld list
final int p = host.lastIndexOf('.');
@ -653,5 +648,4 @@ public class serverDomains {
return null;
}
}
}

Write Preview
Loading…
Cancel
Save