|
|
|
@ -1,9 +1,12 @@
|
|
|
|
|
package net.yacy.repository;
|
|
|
|
|
|
|
|
|
|
import java.util.HashMap;
|
|
|
|
|
import java.util.HashSet;
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
import java.util.Set;
|
|
|
|
|
import java.util.regex.Pattern;
|
|
|
|
|
import net.yacy.cora.document.id.Punycode;
|
|
|
|
|
|
|
|
|
|
import org.junit.Assert;
|
|
|
|
|
import org.junit.Test;
|
|
|
|
|
import static org.junit.Assert.*;
|
|
|
|
|
|
|
|
|
@ -14,7 +17,7 @@ public class BlacklistTest {
|
|
|
|
|
* needed and works
|
|
|
|
|
*/
|
|
|
|
|
@Test
|
|
|
|
|
public void testContains() throws Punycode.PunycodeException {
|
|
|
|
|
public void testContains() {
|
|
|
|
|
String path = ".*"; // simplest test pattern
|
|
|
|
|
|
|
|
|
|
Pattern pattern = Pattern.compile(path, Pattern.CASE_INSENSITIVE);
|
|
|
|
@ -39,5 +42,63 @@ public class BlacklistTest {
|
|
|
|
|
}
|
|
|
|
|
assertTrue("match blacklist pattern " + path, ret);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Tests static Blacklist.isListed() function with some sample patterns.
|
|
|
|
|
*/
|
|
|
|
|
@Test
|
|
|
|
|
public void testIsListed() {
|
|
|
|
|
final Map<String, Set<Pattern>> blacklistMapMatched = new HashMap<>();
|
|
|
|
|
Set<Pattern> patterns = new HashSet<>();
|
|
|
|
|
patterns.add(Pattern.compile(".*"));
|
|
|
|
|
blacklistMapMatched.put("stats.example.com", patterns);
|
|
|
|
|
|
|
|
|
|
patterns = new HashSet<>();
|
|
|
|
|
patterns.add(Pattern.compile(".*"));
|
|
|
|
|
blacklistMapMatched.put("site.blacklisted.net", patterns);
|
|
|
|
|
|
|
|
|
|
patterns = new HashSet<>();
|
|
|
|
|
patterns.add(Pattern.compile("data/js/\\d*\\.js"));
|
|
|
|
|
blacklistMapMatched.put("js.blacklisted.org", patterns);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
patterns = new HashSet<>();
|
|
|
|
|
patterns.add(Pattern.compile(".*"));
|
|
|
|
|
blacklistMapMatched.put("ftp.*", patterns);
|
|
|
|
|
|
|
|
|
|
patterns = new HashSet<>();
|
|
|
|
|
patterns.add(Pattern.compile("bestenlisten/.*"));
|
|
|
|
|
patterns.add(Pattern.compile("produkte/.*"));
|
|
|
|
|
blacklistMapMatched.put("esample.de", patterns);
|
|
|
|
|
|
|
|
|
|
final Map<String, Set<Pattern>> blacklistMapNotMatched = new HashMap<>();
|
|
|
|
|
patterns = new HashSet<>();
|
|
|
|
|
patterns.add(Pattern.compile(".*"));
|
|
|
|
|
blacklistMapNotMatched.put("mobil\\..*", patterns);
|
|
|
|
|
|
|
|
|
|
patterns = new HashSet<>();
|
|
|
|
|
patterns.add(Pattern.compile("counter\\?.*"));
|
|
|
|
|
blacklistMapNotMatched.put(".*samples.fr", patterns);
|
|
|
|
|
|
|
|
|
|
patterns = new HashSet<>();
|
|
|
|
|
patterns.add(Pattern.compile(".*\\.js"));
|
|
|
|
|
patterns.add(Pattern.compile(".*\\.jpg"));
|
|
|
|
|
patterns.add(Pattern.compile(".*BannerAd.*"));
|
|
|
|
|
patterns.add(Pattern.compile("(.*/)*search.*"));
|
|
|
|
|
patterns.add(Pattern.compile("(.*/)*bizad.*"));
|
|
|
|
|
patterns.add(Pattern.compile("(.*/)*member/.*"));
|
|
|
|
|
blacklistMapNotMatched.put(".*.*", patterns);
|
|
|
|
|
|
|
|
|
|
Assert.assertTrue(Blacklist.isListed("site.blacklisted.net", "", blacklistMapMatched, blacklistMapNotMatched));
|
|
|
|
|
Assert.assertTrue(Blacklist.isListed("site.blacklisted.net", "/index.html", blacklistMapMatched, blacklistMapNotMatched));
|
|
|
|
|
Assert.assertTrue(Blacklist.isListed("mobil.news.fr", "/index.htm", blacklistMapMatched, blacklistMapNotMatched));
|
|
|
|
|
Assert.assertTrue(Blacklist.isListed("mobil.news.fr", "/news/latest.html", blacklistMapMatched, blacklistMapNotMatched));
|
|
|
|
|
Assert.assertTrue(Blacklist.isListed("fr.notblacklisted.org", "/script.js", blacklistMapMatched, blacklistMapNotMatched));
|
|
|
|
|
Assert.assertTrue(Blacklist.isListed("fr.notblacklisted.org", "/js/script.js", blacklistMapMatched, blacklistMapNotMatched));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Assert.assertFalse(Blacklist.isListed("fr.notblacklisted.org", "/index.html", blacklistMapMatched, blacklistMapNotMatched));
|
|
|
|
|
Assert.assertFalse(Blacklist.isListed("js.blacklisted.org", "/index.html", blacklistMapMatched, blacklistMapNotMatched));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|