boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added my Blacklistengines

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4608 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
lulabad 17 years ago
parent bca87f1e38
commit 9fb5d661f2
5 changed files with 600 additions and 1 deletions
Show Stats Download Patch File Download Diff File

2
defaults/yacy.init
Unescape View File

@ -279,7 +279,7 @@ proxyYellowList=yacy.yellow
# the black-list; URLs appearing in this list will not be loaded; # the black-list; URLs appearing in this list will not be loaded;
# instead always a 404 is returned # instead always a 404 is returned
# all these files will be placed in the listsPath # all these files will be placed in the listsPath
BlackLists.class=de.anomic.plasma.urlPattern.defaultURLPattern BlackLists.class=de.anomic.index.indexDefaultReferenceBlacklist
BlackLists.Shared=url.default.black BlackLists.Shared=url.default.black
BlackLists.DefaultList=url.default.black BlackLists.DefaultList=url.default.black

164
source/de/anomic/index/indexAdvancedDefaultReferenceBlacklist.java
Unescape View File

@ -0,0 +1,164 @@
// advancedURLPattern.java
// -----------------------
// part of YaCy
// (C) by Alexander Fieger webmaster@lulabad.de
// first published on http://www.lulabad.de
// Ingolstadt, Germany, 2006
// last major change: 12.08.2006
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.index;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
public class indexAdvancedDefaultReferenceBlacklist extends indexAbstractReferenceBlacklist implements indexReferenceBlacklist {
public indexAdvancedDefaultReferenceBlacklist(File rootPath) {
super(rootPath);
}
public String getEngineInfo() {
return "Advanced YaCy Blacklist Engine by lulabad";
}
public boolean isListed(String blacklistType, String hostlow, String path) {
if (hostlow == null) throw new NullPointerException();
if (path == null) throw new NullPointerException();
// getting the proper blacklist
HashMap blacklistMap = super.getBlacklistMap(blacklistType);
if (path.length() > 0 && path.charAt(0) == '/') path = path.substring(1);
String pp = ""; // path-pattern
String tmp1 = "";
String tmp2 = "";
ArrayList app;
boolean matched = false;
String[] result = hostlow.split("\\.");
for (int x=0; x<result.length; x++) {
if ((app = (ArrayList) blacklistMap.get("*." + result[x])) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get(result[x] + ".*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get("*." + result[x] + ".*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
tmp1 = tmp1.concat("." + result[x]);
tmp2 = result[result.length-1-x].concat("."+ tmp2);
if ((app = (ArrayList) blacklistMap.get(tmp1.substring(1))) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get("*" + tmp1)) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get(tmp1.substring(1) + ".*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get("*" + tmp1 + ".*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get(tmp2.substring(0,tmp2.length()-1))) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get("*." + tmp2.substring(0,tmp2.length()-1))) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get(tmp2 + "*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
if ((app = (ArrayList) blacklistMap.get("*." + tmp2 + "*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return matched;
}
}
return false;
}
}

164
source/de/anomic/index/indexAdvancedDefaultWhiteReferenceBlacklist.java
Unescape View File

@ -0,0 +1,164 @@
// advancedURLPattern.java
// -----------------------
// part of YaCy
// (C) by Alexander Fieger webmaster@lulabad.de
// first published on http://www.lulabad.de
// Ingolstadt, Germany, 2006
// last major change: 12.08.2006
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.index;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
public class indexAdvancedDefaultWhiteReferenceBlacklist extends indexAbstractReferenceBlacklist implements indexReferenceBlacklist {
public indexAdvancedDefaultWhiteReferenceBlacklist(File rootPath) {
super(rootPath);
}
public String getEngineInfo() {
return "Advanced YaCy Whitelist Engine by lulabad";
}
public boolean isListed(String blacklistType, String hostlow, String path) {
if (hostlow == null) throw new NullPointerException();
if (path == null) throw new NullPointerException();
// getting the proper blacklist
HashMap blacklistMap = super.getBlacklistMap(blacklistType);
if (path.length() > 0 && path.charAt(0) == '/') path = path.substring(1);
String pp = ""; // path-pattern
String tmp1 = "";
String tmp2 = "";
ArrayList app;
boolean matched = true;
String[] result = hostlow.split("\\.");
for (int x=0; x<result.length; x++) {
if ((app = (ArrayList) blacklistMap.get("*." + result[x])) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get(result[x] + ".*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get("*." + result[x] + ".*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched |= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
tmp1 = tmp1.concat("." + result[x]);
tmp2 = result[result.length-1-x].concat("."+ tmp2);
if ((app = (ArrayList) blacklistMap.get(tmp1.substring(1))) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get("*" + tmp1)) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get(tmp1.substring(1) + ".*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get("*" + tmp1 + ".*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get(tmp2.substring(0,tmp2.length()-1))) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get("*." + tmp2.substring(0,tmp2.length()-1))) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get(tmp2 + "*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
if ((app = (ArrayList) blacklistMap.get("*." + tmp2 + "*")) != null) {
for (int i=app.size()-1; !matched && i>-1; i--) {
pp = (String)app.get(i);
matched ^= ((pp.equals("*")) || (path.matches(pp)));
}
return !matched;
}
}
return true;
}
}

136
source/de/anomic/index/indexRegexReferenceBlacklist.java
Unescape View File

@ -0,0 +1,136 @@
// advancedURLPattern.java
// -----------------------
// part of YaCy
// (C) by Alexander Fieger webmaster@lulabad.de
// first published on http://www.lulabad.de
// Ingolstadt, Germany, 2006
// last major change: 12.08.2006
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.index;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.regex.PatternSyntaxException;
import de.anomic.kelondro.kelondroMSetTools;
public class indexRegexReferenceBlacklist extends indexAbstractReferenceBlacklist implements indexReferenceBlacklist {
public indexRegexReferenceBlacklist(File rootPath) {
super(rootPath);
}
public String getEngineInfo() {
return "Regex YaCy Blacklist Engine by lulabad";
}
public boolean isListed(String blacklistType, String hostlow, String path) {
if (hostlow == null) throw new NullPointerException();
if (path == null) throw new NullPointerException();
// getting the proper blacklist
HashMap blacklistMap = super.getBlacklistMap(blacklistType);
ArrayList app;
if (path.length() > 0 && path.charAt(0) == '/') path = path.substring(1);
Iterator it = blacklistMap.entrySet().iterator();
while( it.hasNext() ) {
Map.Entry me = (Map.Entry)it.next();
try {
Matcher domainregex = Pattern.compile((String)me.getKey()).matcher(hostlow);
app = (ArrayList) me.getValue();
for (int i=0; i<app.size(); i++) {
Matcher pathregex = Pattern.compile((String)app.get(i)).matcher(path);
if ((domainregex.matches()) && (pathregex.matches())) return true;
}
} catch (PatternSyntaxException e) {}
}
return false;
}
public void add(String blacklistType, String host, String path) {
if (host == null) throw new NullPointerException();
if (path == null) throw new NullPointerException();
if (path.length() > 0 && path.charAt(0) == '/') path = path.substring(1);
StringBuffer sb = new StringBuffer(host);
if(sb.charAt(0) == '*') sb.insert(0,'.');
HashMap<String, ArrayList<String>> blacklistMap = super.getBlacklistMap(blacklistType);
ArrayList<String> hostList = blacklistMap.get(sb.toString().toLowerCase());
if (hostList == null)
blacklistMap.put(sb.toString().toLowerCase(), (hostList = new ArrayList<String>()));
hostList.add(path);
}
public void remove(String blacklistType, String host, String path) {
StringBuffer sb = new StringBuffer(host);
if(sb.charAt(0) == '*') sb.insert(0,'.');
HashMap blacklistMap = getBlacklistMap(blacklistType);
ArrayList hostList = (ArrayList)blacklistMap.get(sb.toString());
hostList.remove(path);
if (hostList.size() == 0)
blacklistMap.remove(sb.toString());
}
public void loadList(String blacklistType, String filenames, String sep) {
HashMap<String,ArrayList<String>> blacklistMapTmp = new HashMap<String,ArrayList<String>>();
String[] filenamesarray = filenames.split(",");
if( filenamesarray.length > 0) {
for (int i = 0; i < filenamesarray.length; i++) {
blacklistMapTmp.putAll(kelondroMSetTools.loadMapMultiValsPerKey(new File(this.blacklistRootPath, filenamesarray[i]).toString(), sep));
}
}
Iterator it = blacklistMapTmp.entrySet().iterator();
ArrayList app;
while( it.hasNext() ) {
Map.Entry me = (Map.Entry)it.next();
app = (ArrayList) me.getValue();
for (int i=0; i<app.size(); i++) {
this.add(blacklistType,(String) me.getKey(),(String) app.get(i));
}
}
}
}

135
source/de/anomic/index/indexRegexWhiteReferenceBlacklist.java
Unescape View File

@ -0,0 +1,135 @@
// advancedURLPattern.java
// -----------------------
// part of YaCy
// (C) by Alexander Fieger webmaster@lulabad.de
// first published on http://www.lulabad.de
// Ingolstadt, Germany, 2006
// last major change: 12.08.2006
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.index;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.regex.PatternSyntaxException;
import de.anomic.kelondro.kelondroMSetTools;
public class indexRegexWhiteReferenceBlacklist extends indexAbstractReferenceBlacklist implements indexReferenceBlacklist {
public indexRegexWhiteReferenceBlacklist(File rootPath) {
super(rootPath);
}
public String getEngineInfo() {
return "Regex YaCy Whitelist Engine by lulabad";
}
public boolean isListed(String blacklistType, String hostlow, String path) {
if (hostlow == null) throw new NullPointerException();
if (path == null) throw new NullPointerException();
// getting the proper blacklist
HashMap blacklistMap = super.getBlacklistMap(blacklistType);
ArrayList app;
if (path.length() > 0 && path.charAt(0) == '/') path = path.substring(1);
Iterator it = blacklistMap.entrySet().iterator();
while( it.hasNext() ) {
Map.Entry me = (Map.Entry)it.next();
try {
Matcher domainregex = Pattern.compile((String)me.getKey()).matcher(hostlow);
app = (ArrayList) me.getValue();
for (int i=0; i<app.size(); i++) {
Matcher pathregex = Pattern.compile((String)app.get(i)).matcher(path);
if ((domainregex.matches()) && (pathregex.matches())) return false;
}
} catch (PatternSyntaxException e) {}
}
return true;
}
public void add(String blacklistType, String host, String path) {
if (host == null) throw new NullPointerException();
if (path == null) throw new NullPointerException();
if (path.length() > 0 && path.charAt(0) == '/') path = path.substring(1);
StringBuffer sb = new StringBuffer(host);
if(sb.charAt(0) == '*') sb.insert(0,'.');
HashMap<String, ArrayList<String>> blacklistMap = super.getBlacklistMap(blacklistType);
ArrayList<String> hostList = blacklistMap.get(sb.toString().toLowerCase());
if (hostList == null)
blacklistMap.put(sb.toString().toLowerCase(), (hostList = new ArrayList<String>()));
hostList.add(path);
}
public void remove(String blacklistType, String host, String path) {
StringBuffer sb = new StringBuffer(host);
if(sb.charAt(0) == '*') sb.insert(0,'.');
HashMap blacklistMap = getBlacklistMap(blacklistType);
ArrayList hostList = (ArrayList)blacklistMap.get(sb.toString());
if(hostList != null)
hostList.remove(path);
if (hostList.size() == 0)
blacklistMap.remove(sb.toString());
}
public void loadList(String blacklistType, String filenames, String sep) {
HashMap<String,ArrayList<String>> blacklistMapTmp = new HashMap<String,ArrayList<String>>();
String[] filenamesarray = filenames.split(",");
if( filenamesarray.length > 0) {
for (int i = 0; i < filenamesarray.length; i++) {
blacklistMapTmp.putAll(kelondroMSetTools.loadMapMultiValsPerKey(new File(this.blacklistRootPath, filenamesarray[i]).toString(), sep));
}
}
Iterator it = blacklistMapTmp.entrySet().iterator();
ArrayList app;
while( it.hasNext() ) {
Map.Entry me = (Map.Entry)it.next();
app = (ArrayList) me.getValue();
for (int i=0; i<app.size(); i++) {
this.add(blacklistType,(String) me.getKey(),(String) app.get(i));
}
}
}
}
Write Preview
Loading…
Cancel
Save