boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'nutomics/blacklist_structure'

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent dea71851d2 44f8fcf62e
commit 07261fe274
2 changed files with 148 additions and 127 deletions
Show Stats Download Patch File Download Diff File

193
htroot/Blacklist_p.java
Unescape View File

@ -30,9 +30,7 @@
// if the shell's current path is HTROOT
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.util.Arrays;
import java.util.List;
@ -251,7 +249,7 @@ public class Blacklist_p {
if (selectedBlacklistEntries.length > 0) {
String temp = null;
for (final String selectedBlacklistEntry : selectedBlacklistEntries) {
if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) {
if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header)) != null) {
prop.put("LOCATION", temp);
return prop;
}
@ -273,7 +271,7 @@ public class Blacklist_p {
// store this call as api call
ListManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + blentry);
final String temp = addBlacklistEntry(blacklistToUse, blentry, header, BlacklistType.values());
final String temp = addBlacklistEntry(blacklistToUse, blentry, header);
if (temp != null) {
prop.put("LOCATION", temp);
return prop;
@ -300,12 +298,12 @@ public class Blacklist_p {
!targetBlacklist.equals(blacklistToUse)) {
String temp;
for (final String selectedBlacklistEntry : selectedBlacklistEntries) {
if ((temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntry, header, BlacklistType.values())) != null) {
if ((temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntry, header)) != null) {
prop.put("LOCATION", temp);
return prop;
}
if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header, BlacklistType.values())) != null) {
if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntry, header)) != null) {
prop.put("LOCATION", temp);
return prop;
@ -342,12 +340,12 @@ public class Blacklist_p {
if (!selectedBlacklistEntries[i].equals(editedBlacklistEntries[i])) {
if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, BlacklistType.values())) != null) {
if ((temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header)) != null) {
prop.put("LOCATION", temp);
return prop;
}
if ((temp = addBlacklistEntry(blacklistToUse, editedBlacklistEntries[i], header, BlacklistType.values())) != null) {
if ((temp = addBlacklistEntry(blacklistToUse, editedBlacklistEntries[i], header)) != null) {
prop.put("LOCATION", temp);
return prop;
}
@ -519,49 +517,19 @@ public class Blacklist_p {
return prop;
}
/**
* This method adds a new entry to the chosen blacklist.
* @param blacklistToUse the name of the blacklist the entry is to be added to
* @param newEntry the entry that is to be added
* @param header
* @param supportedBlacklistTypes
* @return null if no error occurred, else a String to put into LOCATION
*/
private static String addBlacklistEntry(
final String blacklistToUse,
final String newEntry,
final RequestHeader header,
final BlacklistType[] supportedBlacklistTypes) {
if (blacklistToUse == null || blacklistToUse.isEmpty()) {
return "";
}
if (newEntry == null || newEntry.isEmpty()) {
return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
}
addBlacklistEntry(ListManager.listsPath, blacklistToUse, newEntry, supportedBlacklistTypes);
SearchEventCache.cleanupEvents(true);
return null;
}
/**
* This method deletes a blacklist entry.
* @param blacklistToUse the name of the blacklist the entry is to be deleted from
* @param oldEntry the entry that is to be deleted
* @param entry the entry that is to be deleted
* @param header
* @param supportedBlacklistTypes
* @return null if no error occured, else a String to put into LOCATION
* @return null if no error occurred, else a String to put into LOCATION
*/
private static String deleteBlacklistEntry(
final String blacklistToUse,
final String oldEntry,
final RequestHeader header,
final BlacklistType[] supportedBlacklistTypes) {
final String entry,
final RequestHeader header) {
String oldEntry = entry;
if (blacklistToUse == null || blacklistToUse.isEmpty()) {
return "";
@ -571,69 +539,59 @@ public class Blacklist_p {
return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
}
deleteBlacklistEntry(ListManager.listsPath, blacklistToUse, oldEntry, supportedBlacklistTypes);
SearchEventCache.cleanupEvents(true);
return null;
}
/**
* This method deletes a blacklist entry.
* @param blacklistToUse the name of the blacklist the entry is to be deleted from
* @param oldEntry the entry that is to be deleted
* @param supportedBlacklistTypes
*/
private static void deleteBlacklistEntry(
final File listsPath,
final String blacklistToUse,
String oldEntry,
final BlacklistType[] supportedBlacklistTypes) {
// load blacklist data from file
final List<String> list = FileUtils.getListArray(new File(listsPath, blacklistToUse));
// delete the old entry from file
if (list != null) {
for (final String entry : list) {
if (entry.equals(oldEntry)) {
list.remove(entry);
break;
}
}
FileUtils.writeList(new File(listsPath, blacklistToUse), list.toArray(new String[list.size()]));
}
// remove the entry from the running blacklist engine
int pos = oldEntry.indexOf('/',0);
if (pos < 0) {
// add default empty path pattern
pos = oldEntry.length();
oldEntry = oldEntry + "/.*";
String host = oldEntry.substring(0, pos);
String path = "";
if (pos > 0) {
path = oldEntry.substring(pos + 1);
}
if (Switchboard.urlBlacklist.getFileName(BlacklistType.DHT) == blacklistToUse) {
Switchboard.urlBlacklist.remove(host, path);
}
for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) {
if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) {
Switchboard.urlBlacklist.remove(supportedBlacklistType,oldEntry.substring(0, pos), oldEntry.substring(pos + 1));
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) != blacklistToUse) {
continue;
}
Blacklist bl = new Blacklist(ListManager.listsPath);
bl.loadList(supportedBlacklistType, blacklistToUse, "/");
bl.remove(host, path);
}
SearchEventCache.cleanupEvents(true);
return null;
}
/**
* This method adds a new entry to the chosen blacklist.
* @param blacklistToUse the name of the blacklist the entry is to be added to
* @param newEntry the entry that is to be added
* @param header
* @param supportedBlacklistTypes
* @return null if no error occurred, else a String to put into LOCATION
*/
private static void addBlacklistEntry(
final File listsPath,
final String blacklistToUse,
String newEntry,
final BlacklistType[] supportedBlacklistTypes) {
private static String addBlacklistEntry(
final String blacklistToUse,
final String entry,
final RequestHeader header) {
String newEntry = entry;
if (blacklistToUse == null || blacklistToUse.isEmpty()) {
return "";
}
if (newEntry == null || newEntry.isEmpty()) {
return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
}
// ignore empty entries
if(newEntry == null || newEntry.isEmpty()) {
Log.logWarning("Blacklist", "skipped adding an empty entry");
return;
return "";
}
if (newEntry.startsWith("http://") ){
@ -645,8 +603,8 @@ public class Blacklist_p {
if (newEntry.indexOf("*") < 0) {
// user did not use any wild cards and just submitted a word
addBlacklistEntry0(listsPath, blacklistToUse, ".*" + newEntry + ".*/.*", supportedBlacklistTypes);
addBlacklistEntry0(listsPath, blacklistToUse, ".*.*/.*" + newEntry + ".*", supportedBlacklistTypes);
newEntry = ".*" + newEntry + ".*/.*";
newEntry = ".*.*/.*" + newEntry + ".*";
} else {
@ -655,44 +613,29 @@ public class Blacklist_p {
// add default empty path pattern
newEntry = newEntry + "/.*";
}
addBlacklistEntry0(listsPath, blacklistToUse, newEntry, supportedBlacklistTypes);
}
}
private static void addBlacklistEntry0(
final File listsPath,
final String blacklistToUse,
String newEntry,
final BlacklistType[] supportedBlacklistTypes) {
if (!Blacklist.blacklistFileContains(listsPath, blacklistToUse, newEntry)) {
// append the line to the file
PrintWriter pw = null;
try {
pw = new PrintWriter(new FileWriter(new File(listsPath, blacklistToUse), true));
pw.println(newEntry);
pw.close();
} catch (final IOException e) {
Log.logException(e);
} finally {
if (pw != null) {
try {
pw.close();
} catch (final Exception e) {
Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage());
}
int pos = newEntry.indexOf('/',0);
String host = newEntry.substring(0, pos);
String path = newEntry.substring(pos + 1);
}
}
// add to blacklist
int pos = newEntry.indexOf('/',0);
for (final BlacklistType supportedBlacklistType : supportedBlacklistTypes) {
if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistToUse)) {
Switchboard.urlBlacklist.add(supportedBlacklistType, newEntry.substring(0, pos), newEntry.substring(pos + 1));
}
if (Switchboard.urlBlacklist.getFileName(BlacklistType.DHT) == blacklistToUse) {
Switchboard.urlBlacklist.add(host, path);
}
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) != blacklistToUse) {
continue;
}
Blacklist bl = new Blacklist(ListManager.listsPath);
bl.loadList(supportedBlacklistType, blacklistToUse, "/");
bl.add(supportedBlacklistType, host, path);
}
SearchEventCache.cleanupEvents(true);
return null;
}
}

82
source/net/yacy/repository/Blacklist.java
Unescape View File

@ -28,15 +28,18 @@ package net.yacy.repository;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.PrintWriter;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Pattern;
@ -44,6 +47,7 @@ import java.util.regex.PatternSyntaxException;
import net.yacy.cora.storage.HandleSet;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.data.ListManager;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.data.meta.URIMetadataRow;
@ -92,6 +96,7 @@ public class Blacklist {
}
private File blacklistRootPath = null;
private Map<BlacklistType, String> blacklistFiles = new TreeMap<BlacklistType, String>();
private final ConcurrentMap<BlacklistType, HandleSet> cachedUrlHashs;
private final ConcurrentMap<BlacklistType, Map<String, Set<Pattern>>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
private final ConcurrentMap<BlacklistType, Map<String, Set<Pattern>>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
@ -148,6 +153,14 @@ public class Blacklist {
return this.cachedUrlHashs.get(blacklistType);
}
public final String getFileName(BlacklistType type) {
return blacklistFiles.get(type);
}
public final File getRootPath() {
return blacklistRootPath;
}
public final void clear() {
for (final Map<String, Set<Pattern>> entry : this.hostpaths_matchable.values()) {
entry.clear();
@ -158,6 +171,9 @@ public class Blacklist {
for (final HandleSet entry : this.cachedUrlHashs.values()) {
entry.clear();
}
blacklistFiles.clear();
blacklistRootPath = null;
}
public final int size() {
@ -177,7 +193,7 @@ public class Blacklist {
public final void loadList(final BlacklistFile[] blFiles, final String sep) {
for (final BlacklistFile blf : blFiles) {
loadList(blf.getType(), blf.getFileName(), sep);
loadList(blf, sep);
}
}
@ -188,6 +204,10 @@ public class Blacklist {
* @param sep
*/
private void loadList(final BlacklistFile blFile, final String sep) {
if (!blacklistFiles.containsKey(blFile.getType())) {
blacklistFiles.put(blFile.getType(), blFile.getFileName());
}
final Map<String, Set<Pattern>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true);
final Map<String, Set<Pattern>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false);
Set<Map.Entry<String, List<String>>> loadedBlacklist;
@ -250,6 +270,15 @@ public class Blacklist {
getBlacklistMap(blacklistType, false).remove(host);
}
/**
* Removes entry for all blacklist types.
*/
public final void remove(final String host, final String path) {
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path);
}
}
public final void remove(final BlacklistType blacklistType, final String host, final String path) {
final Map<String, Set<Pattern>> blacklistMap = getBlacklistMap(blacklistType, true);
@ -269,9 +298,35 @@ public class Blacklist {
blacklistMapNotMatch.remove(host);
}
}
// load blacklist data from file
final List<String> list = FileUtils.getListArray(new File(ListManager.listsPath, getFileName(blacklistType)));
// delete the old entry from file
if (list != null) {
for (final String e : list) {
if (e.equals(host + "/" + path)) {
list.remove(e);
break;
}
}
FileUtils.writeList(new File(ListManager.listsPath, getFileName(blacklistType)), list.toArray(new String[list.size()]));
}
}
/**
* Adds a new blacklist entry for all types.
*/
public final void add(final String host, final String path) {
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
add(supportedBlacklistType, host, path);
}
}
public final void add(final BlacklistType blacklistType, final String host, final String path) {
if (contains(blacklistType, host, path)) {
return;
}
if (host == null) {
throw new IllegalArgumentException("host may not be null");
}
@ -292,8 +347,31 @@ public class Blacklist {
if (!(blacklistMap.containsKey(h) && ((hostList = blacklistMap.get(h)) != null))) {
blacklistMap.put(h, (hostList = new HashSet<Pattern>()));
}
// Create add case insesitive regex
Pattern pattern = Pattern.compile("(?i)" + p);
hostList.add(pattern);
// append the line to the file
PrintWriter pw = null;
try {
pw = new PrintWriter(new FileWriter(new File(blacklistRootPath, getFileName(blacklistType)), true));
pw.println(pattern);
pw.close();
} catch (final IOException e) {
Log.logException(e);
} finally {
if (pw != null) {
try {
pw.close();
} catch (final Exception e) {
Log.logWarning("Blacklist", "could not close stream to " +
getFileName(blacklistType) + "! " + e.getMessage());
}
hostList.add(Pattern.compile("(?i)" + p)); // add case insesitive regex
}
}
}
public final int blacklistCacheSize() {

Write Preview
Loading…
Cancel
Save