Marc Nause
11 years ago
parent
d4f65833a1
commit
c97da1a0d8
@ -0,0 +1,47 @@
|
|||||||
|
package blacklists;
|
||||||
|
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.data.ListManager;
|
||||||
|
import net.yacy.data.WorkTables;
|
||||||
|
import net.yacy.repository.BlacklistHelper;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
|
||||||
|
public class add_entry_p {
|
||||||
|
|
||||||
|
private static final String RESULT_FAILURE = "0";
|
||||||
|
private static final String RESULT_SUCCESS = "1";
|
||||||
|
private static final String XML_ITEM_STATUS = "status";
|
||||||
|
private static final String KEY_NEW_ENTRY = "entry";
|
||||||
|
private static final String KEY_CURRENT_BLACKLIST = "blacklist";
|
||||||
|
|
||||||
|
public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||||
|
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
|
||||||
|
if (post.containsKey(KEY_CURRENT_BLACKLIST) && post.containsKey(KEY_NEW_ENTRY)) {
|
||||||
|
|
||||||
|
final String blacklistToUse = post.get(KEY_CURRENT_BLACKLIST, "").trim();
|
||||||
|
final String entry = post.get(KEY_NEW_ENTRY, "").trim();
|
||||||
|
|
||||||
|
// store this call as api call
|
||||||
|
ListManager.switchboard.tables.recordAPICall(post, "add_entry_p.xml", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + entry);
|
||||||
|
|
||||||
|
if (BlacklistHelper.addBlacklistEntry(blacklistToUse, entry, header) == null) {
|
||||||
|
prop.put(XML_ITEM_STATUS, RESULT_SUCCESS);
|
||||||
|
|
||||||
|
Switchboard.urlBlacklist.clear();
|
||||||
|
ListManager.reloadBlacklists();
|
||||||
|
} else {
|
||||||
|
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<?xml version='1.0' encoding="UTF-8" standalone='yes'?>
|
||||||
|
<status code="#(status)#error::ok#(/status)#" />
|
||||||
@ -0,0 +1,47 @@
|
|||||||
|
package blacklists;
|
||||||
|
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.data.ListManager;
|
||||||
|
import net.yacy.data.WorkTables;
|
||||||
|
import net.yacy.repository.BlacklistHelper;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
|
||||||
|
public class delete_entry_p {
|
||||||
|
|
||||||
|
private static final String RESULT_FAILURE = "0";
|
||||||
|
private static final String RESULT_SUCCESS = "1";
|
||||||
|
private static final String XML_ITEM_STATUS = "status";
|
||||||
|
private static final String KEY_NEW_ENTRY = "entry";
|
||||||
|
private static final String KEY_CURRENT_BLACKLIST = "blacklist";
|
||||||
|
|
||||||
|
public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||||
|
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
|
||||||
|
if (post.containsKey(KEY_CURRENT_BLACKLIST) && post.containsKey(KEY_NEW_ENTRY)) {
|
||||||
|
|
||||||
|
final String blacklistToUse = post.get(KEY_CURRENT_BLACKLIST, "").trim();
|
||||||
|
final String entry = post.get(KEY_NEW_ENTRY, "").trim();
|
||||||
|
|
||||||
|
// store this call as api call
|
||||||
|
ListManager.switchboard.tables.recordAPICall(post, "add_entry_p.xml", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + entry);
|
||||||
|
|
||||||
|
if (BlacklistHelper.deleteBlacklistEntry(blacklistToUse, entry, header) == null) {
|
||||||
|
prop.put(XML_ITEM_STATUS, RESULT_SUCCESS);
|
||||||
|
|
||||||
|
Switchboard.urlBlacklist.clear();
|
||||||
|
ListManager.reloadBlacklists();
|
||||||
|
} else {
|
||||||
|
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<?xml version='1.0' encoding="UTF-8" standalone='yes'?>
|
||||||
|
<status code="#(status)#error::ok#(/status)#" />
|
||||||
@ -0,0 +1,75 @@
|
|||||||
|
package blacklists;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.util.Collection;
|
||||||
|
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.data.ListManager;
|
||||||
|
import net.yacy.kelondro.util.FileUtils;
|
||||||
|
import net.yacy.repository.Blacklist;
|
||||||
|
import net.yacy.repository.Blacklist.BlacklistType;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
|
||||||
|
public class get_list_p {
|
||||||
|
|
||||||
|
private static final String ITEMS = "items";
|
||||||
|
private static final String POSTFIX_ITEM = "_item";
|
||||||
|
private static final String PREFIX_ITEMS = "items_";
|
||||||
|
private static final String SHARED = "shared";
|
||||||
|
private static final String NAME = "name";
|
||||||
|
private static final String TYPES = "types";
|
||||||
|
private static final String PREFIX_TYPES = "types_";
|
||||||
|
private static final String POSTFIX_VALUE = "_value";
|
||||||
|
private static final String POSTFIX_NAME = "_name";
|
||||||
|
private static final String TYPES_EXT = ".BlackLists";
|
||||||
|
private static final String BLACK_LISTS_SHARED = "BlackLists.Shared";
|
||||||
|
|
||||||
|
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||||
|
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
|
||||||
|
final Collection<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
|
||||||
|
|
||||||
|
final String blackListName = (post == null) ? "" : post.get("name", "");
|
||||||
|
|
||||||
|
int count;
|
||||||
|
if (dirlist != null) {
|
||||||
|
for (final String element : dirlist) {
|
||||||
|
if (element.equals(blackListName)) {
|
||||||
|
|
||||||
|
prop.putXML(NAME, element);
|
||||||
|
|
||||||
|
prop.put(SHARED, ListManager.listSetContains(BLACK_LISTS_SHARED, element));
|
||||||
|
|
||||||
|
int j = 0;
|
||||||
|
for (final BlacklistType type : BlacklistType.values()) {
|
||||||
|
prop.putXML(PREFIX_TYPES + j + POSTFIX_NAME, type.toString());
|
||||||
|
prop.put(PREFIX_TYPES + j + POSTFIX_VALUE,
|
||||||
|
ListManager.listSetContains(type + TYPES_EXT, element));
|
||||||
|
j++;
|
||||||
|
}
|
||||||
|
prop.put(TYPES, BlacklistType.values().length);
|
||||||
|
|
||||||
|
prop.putXML(NAME, element);
|
||||||
|
|
||||||
|
final Collection<String> list = FileUtils.getListArray(new File(ListManager.listsPath, element));
|
||||||
|
|
||||||
|
count = 0;
|
||||||
|
for (final String entry : list){
|
||||||
|
if (entry.isEmpty()) continue;
|
||||||
|
if (entry.charAt(0) == '#') continue;
|
||||||
|
|
||||||
|
prop.putXML(PREFIX_ITEMS + count + POSTFIX_ITEM, entry);
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
prop.put(ITEMS, count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
<?xml version="1.0"?>
|
||||||
|
<list name="#[name]#" shared="#[shared]#" #{types}#
|
||||||
|
#[name]#="#[value]#"#{/types}#>
|
||||||
|
<items>
|
||||||
|
#{items}#
|
||||||
|
<item>#[item]#</item>
|
||||||
|
#{/items}#
|
||||||
|
</items>
|
||||||
|
</list>
|
||||||
@ -0,0 +1,55 @@
|
|||||||
|
package blacklists;
|
||||||
|
|
||||||
|
import java.util.Collection;
|
||||||
|
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.data.ListManager;
|
||||||
|
import net.yacy.kelondro.util.FileUtils;
|
||||||
|
import net.yacy.repository.Blacklist;
|
||||||
|
import net.yacy.repository.Blacklist.BlacklistType;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
|
||||||
|
public class get_metadata_p {
|
||||||
|
|
||||||
|
private static final String LISTS = "lists";
|
||||||
|
private static final String POSTFIX_SHARED = "_shared";
|
||||||
|
private static final String POSTFIX_TYPES = "_types";
|
||||||
|
private static final String POSTFIX_VALUE = "_value";
|
||||||
|
private static final String POSTFIX_NAME = "_name";
|
||||||
|
private static final String TYPES_EXT = ".BlackLists";
|
||||||
|
private static final String INFIX_TYPES = "_types_";
|
||||||
|
private static final String PREFIX_LISTS = "lists_";
|
||||||
|
private static final String BLACK_LISTS_SHARED = "BlackLists.Shared";
|
||||||
|
|
||||||
|
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||||
|
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
|
||||||
|
final Collection<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
|
||||||
|
int blacklistCount=0;
|
||||||
|
|
||||||
|
if (dirlist != null) {
|
||||||
|
for (final String element : dirlist) {
|
||||||
|
prop.putXML(PREFIX_LISTS + blacklistCount + POSTFIX_NAME, element);
|
||||||
|
|
||||||
|
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_SHARED, ListManager.listSetContains(BLACK_LISTS_SHARED, element));
|
||||||
|
|
||||||
|
int j = 0;
|
||||||
|
for (final BlacklistType type : BlacklistType.values()) {
|
||||||
|
prop.putXML(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_NAME, type.toString());
|
||||||
|
prop.put(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_VALUE,
|
||||||
|
ListManager.listSetContains(type + TYPES_EXT, element));
|
||||||
|
j++;
|
||||||
|
}
|
||||||
|
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_TYPES, BlacklistType.values().length);
|
||||||
|
|
||||||
|
blacklistCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prop.put(LISTS, blacklistCount);
|
||||||
|
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
<?xml version='1.0' encoding="UTF-8" standalone='yes'?>
|
||||||
|
<blacklists>
|
||||||
|
#{lists}#<list name="#[name]#" shared="#[shared]#" #{types}#
|
||||||
|
#[name]#="#[value]#"#{/types}# />
|
||||||
|
#{/lists}#
|
||||||
|
</blacklists>
|
||||||
@ -0,0 +1,129 @@
|
|||||||
|
package net.yacy.repository;
|
||||||
|
|
||||||
|
import net.yacy.cora.document.id.Punycode.PunycodeException;
|
||||||
|
import net.yacy.cora.protocol.HeaderFramework;
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.cora.util.ConcurrentLog;
|
||||||
|
import net.yacy.data.ListManager;
|
||||||
|
import net.yacy.repository.Blacklist.BlacklistType;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.search.query.SearchEventCache;
|
||||||
|
|
||||||
|
public final class BlacklistHelper {
|
||||||
|
|
||||||
|
/** Used for logging. */
|
||||||
|
public static final String APP_NAME = "Blacklist";
|
||||||
|
|
||||||
|
/** Private constructor to avoid instantiation of static helper class. */
|
||||||
|
private BlacklistHelper() {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds a new entry to the chosen blacklist.
|
||||||
|
* @param blacklistToUse the name of the blacklist the entry is to be added to
|
||||||
|
* @param newEntry the entry that is to be added
|
||||||
|
* @param header
|
||||||
|
* @param supportedBlacklistTypes
|
||||||
|
* @return null if no error occurred, else a String to put into LOCATION
|
||||||
|
*/
|
||||||
|
public static String addBlacklistEntry(
|
||||||
|
final String blacklistToUse,
|
||||||
|
final String entry,
|
||||||
|
final RequestHeader header) {
|
||||||
|
String newEntry = entry;
|
||||||
|
|
||||||
|
if (blacklistToUse == null || blacklistToUse.isEmpty()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (newEntry == null || newEntry.isEmpty()) {
|
||||||
|
return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ignore empty entries
|
||||||
|
if(newEntry == null || newEntry.isEmpty()) {
|
||||||
|
ConcurrentLog.warn(APP_NAME, "skipped adding an empty entry");
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (newEntry.startsWith("http://") ){
|
||||||
|
newEntry = newEntry.substring(7);
|
||||||
|
} else if (newEntry.startsWith("https://")) {
|
||||||
|
newEntry = newEntry.substring(8);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (newEntry.indexOf("*") < 0) {
|
||||||
|
// user did not use any wild cards and just submitted a word
|
||||||
|
|
||||||
|
newEntry = ".*" + newEntry + ".*/.*";
|
||||||
|
newEntry = ".*.*/.*" + newEntry + ".*";
|
||||||
|
|
||||||
|
} else {
|
||||||
|
|
||||||
|
int pos = newEntry.indexOf('/',0);
|
||||||
|
if (pos < 0) {
|
||||||
|
// add default empty path pattern
|
||||||
|
newEntry = newEntry + "/.*";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int pos = newEntry.indexOf('/',0);
|
||||||
|
String host = newEntry.substring(0, pos);
|
||||||
|
String path = newEntry.substring(pos + 1);
|
||||||
|
|
||||||
|
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
|
||||||
|
if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistToUse)) {
|
||||||
|
try {
|
||||||
|
Switchboard.urlBlacklist.add(supportedBlacklistType, blacklistToUse, host, path);
|
||||||
|
} catch (PunycodeException e) {
|
||||||
|
ConcurrentLog.warn(APP_NAME, "Unable to add blacklist entry to blacklist " + supportedBlacklistType, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SearchEventCache.cleanupEvents(true);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes a blacklist entry.
|
||||||
|
* @param blacklistToUse the name of the blacklist the entry is to be deleted from
|
||||||
|
* @param entry the entry that is to be deleted
|
||||||
|
* @param header
|
||||||
|
* @param supportedBlacklistTypes
|
||||||
|
* @return null if no error occurred, else a String to put into LOCATION
|
||||||
|
*/
|
||||||
|
public static String deleteBlacklistEntry(
|
||||||
|
final String blacklistToUse,
|
||||||
|
final String entry,
|
||||||
|
final RequestHeader header) {
|
||||||
|
String oldEntry = entry;
|
||||||
|
|
||||||
|
if (blacklistToUse == null || blacklistToUse.isEmpty()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (oldEntry == null || oldEntry.isEmpty()) {
|
||||||
|
return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// remove the entry from the running blacklist engine
|
||||||
|
int pos = oldEntry.indexOf('/',0);
|
||||||
|
String host = oldEntry.substring(0, pos);
|
||||||
|
String path = "";
|
||||||
|
if (pos > 0) {
|
||||||
|
path = oldEntry.substring(pos + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
|
||||||
|
if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) {
|
||||||
|
Switchboard.urlBlacklist.remove(supportedBlacklistType, blacklistToUse, host, path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SearchEventCache.cleanupEvents(true);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in new issue