Marc Nause
11 years ago
parent
d4f65833a1
commit
c97da1a0d8
@ -0,0 +1,47 @@
|
||||
package blacklists;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.data.ListManager;
|
||||
import net.yacy.data.WorkTables;
|
||||
import net.yacy.repository.BlacklistHelper;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class add_entry_p {
|
||||
|
||||
private static final String RESULT_FAILURE = "0";
|
||||
private static final String RESULT_SUCCESS = "1";
|
||||
private static final String XML_ITEM_STATUS = "status";
|
||||
private static final String KEY_NEW_ENTRY = "entry";
|
||||
private static final String KEY_CURRENT_BLACKLIST = "blacklist";
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
if (post.containsKey(KEY_CURRENT_BLACKLIST) && post.containsKey(KEY_NEW_ENTRY)) {
|
||||
|
||||
final String blacklistToUse = post.get(KEY_CURRENT_BLACKLIST, "").trim();
|
||||
final String entry = post.get(KEY_NEW_ENTRY, "").trim();
|
||||
|
||||
// store this call as api call
|
||||
ListManager.switchboard.tables.recordAPICall(post, "add_entry_p.xml", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + entry);
|
||||
|
||||
if (BlacklistHelper.addBlacklistEntry(blacklistToUse, entry, header) == null) {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_SUCCESS);
|
||||
|
||||
Switchboard.urlBlacklist.clear();
|
||||
ListManager.reloadBlacklists();
|
||||
} else {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||
}
|
||||
|
||||
} else {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||
}
|
||||
|
||||
return prop;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,2 @@
|
||||
<?xml version='1.0' encoding="UTF-8" standalone='yes'?>
|
||||
<status code="#(status)#error::ok#(/status)#" />
|
||||
@ -0,0 +1,47 @@
|
||||
package blacklists;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.data.ListManager;
|
||||
import net.yacy.data.WorkTables;
|
||||
import net.yacy.repository.BlacklistHelper;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class delete_entry_p {
|
||||
|
||||
private static final String RESULT_FAILURE = "0";
|
||||
private static final String RESULT_SUCCESS = "1";
|
||||
private static final String XML_ITEM_STATUS = "status";
|
||||
private static final String KEY_NEW_ENTRY = "entry";
|
||||
private static final String KEY_CURRENT_BLACKLIST = "blacklist";
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
if (post.containsKey(KEY_CURRENT_BLACKLIST) && post.containsKey(KEY_NEW_ENTRY)) {
|
||||
|
||||
final String blacklistToUse = post.get(KEY_CURRENT_BLACKLIST, "").trim();
|
||||
final String entry = post.get(KEY_NEW_ENTRY, "").trim();
|
||||
|
||||
// store this call as api call
|
||||
ListManager.switchboard.tables.recordAPICall(post, "add_entry_p.xml", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + entry);
|
||||
|
||||
if (BlacklistHelper.deleteBlacklistEntry(blacklistToUse, entry, header) == null) {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_SUCCESS);
|
||||
|
||||
Switchboard.urlBlacklist.clear();
|
||||
ListManager.reloadBlacklists();
|
||||
} else {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||
}
|
||||
|
||||
} else {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||
}
|
||||
|
||||
return prop;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,2 @@
|
||||
<?xml version='1.0' encoding="UTF-8" standalone='yes'?>
|
||||
<status code="#(status)#error::ok#(/status)#" />
|
||||
@ -0,0 +1,75 @@
|
||||
package blacklists;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Collection;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.data.ListManager;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.repository.Blacklist;
|
||||
import net.yacy.repository.Blacklist.BlacklistType;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class get_list_p {
|
||||
|
||||
private static final String ITEMS = "items";
|
||||
private static final String POSTFIX_ITEM = "_item";
|
||||
private static final String PREFIX_ITEMS = "items_";
|
||||
private static final String SHARED = "shared";
|
||||
private static final String NAME = "name";
|
||||
private static final String TYPES = "types";
|
||||
private static final String PREFIX_TYPES = "types_";
|
||||
private static final String POSTFIX_VALUE = "_value";
|
||||
private static final String POSTFIX_NAME = "_name";
|
||||
private static final String TYPES_EXT = ".BlackLists";
|
||||
private static final String BLACK_LISTS_SHARED = "BlackLists.Shared";
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
final Collection<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
|
||||
|
||||
final String blackListName = (post == null) ? "" : post.get("name", "");
|
||||
|
||||
int count;
|
||||
if (dirlist != null) {
|
||||
for (final String element : dirlist) {
|
||||
if (element.equals(blackListName)) {
|
||||
|
||||
prop.putXML(NAME, element);
|
||||
|
||||
prop.put(SHARED, ListManager.listSetContains(BLACK_LISTS_SHARED, element));
|
||||
|
||||
int j = 0;
|
||||
for (final BlacklistType type : BlacklistType.values()) {
|
||||
prop.putXML(PREFIX_TYPES + j + POSTFIX_NAME, type.toString());
|
||||
prop.put(PREFIX_TYPES + j + POSTFIX_VALUE,
|
||||
ListManager.listSetContains(type + TYPES_EXT, element));
|
||||
j++;
|
||||
}
|
||||
prop.put(TYPES, BlacklistType.values().length);
|
||||
|
||||
prop.putXML(NAME, element);
|
||||
|
||||
final Collection<String> list = FileUtils.getListArray(new File(ListManager.listsPath, element));
|
||||
|
||||
count = 0;
|
||||
for (final String entry : list){
|
||||
if (entry.isEmpty()) continue;
|
||||
if (entry.charAt(0) == '#') continue;
|
||||
|
||||
prop.putXML(PREFIX_ITEMS + count + POSTFIX_ITEM, entry);
|
||||
count++;
|
||||
}
|
||||
prop.put(ITEMS, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return prop;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
<?xml version="1.0"?>
|
||||
<list name="#[name]#" shared="#[shared]#" #{types}#
|
||||
#[name]#="#[value]#"#{/types}#>
|
||||
<items>
|
||||
#{items}#
|
||||
<item>#[item]#</item>
|
||||
#{/items}#
|
||||
</items>
|
||||
</list>
|
||||
@ -0,0 +1,55 @@
|
||||
package blacklists;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.data.ListManager;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.repository.Blacklist;
|
||||
import net.yacy.repository.Blacklist.BlacklistType;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class get_metadata_p {
|
||||
|
||||
private static final String LISTS = "lists";
|
||||
private static final String POSTFIX_SHARED = "_shared";
|
||||
private static final String POSTFIX_TYPES = "_types";
|
||||
private static final String POSTFIX_VALUE = "_value";
|
||||
private static final String POSTFIX_NAME = "_name";
|
||||
private static final String TYPES_EXT = ".BlackLists";
|
||||
private static final String INFIX_TYPES = "_types_";
|
||||
private static final String PREFIX_LISTS = "lists_";
|
||||
private static final String BLACK_LISTS_SHARED = "BlackLists.Shared";
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
final Collection<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
|
||||
int blacklistCount=0;
|
||||
|
||||
if (dirlist != null) {
|
||||
for (final String element : dirlist) {
|
||||
prop.putXML(PREFIX_LISTS + blacklistCount + POSTFIX_NAME, element);
|
||||
|
||||
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_SHARED, ListManager.listSetContains(BLACK_LISTS_SHARED, element));
|
||||
|
||||
int j = 0;
|
||||
for (final BlacklistType type : BlacklistType.values()) {
|
||||
prop.putXML(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_NAME, type.toString());
|
||||
prop.put(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_VALUE,
|
||||
ListManager.listSetContains(type + TYPES_EXT, element));
|
||||
j++;
|
||||
}
|
||||
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_TYPES, BlacklistType.values().length);
|
||||
|
||||
blacklistCount++;
|
||||
}
|
||||
}
|
||||
prop.put(LISTS, blacklistCount);
|
||||
|
||||
return prop;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
<?xml version='1.0' encoding="UTF-8" standalone='yes'?>
|
||||
<blacklists>
|
||||
#{lists}#<list name="#[name]#" shared="#[shared]#" #{types}#
|
||||
#[name]#="#[value]#"#{/types}# />
|
||||
#{/lists}#
|
||||
</blacklists>
|
||||
@ -0,0 +1,129 @@
|
||||
package net.yacy.repository;
|
||||
|
||||
import net.yacy.cora.document.id.Punycode.PunycodeException;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.data.ListManager;
|
||||
import net.yacy.repository.Blacklist.BlacklistType;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.query.SearchEventCache;
|
||||
|
||||
public final class BlacklistHelper {
|
||||
|
||||
/** Used for logging. */
|
||||
public static final String APP_NAME = "Blacklist";
|
||||
|
||||
/** Private constructor to avoid instantiation of static helper class. */
|
||||
private BlacklistHelper() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new entry to the chosen blacklist.
|
||||
* @param blacklistToUse the name of the blacklist the entry is to be added to
|
||||
* @param newEntry the entry that is to be added
|
||||
* @param header
|
||||
* @param supportedBlacklistTypes
|
||||
* @return null if no error occurred, else a String to put into LOCATION
|
||||
*/
|
||||
public static String addBlacklistEntry(
|
||||
final String blacklistToUse,
|
||||
final String entry,
|
||||
final RequestHeader header) {
|
||||
String newEntry = entry;
|
||||
|
||||
if (blacklistToUse == null || blacklistToUse.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (newEntry == null || newEntry.isEmpty()) {
|
||||
return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
|
||||
}
|
||||
|
||||
// ignore empty entries
|
||||
if(newEntry == null || newEntry.isEmpty()) {
|
||||
ConcurrentLog.warn(APP_NAME, "skipped adding an empty entry");
|
||||
return "";
|
||||
}
|
||||
|
||||
if (newEntry.startsWith("http://") ){
|
||||
newEntry = newEntry.substring(7);
|
||||
} else if (newEntry.startsWith("https://")) {
|
||||
newEntry = newEntry.substring(8);
|
||||
}
|
||||
|
||||
if (newEntry.indexOf("*") < 0) {
|
||||
// user did not use any wild cards and just submitted a word
|
||||
|
||||
newEntry = ".*" + newEntry + ".*/.*";
|
||||
newEntry = ".*.*/.*" + newEntry + ".*";
|
||||
|
||||
} else {
|
||||
|
||||
int pos = newEntry.indexOf('/',0);
|
||||
if (pos < 0) {
|
||||
// add default empty path pattern
|
||||
newEntry = newEntry + "/.*";
|
||||
}
|
||||
}
|
||||
|
||||
int pos = newEntry.indexOf('/',0);
|
||||
String host = newEntry.substring(0, pos);
|
||||
String path = newEntry.substring(pos + 1);
|
||||
|
||||
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
|
||||
if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistToUse)) {
|
||||
try {
|
||||
Switchboard.urlBlacklist.add(supportedBlacklistType, blacklistToUse, host, path);
|
||||
} catch (PunycodeException e) {
|
||||
ConcurrentLog.warn(APP_NAME, "Unable to add blacklist entry to blacklist " + supportedBlacklistType, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SearchEventCache.cleanupEvents(true);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a blacklist entry.
|
||||
* @param blacklistToUse the name of the blacklist the entry is to be deleted from
|
||||
* @param entry the entry that is to be deleted
|
||||
* @param header
|
||||
* @param supportedBlacklistTypes
|
||||
* @return null if no error occurred, else a String to put into LOCATION
|
||||
*/
|
||||
public static String deleteBlacklistEntry(
|
||||
final String blacklistToUse,
|
||||
final String entry,
|
||||
final RequestHeader header) {
|
||||
String oldEntry = entry;
|
||||
|
||||
if (blacklistToUse == null || blacklistToUse.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (oldEntry == null || oldEntry.isEmpty()) {
|
||||
return header.get(HeaderFramework.CONNECTION_PROP_PATH) + "?selectList=&selectedListName=" + blacklistToUse;
|
||||
}
|
||||
|
||||
|
||||
// remove the entry from the running blacklist engine
|
||||
int pos = oldEntry.indexOf('/',0);
|
||||
String host = oldEntry.substring(0, pos);
|
||||
String path = "";
|
||||
if (pos > 0) {
|
||||
path = oldEntry.substring(pos + 1);
|
||||
}
|
||||
|
||||
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
|
||||
if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) {
|
||||
Switchboard.urlBlacklist.remove(supportedBlacklistType, blacklistToUse, host, path);
|
||||
}
|
||||
}
|
||||
|
||||
SearchEventCache.cleanupEvents(true);
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue