boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for Blacklist (-Administration)

Browse Source
pull/1/head
sixcooler 12 years ago
parent f2d99053ed
commit 7d53ac86a3
9 changed files with 90 additions and 214 deletions
Show Stats Download Patch File Download Diff File

80
htroot/BlacklistCleaner_p.java
Unescape View File

@ -30,9 +30,6 @@
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
@ -52,7 +49,6 @@ import net.yacy.repository.Blacklist;
import net.yacy.repository.Blacklist.BlacklistError; import net.yacy.repository.Blacklist.BlacklistError;
import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache; import net.yacy.search.query.SearchEventCache;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
@ -68,12 +64,9 @@ public class BlacklistCleaner_p {
Blacklist.class Blacklist.class
}; };
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
// initialize the list manager
ListManager.switchboard = (Switchboard) env;
ListManager.listsPath = new File(env.getDataPath(), env.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT));
String blacklistToUse = null; String blacklistToUse = null;
prop.put(DISABLED+"checked", "1"); prop.put(DISABLED+"checked", "1");
@ -272,26 +265,13 @@ public class BlacklistCleaner_p {
* @return Length of the list of entries to be removed. * @return Length of the list of entries to be removed.
*/ */
private static int removeEntries(final String blacklistToUse, final BlacklistType[] supportedBlacklistTypes, final String[] entries) { private static int removeEntries(final String blacklistToUse, final BlacklistType[] supportedBlacklistTypes, final String[] entries) {
// load blacklist data from file
final List<String> list = FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse));
boolean listChanged = false;
// delete the old entry from file
for (final String entry : entries) { for (final String entry : entries) {
String s = entry; String s = entry;
if (list != null){ // get rid of escape characters which make it impossible to
// properly use contains()
// get rid of escape characters which make it impossible to if (s.contains("\\\\")) {
// properly use contains() s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\"));
if (s.contains("\\\\")) {
s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\"));
}
if (list.contains(s)) {
listChanged = list.remove(s);
}
} }
// remove the entry from the running blacklist engine // remove the entry from the running blacklist engine
@ -300,7 +280,7 @@ public class BlacklistCleaner_p {
final String host = (s.indexOf('/',0) == -1) ? s : s.substring(0, s.indexOf('/',0)); final String host = (s.indexOf('/',0) == -1) ? s : s.substring(0, s.indexOf('/',0));
final String path = (s.indexOf('/',0) == -1) ? ".*" : s.substring(s.indexOf('/',0) + 1); final String path = (s.indexOf('/',0) == -1) ? ".*" : s.substring(s.indexOf('/',0) + 1);
try { try {
Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path); Switchboard.urlBlacklist.remove(supportedBlacklistType, blacklistToUse, host, path);
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
ConcurrentLog.severe("BLACKLIST-CLEANER", e.getMessage() + ": " + host + "/" + path); ConcurrentLog.severe("BLACKLIST-CLEANER", e.getMessage() + ": " + host + "/" + path);
} }
@ -308,9 +288,6 @@ public class BlacklistCleaner_p {
} }
SearchEventCache.cleanupEvents(true); SearchEventCache.cleanupEvents(true);
} }
if (listChanged){
FileUtils.writeList(new File(ListManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
}
return entries.length; return entries.length;
} }
@ -328,34 +305,23 @@ public class BlacklistCleaner_p {
final String[] oldEntry, final String[] oldEntry,
final String[] newEntry) { final String[] newEntry) {
removeEntries(blacklistToUse, supportedBlacklistTypes, oldEntry); removeEntries(blacklistToUse, supportedBlacklistTypes, oldEntry);
PrintWriter pw = null; String host, path;
try { for (final String n : newEntry) {
pw = new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklistToUse), true)); final int pos = n.indexOf('/',0);
String host, path; if (pos < 0) {
for (final String n : newEntry) { host = n;
final int pos = n.indexOf('/',0); path = ".*";
if (pos < 0) { } else {
host = n; host = n.substring(0, pos);
path = ".*"; path = n.substring(pos + 1);
} else { }
host = n.substring(0, pos); for (final BlacklistType s : supportedBlacklistTypes) {
path = n.substring(pos + 1); if (ListManager.listSetContains(s + ".BlackLists",blacklistToUse)) {
} Switchboard.urlBlacklist.add(s, blacklistToUse, host, path);
pw.println(host + "/" + path); }
for (final BlacklistType s : supportedBlacklistTypes) { }
if (ListManager.listSetContains(s + ".BlackLists",blacklistToUse)) { SearchEventCache.cleanupEvents(true);
Switchboard.urlBlacklist.add( }
s,
host,
path);
}
}
SearchEventCache.cleanupEvents(true);
}
pw.close();
} catch (final IOException e) {
ConcurrentLog.severe("BLACKLIST-CLEANER", "error on writing altered entries to blacklist", e);
}
return newEntry.length; return newEntry.length;
} }
} }

6
htroot/BlacklistImpExp_p.java
Unescape View File

@ -29,7 +29,6 @@
// javac -classpath .:../classes Blacklist_p.java // javac -classpath .:../classes Blacklist_p.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.io.File;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.TreeMap; import java.util.TreeMap;
@ -41,7 +40,6 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
@ -51,10 +49,6 @@ public class BlacklistImpExp_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
// initialize the list manager
ListManager.switchboard = (Switchboard) env;
ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT));
// loading all blacklist files located in the directory // loading all blacklist files located in the directory
final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);

9
htroot/BlacklistTest_p.java
Unescape View File

@ -29,26 +29,19 @@
// javac -classpath .:../classes Blacklist_p.java // javac -classpath .:../classes Blacklist_p.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.io.File;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.ListManager;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist;
import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
public class BlacklistTest_p { public class BlacklistTest_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
// initialize the list manager
ListManager.switchboard = (Switchboard) env;
ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT));
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
prop.putHTML("blacklistEngine", Blacklist.getEngineInfo()); prop.putHTML("blacklistEngine", Blacklist.getEngineInfo());

27
htroot/Blacklist_p.java
Unescape View File

@ -45,7 +45,6 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist;
import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache; import net.yacy.search.query.SearchEventCache;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
@ -58,11 +57,7 @@ public class Blacklist_p {
private final static String BLACKLIST_SHARED = "BlackLists.Shared"; private final static String BLACKLIST_SHARED = "BlackLists.Shared";
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
// initialize the list manager
ListManager.switchboard = (Switchboard) env;
ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT));
// load all blacklist files located in the directory // load all blacklist files located in the directory
List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
@ -550,14 +545,8 @@ public class Blacklist_p {
} }
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) != blacklistToUse) { if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) {
Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path); Switchboard.urlBlacklist.remove(supportedBlacklistType, blacklistToUse, host, path);
}
else {
Blacklist bl = new Blacklist(ListManager.listsPath);
bl.loadList(supportedBlacklistType, blacklistToUse, "/");
bl.remove(host, path);
} }
} }
@ -619,14 +608,8 @@ public class Blacklist_p {
String path = newEntry.substring(pos + 1); String path = newEntry.substring(pos + 1);
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
if (Switchboard.urlBlacklist.getFileName(supportedBlacklistType) == blacklistToUse) { if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",blacklistToUse)) {
Switchboard.urlBlacklist.add(supportedBlacklistType, host, path); Switchboard.urlBlacklist.add(supportedBlacklistType, blacklistToUse, host, path);
}
else {
Blacklist bl = new Blacklist(ListManager.listsPath);
bl.loadList(supportedBlacklistType, blacklistToUse, "/");
bl.add(supportedBlacklistType, host, path);
} }
} }

115
htroot/IndexControlRWIs_p.java
Unescape View File

@ -25,10 +25,7 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
@ -367,72 +364,58 @@ public class IndexControlRWIs_p {
URIMetadataRow.rowdef.objectOrder, URIMetadataRow.rowdef.objectOrder,
urlb.size()); urlb.size());
if ( post.containsKey("blacklisturls") ) { if ( post.containsKey("blacklisturls") ) {
PrintWriter pw; final String[] supportedBlacklistTypes =
try { env.getConfig("BlackLists.types", "").split(",");
final String[] supportedBlacklistTypes = DigestURI url;
env.getConfig("BlackLists.types", "").split(","); for ( final byte[] b : urlb ) {
pw = try {
new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklist), true)); urlHashes.put(b);
DigestURI url; } catch (final SpaceExceededException e ) {
for ( final byte[] b : urlb ) { ConcurrentLog.logException(e);
try { }
urlHashes.put(b); url = segment.fulltext().getURL(b);
} catch (final SpaceExceededException e ) { segment.fulltext().remove(b);
ConcurrentLog.logException(e); if ( url != null ) {
} for ( final String supportedBlacklistType : supportedBlacklistTypes ) {
url = segment.fulltext().getURL(b); if ( ListManager.listSetContains(
segment.fulltext().remove(b); supportedBlacklistType + ".BlackLists",
if ( url != null ) { blacklist) ) {
pw.println(url.getHost() + "/" + url.getFile()); Switchboard.urlBlacklist.add(
for ( final String supportedBlacklistType : supportedBlacklistTypes ) { BlacklistType.valueOf(supportedBlacklistType),
if ( ListManager.listSetContains( blacklist,
supportedBlacklistType + ".BlackLists", url.getHost(),
blacklist) ) { url.getFile());
Switchboard.urlBlacklist.add( }
BlacklistType.valueOf(supportedBlacklistType), }
url.getHost(), SearchEventCache.cleanupEvents(true);
url.getFile()); }
} }
}
SearchEventCache.cleanupEvents(true);
}
}
pw.close();
} catch (final IOException e ) {
}
} }
if ( post.containsKey("blacklistdomains") ) { if ( post.containsKey("blacklistdomains") ) {
PrintWriter pw; DigestURI url;
try { for ( final byte[] b : urlb ) {
pw = try {
new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklist), true)); urlHashes.put(b);
DigestURI url; } catch (final SpaceExceededException e ) {
for ( final byte[] b : urlb ) { ConcurrentLog.logException(e);
try { }
urlHashes.put(b); url = segment.fulltext().getURL(b);
} catch (final SpaceExceededException e ) { segment.fulltext().remove(b);
ConcurrentLog.logException(e); if ( url != null ) {
} for ( final BlacklistType supportedBlacklistType : BlacklistType.values() ) {
url = segment.fulltext().getURL(b); if ( ListManager.listSetContains(
segment.fulltext().remove(b); supportedBlacklistType + ".BlackLists",
if ( url != null ) { blacklist) ) {
pw.println(url.getHost() + "/.*"); Switchboard.urlBlacklist.add(
for ( final BlacklistType supportedBlacklistType : BlacklistType.values() ) { supportedBlacklistType,
if ( ListManager.listSetContains( blacklist,
supportedBlacklistType + ".BlackLists", url.getHost(),
blacklist) ) { ".*");
Switchboard.urlBlacklist.add( }
supportedBlacklistType, }
url.getHost(), }
".*"); }
}
}
}
}
pw.close();
} catch (final IOException e ) {
}
} }
try { try {
segment.termIndex().remove(keyhash, urlHashes); segment.termIndex().remove(keyhash, urlHashes);

2
htroot/api/blacklists.java
Unescape View File

@ -5,7 +5,6 @@ import java.util.List;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.ListManager; import net.yacy.data.ListManager;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
@ -14,7 +13,6 @@ public class blacklists {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT));
final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath); final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath);
int blacklistCount = 0; int blacklistCount = 0;

2
htroot/api/blacklists_p.java
Unescape View File

@ -7,7 +7,6 @@ import net.yacy.data.ListManager;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist;
import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
@ -17,7 +16,6 @@ public class blacklists_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT));
final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
int blacklistCount=0; int blacklistCount=0;

18
htroot/sharedBlacklist_p.java
Unescape View File

@ -30,9 +30,7 @@
//if the shell's current path is HTROOT //if the shell's current path is HTROOT
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader; import java.io.StringReader;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashSet; import java.util.HashSet;
@ -53,7 +51,6 @@ import net.yacy.peers.Seed;
import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist;
import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache; import net.yacy.search.query.SearchEventCache;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
@ -89,10 +86,6 @@ public class sharedBlacklist_p {
if (post != null) { if (post != null) {
// initialize the list manager
ListManager.switchboard = (Switchboard) env;
ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", SwitchboardConstants.LISTS_PATH_DEFAULT));
// loading all blacklist files located in the directory // loading all blacklist files located in the directory
final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
@ -209,11 +202,7 @@ public class sharedBlacklist_p {
prop.put("page", "1"); //result page prop.put("page", "1"); //result page
prop.put("status", STATUS_ENTRIES_ADDED); //list of added Entries prop.put("status", STATUS_ENTRIES_ADDED); //list of added Entries
PrintWriter pw = null;
try { try {
// open the blacklist file
pw = new PrintWriter(new FileWriter(new File(ListManager.listsPath, selectedBlacklistName), true));
// loop through the received entry list // loop through the received entry list
final int num = post.getInt("num", 0); final int num = post.getInt("num", 0);
for(int i = 0; i < num; i++){ for(int i = 0; i < num; i++){
@ -233,13 +222,10 @@ public class sharedBlacklist_p {
newItem = newItem + "/.*"; newItem = newItem + "/.*";
} }
// append the item to the file
pw.println(newItem);
if (Switchboard.urlBlacklist != null) { if (Switchboard.urlBlacklist != null) {
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) { for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",selectedBlacklistName)) { if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists",selectedBlacklistName)) {
Switchboard.urlBlacklist.add(supportedBlacklistType,newItem.substring(0, pos), newItem.substring(pos + 1)); Switchboard.urlBlacklist.add(supportedBlacklistType,selectedBlacklistName,newItem.substring(0, pos), newItem.substring(pos + 1));
} }
} }
SearchEventCache.cleanupEvents(true); SearchEventCache.cleanupEvents(true);
@ -249,8 +235,6 @@ public class sharedBlacklist_p {
} catch (final Exception e) { } catch (final Exception e) {
prop.put("status", "1"); prop.put("status", "1");
prop.putHTML("status_error", e.getLocalizedMessage()); prop.putHTML("status_error", e.getLocalizedMessage());
} finally {
if (pw != null) try { pw.close(); } catch (final Exception e){ /* */}
} }
/* unable to use prop.putHTML() or prop.putXML() here because they /* unable to use prop.putHTML() or prop.putXML() here because they

45
source/net/yacy/repository/Blacklist.java
Unescape View File

@ -132,7 +132,7 @@ public class Blacklist {
ConcurrentLog.fine("Blacklist", "All blacklists has been shutdown."); ConcurrentLog.fine("Blacklist", "All blacklists has been shutdown.");
} }
public final void setRootPath(final File rootPath) { private final void setRootPath(final File rootPath) {
if (rootPath == null) { if (rootPath == null) {
throw new NullPointerException("The blacklist root path must not be null."); throw new NullPointerException("The blacklist root path must not be null.");
} }
@ -154,10 +154,6 @@ public class Blacklist {
return this.cachedUrlHashs.get(blacklistType); return this.cachedUrlHashs.get(blacklistType);
} }
public final String getFileName(BlacklistType type) {
return blacklistFiles.get(type);
}
public final File getRootPath() { public final File getRootPath() {
return blacklistRootPath; return blacklistRootPath;
} }
@ -173,8 +169,6 @@ public class Blacklist {
entry.clear(); entry.clear();
} }
blacklistFiles.clear(); blacklistFiles.clear();
blacklistRootPath = null;
} }
public final int size() { public final int size() {
@ -271,16 +265,7 @@ public class Blacklist {
getBlacklistMap(blacklistType, false).remove(host); getBlacklistMap(blacklistType, false).remove(host);
} }
/** public final void remove(final BlacklistType blacklistType, final String blacklistToUse, final String host, final String path) {
* Removes entry for all blacklist types.
*/
public final void remove(final String host, final String path) {
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path);
}
}
public final void remove(final BlacklistType blacklistType, final String host, final String path) {
final Map<String, Set<Pattern>> blacklistMap = getBlacklistMap(blacklistType, true); final Map<String, Set<Pattern>> blacklistMap = getBlacklistMap(blacklistType, true);
Set<Pattern> hostList = blacklistMap.get(host); Set<Pattern> hostList = blacklistMap.get(host);
@ -301,7 +286,7 @@ public class Blacklist {
} }
// load blacklist data from file // load blacklist data from file
final List<String> list = FileUtils.getListArray(new File(ListManager.listsPath, getFileName(blacklistType))); final List<String> list = FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse));
// delete the old entry from file // delete the old entry from file
if (list != null) { if (list != null) {
@ -311,20 +296,11 @@ public class Blacklist {
break; break;
} }
} }
FileUtils.writeList(new File(ListManager.listsPath, getFileName(blacklistType)), list.toArray(new String[list.size()])); FileUtils.writeList(new File(ListManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
} }
} }
/** public final void add(final BlacklistType blacklistType, final String blacklistToUse, final String host, final String path) {
* Adds a new blacklist entry for all types.
*/
public final void add(final String host, final String path) {
for (final BlacklistType supportedBlacklistType : BlacklistType.values()) {
add(supportedBlacklistType, host, path);
}
}
public final void add(final BlacklistType blacklistType, final String host, final String path) {
if (contains(blacklistType, host, path)) { if (contains(blacklistType, host, path)) {
return; return;
} }
@ -355,12 +331,13 @@ public class Blacklist {
// Append the line to the file. // Append the line to the file.
PrintWriter pw = null; PrintWriter pw = null;
try { try {
final String newEntry = h + "/" + pattern;
if (!blacklistFileContains(blacklistRootPath, if (!blacklistFileContains(blacklistRootPath,
getFileName(blacklistType), pattern.toString())) { blacklistToUse, newEntry)) {
pw = new PrintWriter(new FileWriter(new File(blacklistRootPath, pw = new PrintWriter(new FileWriter(new File(blacklistRootPath,
getFileName(blacklistType)), true)); blacklistToUse), true));
pw.println(pattern); pw.println(newEntry);
pw.close(); pw.close();
} }
} catch (final IOException e) { } catch (final IOException e) {
@ -371,7 +348,7 @@ public class Blacklist {
pw.close(); pw.close();
} catch (final Exception e) { } catch (final Exception e) {
ConcurrentLog.warn("Blacklist", "could not close stream to " + ConcurrentLog.warn("Blacklist", "could not close stream to " +
getFileName(blacklistType) + "! " + e.getMessage()); blacklistToUse + "! " + e.getMessage());
} }
} }
} }

Write Preview
Loading…
Cancel
Save