From 5399d1e2bce9018db3a439fd41dd7ea681de23a8 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 8 Nov 2009 22:58:57 +0000 Subject: [PATCH] refactoring (reason: get more abstraction to use the blacklist class; for integration in other servlets) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6471 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/BlacklistCleaner_p.java | 11 +- htroot/BlacklistImpExp_p.java | 4 +- htroot/Blacklist_p.java | 176 ++++++++------- htroot/ConfigAppearance_p.java | 5 +- htroot/ConfigLanguage_p.java | 5 +- htroot/IndexControlRWIs_p.java | 3 +- htroot/api/blacklists.java | 6 +- htroot/api/blacklists_p.java | 5 +- htroot/sharedBlacklist_p.java | 4 +- htroot/yacy/list.java | 5 +- source/de/anomic/data/listManager.java | 202 +----------------- source/de/anomic/data/translator.java | 5 +- source/net/yacy/kelondro/util/FileUtils.java | 196 +++++++++++++++++ source/net/yacy/migration.java | 3 +- source/net/yacy/repository/Blacklist.java | 28 ++- source/net/yacy/repository/BlacklistFile.java | 2 +- 16 files changed, 353 insertions(+), 307 deletions(-) diff --git a/htroot/BlacklistCleaner_p.java b/htroot/BlacklistCleaner_p.java index c8fc5b420..1af2066bd 100644 --- a/htroot/BlacklistCleaner_p.java +++ b/htroot/BlacklistCleaner_p.java @@ -52,6 +52,7 @@ import de.anomic.server.serverSwitch; import java.util.Set; import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; public class BlacklistCleaner_p { @@ -94,7 +95,7 @@ public class BlacklistCleaner_p { } } - putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse); + putBlacklists(prop, FileUtils.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse); if (blacklistToUse != null) { prop.put("results", "1"); @@ -127,7 +128,7 @@ public class BlacklistCleaner_p { } } else { prop.put("results", "0"); - putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse); + putBlacklists(prop, FileUtils.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse); } return prop; @@ -241,7 +242,7 @@ public class BlacklistCleaner_p { final Map illegalEntries = new HashMap(); final Set legalEntries = new HashSet(); - final List list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse)); + final List list = FileUtils.getListArray(new File(listManager.listsPath, blacklistToUse)); final Map properties= new HashMap(); properties.put("allowRegex", String.valueOf(allowRegex)); @@ -276,7 +277,7 @@ public class BlacklistCleaner_p { */ private static int removeEntries(final String blacklistToUse, final String[] supportedBlacklistTypes, final String[] entries) { // load blacklist data from file - final ArrayList list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse)); + final ArrayList list = FileUtils.getListArray(new File(listManager.listsPath, blacklistToUse)); boolean listChanged = false; @@ -315,7 +316,7 @@ public class BlacklistCleaner_p { SearchEventCache.cleanupEvents(true); } if (listChanged){ - listManager.writeList(new File(listManager.listsPath, blacklistToUse), list.toArray(new String[list.size()])); + FileUtils.writeList(new File(listManager.listsPath, blacklistToUse), list.toArray(new String[list.size()])); } return entries.length; } diff --git a/htroot/BlacklistImpExp_p.java b/htroot/BlacklistImpExp_p.java index f1f627e9f..1a61dae32 100644 --- a/htroot/BlacklistImpExp_p.java +++ b/htroot/BlacklistImpExp_p.java @@ -41,6 +41,8 @@ import de.anomic.server.serverSwitch; import de.anomic.yacy.yacySeed; import java.util.List; +import net.yacy.kelondro.util.FileUtils; + public class BlacklistImpExp_p { private final static String DISABLED = "disabled_"; @@ -52,7 +54,7 @@ public class BlacklistImpExp_p { listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS")); // loading all blacklist files located in the directory - final List dirlist = listManager.getDirListing(listManager.listsPath); + final List dirlist = FileUtils.getDirListing(listManager.listsPath); String blacklistToUse = null; final serverObjects prop = new serverObjects(); diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index ddaa97b9e..46e4a05b8 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -36,11 +36,11 @@ import java.io.PrintWriter; import java.net.MalformedURLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; import de.anomic.data.listManager; @@ -57,7 +57,6 @@ public class Blacklist_p { private final static String BLACKLIST_MOVE = "blackListsMove_"; private final static String BLACKLIST_SHARED = "BlackLists.Shared"; - private final static String BLACKLIST_FILENAME_FILTER = "^.*\\.black$"; public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { @@ -70,7 +69,7 @@ public class Blacklist_p { final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(","); // load all blacklist files located in the directory - List dirlist = listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER); + List dirlist = FileUtils.getDirListing(listManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); String blacklistToUse = null; final serverObjects prop = new serverObjects(); @@ -152,7 +151,7 @@ public class Blacklist_p { } // reload Blacklists - dirlist = listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER); + dirlist = FileUtils.getDirListing(listManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); } } else if (post.containsKey("deleteList")) { @@ -180,7 +179,7 @@ public class Blacklist_p { blacklistToUse = null; // reload Blacklists - dirlist = listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER); + dirlist = FileUtils.getDirListing(listManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); } else if (post.containsKey("activateList")) { @@ -235,8 +234,7 @@ public class Blacklist_p { if (selectedBlacklistEntries.length > 0) { for (int i = 0; i < selectedBlacklistEntries.length; i++) { - temp = deleteBlacklistEntry(blacklistToUse, - selectedBlacklistEntries[i], header, supportedBlacklistTypes); + temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, supportedBlacklistTypes); if (temp != null) { prop.put("LOCATION", temp); return prop; @@ -253,8 +251,7 @@ public class Blacklist_p { blacklistToUse = post.get("currentBlacklist"); - final String temp = addBlacklistEntry(post.get("currentBlacklist"), - post.get("newEntry"), header, supportedBlacklistTypes); + final String temp = addBlacklistEntry(post.get("currentBlacklist"), post.get("newEntry"), header, supportedBlacklistTypes); if (temp != null) { prop.put("LOCATION", temp); return prop; @@ -280,15 +277,13 @@ public class Blacklist_p { !targetBlacklist.equals(blacklistToUse)) { for (int i = 0; i < selectedBlacklistEntries.length; i++) { - temp = addBlacklistEntry(targetBlacklist, - selectedBlacklistEntries[i], header, supportedBlacklistTypes); + temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntries[i], header, supportedBlacklistTypes); if (temp != null) { prop.put("LOCATION", temp); return prop; } - temp = deleteBlacklistEntry(blacklistToUse, - selectedBlacklistEntries[i], header, supportedBlacklistTypes); + temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, supportedBlacklistTypes); if (temp != null) { prop.put("LOCATION", temp); return prop; @@ -366,7 +361,7 @@ public class Blacklist_p { // Read the blacklist items from file if (blacklistToUse != null) { int entryCount = 0; - final List list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse)); + final List list = FileUtils.getListArray(new File(listManager.listsPath, blacklistToUse)); // sort them final String[] sortedlist = new String[list.size()]; @@ -500,6 +495,8 @@ public class Blacklist_p { return prop; } + + /** * This method adds a new entry to the chosen blacklist. * @param blacklistToUse the name of the blacklist the entry is to be added to @@ -508,8 +505,11 @@ public class Blacklist_p { * @param supportedBlacklistTypes * @return null if no error occured, else a String to put into LOCATION */ - private static String addBlacklistEntry(final String blacklistToUse, String newEntry, - final RequestHeader header, final String[] supportedBlacklistTypes) { + private static String addBlacklistEntry( + final String blacklistToUse, + String newEntry, + final RequestHeader header, + final String[] supportedBlacklistTypes) { if (blacklistToUse == null || blacklistToUse.trim().length() == 0) { return ""; @@ -519,53 +519,12 @@ public class Blacklist_p { return header.get("PATH") + "?selectList=&selectedListName=" + blacklistToUse; } - // TODO: ignore empty entries - - if (newEntry.startsWith("http://") ){ - newEntry = newEntry.substring(7); - } else if (newEntry.startsWith("https://")) { - newEntry = newEntry.substring(8); - } - - int pos = newEntry.indexOf("/"); - if (pos < 0) { - // add default empty path pattern - pos = newEntry.length(); - newEntry = newEntry + "/.*"; - } - - if (!blacklistFileContains(blacklistToUse, newEntry)) { - // append the line to the file - PrintWriter pw = null; - try { - pw = new PrintWriter(new FileWriter(new File(listManager.listsPath, blacklistToUse), true)); - pw.println(newEntry); - pw.close(); - } catch (final IOException e) { - Log.logException(e); - } finally { - if (pw != null) { - try { - pw.close(); - } catch (final Exception e) { - Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage()); - } - - } - } - - // add to blacklist - for (int blTypes = 0; blTypes < supportedBlacklistTypes.length; blTypes++) { - if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) { - Switchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes], newEntry.substring(0, pos), newEntry.substring(pos + 1)); - } - } - SearchEventCache.cleanupEvents(true); - } - + addBlacklistEntry(listManager.listsPath, blacklistToUse, newEntry, supportedBlacklistTypes); + SearchEventCache.cleanupEvents(true); return null; } + /** * This method deletes a blacklist entry. * @param blacklistToUse the name of the blacklist the entry is to be deleted from @@ -574,8 +533,11 @@ public class Blacklist_p { * @param supportedBlacklistTypes * @return null if no error occured, else a String to put into LOCATION */ - private static String deleteBlacklistEntry(final String blacklistToUse, String oldEntry, - final RequestHeader header, final String[] supportedBlacklistTypes) { + private static String deleteBlacklistEntry( + final String blacklistToUse, + String oldEntry, + final RequestHeader header, + final String[] supportedBlacklistTypes) { if (blacklistToUse == null || blacklistToUse.trim().length() == 0) { return ""; @@ -585,8 +547,27 @@ public class Blacklist_p { return header.get("PATH") + "?selectList=&selectedListName=" + blacklistToUse; } + deleteBlacklistEntry(listManager.listsPath, blacklistToUse, oldEntry, supportedBlacklistTypes); + SearchEventCache.cleanupEvents(true); + return null; + } + + + + /** + * This method deletes a blacklist entry. + * @param blacklistToUse the name of the blacklist the entry is to be deleted from + * @param oldEntry the entry that is to be deleted + * @param supportedBlacklistTypes + */ + public static void deleteBlacklistEntry( + final File listsPath, + final String blacklistToUse, + String oldEntry, + final String[] supportedBlacklistTypes) { + // load blacklist data from file - final ArrayList list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse)); + final ArrayList list = FileUtils.getListArray(new File(listsPath, blacklistToUse)); // delete the old entry from file if (list != null) { @@ -596,7 +577,7 @@ public class Blacklist_p { break; } } - listManager.writeList(new File(listManager.listsPath, blacklistToUse), list.toArray(new String[list.size()])); + FileUtils.writeList(new File(listsPath, blacklistToUse), list.toArray(new String[list.size()])); } // remove the entry from the running blacklist engine @@ -611,24 +592,63 @@ public class Blacklist_p { Switchboard.urlBlacklist.remove(supportedBlacklistTypes[blTypes],oldEntry.substring(0, pos), oldEntry.substring(pos + 1)); } } - SearchEventCache.cleanupEvents(true); - - return null; } + + /** - * Checks if a blacklist file contains a certain entry. - * @param blacklistToUse The blacklist. - * @param newEntry The Entry. - * @return True if file contains entry, else false. + * This method adds a new entry to the chosen blacklist. + * @param blacklistToUse the name of the blacklist the entry is to be added to + * @param newEntry the entry that is to be added + * @param supportedBlacklistTypes */ - private static boolean blacklistFileContains(final String blacklistToUse, String newEntry) { - boolean ret = false; - final HashSet Blacklist = new HashSet(listManager.getListArray(new File(listManager.listsPath, blacklistToUse))); - if (Blacklist != null) { - ret = Blacklist.contains(newEntry); + public static void addBlacklistEntry( + final File listsPath, + final String blacklistToUse, + String newEntry, + final String[] supportedBlacklistTypes) { + + // TODO: ignore empty entries + + if (newEntry.startsWith("http://") ){ + newEntry = newEntry.substring(7); + } else if (newEntry.startsWith("https://")) { + newEntry = newEntry.substring(8); + } + + int pos = newEntry.indexOf("/"); + if (pos < 0) { + // add default empty path pattern + pos = newEntry.length(); + newEntry = newEntry + "/.*"; } - return ret; - } + if (!Blacklist.blacklistFileContains(listsPath, blacklistToUse, newEntry)) { + // append the line to the file + PrintWriter pw = null; + try { + pw = new PrintWriter(new FileWriter(new File(listsPath, blacklistToUse), true)); + pw.println(newEntry); + pw.close(); + } catch (final IOException e) { + Log.logException(e); + } finally { + if (pw != null) { + try { + pw.close(); + } catch (final Exception e) { + Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage()); + } + + } + } + + // add to blacklist + for (int blTypes = 0; blTypes < supportedBlacklistTypes.length; blTypes++) { + if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) { + Switchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes], newEntry.substring(0, pos), newEntry.substring(pos + 1)); + } + } + } + } } diff --git a/htroot/ConfigAppearance_p.java b/htroot/ConfigAppearance_p.java index c18032cf8..5a094271d 100644 --- a/htroot/ConfigAppearance_p.java +++ b/htroot/ConfigAppearance_p.java @@ -41,7 +41,6 @@ import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import de.anomic.crawler.retrieval.HTTPLoader; -import de.anomic.data.listManager; import de.anomic.http.client.Client; import de.anomic.http.server.HeaderFramework; import de.anomic.http.server.RequestHeader; @@ -63,7 +62,7 @@ public class ConfigAppearance_p { prop.put("currentskin", ""); prop.put("status", "0"); // nothing - List skinFiles = listManager.getDirListing(skinPath, SKIN_FILENAME_FILTER); + List skinFiles = FileUtils.getDirListing(skinPath, SKIN_FILENAME_FILTER); if (skinFiles == null) { return prop; } @@ -129,7 +128,7 @@ public class ConfigAppearance_p { } // reread skins - skinFiles = listManager.getDirListing(skinPath, SKIN_FILENAME_FILTER); + skinFiles = FileUtils.getDirListing(skinPath, SKIN_FILENAME_FILTER); Collections.sort(skinFiles); int count = 0; for (String skinFile : skinFiles) { diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index 3a1d4c25b..cc2c25d38 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -42,7 +42,6 @@ import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import de.anomic.crawler.retrieval.HTTPLoader; -import de.anomic.data.listManager; import de.anomic.data.translator; import de.anomic.http.client.Client; import de.anomic.http.server.HeaderFramework; @@ -65,7 +64,7 @@ public class ConfigLanguage_p { //prop.put("currentlang", ""); //is done by Translationtemplate prop.put("status", "0");//nothing - List langFiles = listManager.getDirListing(langPath, LANG_FILENAME_FILTER); + List langFiles = FileUtils.getDirListing(langPath, LANG_FILENAME_FILTER); if(langFiles == null){ return prop; } @@ -128,7 +127,7 @@ public class ConfigLanguage_p { } //reread language files - langFiles = listManager.getDirListing(langPath, LANG_FILENAME_FILTER); + langFiles = FileUtils.getDirListing(langPath, LANG_FILENAME_FILTER); Collections.sort(langFiles); final HashMap langNames = translator.langMap(env); String langKey, langName; diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 7b55f2b07..80db10ea7 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -48,6 +48,7 @@ import net.yacy.kelondro.rwi.Reference; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainerCache; import net.yacy.kelondro.util.DateFormatter; +import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; import de.anomic.data.listManager; @@ -451,7 +452,7 @@ public class IndexControlRWIs_p { prop.put("genUrlList_urlList", i); prop.putHTML("genUrlList_keyString", keystring); prop.put("genUrlList_count", i); - putBlacklists(prop, listManager.getDirListing(listManager.listsPath)); + putBlacklists(prop, FileUtils.getDirListing(listManager.listsPath)); } } diff --git a/htroot/api/blacklists.java b/htroot/api/blacklists.java index f5cb39ef7..8ffd8ffcc 100644 --- a/htroot/api/blacklists.java +++ b/htroot/api/blacklists.java @@ -2,6 +2,8 @@ import java.io.File; import java.util.List; +import net.yacy.kelondro.util.FileUtils; + import de.anomic.data.listManager; import de.anomic.http.server.RequestHeader; import de.anomic.server.serverObjects; @@ -13,7 +15,7 @@ public class blacklists { final serverObjects prop = new serverObjects(); listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS")); - final List dirlist = listManager.getDirListing(listManager.listsPath); + final List dirlist = FileUtils.getDirListing(listManager.listsPath); int blacklistCount=0; final String blackListName = (post == null) ? "" : post.get("listname", ""); @@ -27,7 +29,7 @@ public class blacklists { if (listManager.listSetContains("BlackLists.Shared", element)) { - list = listManager.getListArray(new File(listManager.listsPath, element)); + list = FileUtils.getListArray(new File(listManager.listsPath, element)); count=0; for (int j=0;j dirlist = listManager.getDirListing(listManager.listsPath); + final List dirlist = FileUtils.getDirListing(listManager.listsPath); int blacklistCount=0; final String blackListName = (post == null) ? "" : post.get("listname", ""); @@ -44,7 +45,7 @@ public class blacklists_p { prop.put("lists_" + blacklistCount + "_types", types.length); if ( ! (attrOnly.equals("1") || attrOnly.equals("true"))) { - list = listManager.getListArray(new File(listManager.listsPath, element)); + list = FileUtils.getListArray(new File(listManager.listsPath, element)); count=0; for (int j=0;j dirlist = listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER); + final List dirlist = FileUtils.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER); // List BlackLists int blacklistCount = 0; @@ -280,7 +280,7 @@ public class sharedBlacklist_p { // generate the html list if (otherBlacklist != null) { // loading the current blacklist content - final HashSet Blacklist = new HashSet(listManager.getListArray(new File(listManager.listsPath, selectedBlacklistName))); + final HashSet Blacklist = new HashSet(FileUtils.getListArray(new File(listManager.listsPath, selectedBlacklistName))); int count = 0; while (otherBlacklist.hasNext()) { diff --git a/htroot/yacy/list.java b/htroot/yacy/list.java index e9b2d2ecb..0074f84af 100644 --- a/htroot/yacy/list.java +++ b/htroot/yacy/list.java @@ -28,7 +28,8 @@ import java.io.File; -import de.anomic.data.listManager; +import net.yacy.kelondro.util.FileUtils; + import de.anomic.http.server.HeaderFramework; import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; @@ -79,7 +80,7 @@ public final class list { if (blackListName.equals("") || filenamesarray[i].equals(blackListName)) { final String filename = filenamesarray[i]; final File fileObj = new File(listsPath,filename); - out.append(listManager.getListString(fileObj, false)).append(serverCore.CRLF_STRING); + out.append(FileUtils.getListString(fileObj, false)).append(serverCore.CRLF_STRING); } } } diff --git a/source/de/anomic/data/listManager.java b/source/de/anomic/data/listManager.java index 2c2ff5947..e97cdf297 100644 --- a/source/de/anomic/data/listManager.java +++ b/source/de/anomic/data/listManager.java @@ -24,30 +24,21 @@ package de.anomic.data; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileInputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; import java.util.Set; import java.util.Vector; +import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; import net.yacy.repository.BlacklistFile; import de.anomic.search.SearchEventCache; import de.anomic.search.Switchboard; -import de.anomic.server.serverCore; // The Naming of the functions is a bit strange... @@ -118,199 +109,10 @@ public class listManager { //================general Lists================== - /** - * Read lines of a file into an ArrayList. - * - * @param listFile the file - * @return the resulting array as an ArrayList - */ - public static ArrayList getListArray(final File listFile){ - String line; - final ArrayList list = new ArrayList(); - int count = 0; - BufferedReader br = null; - try { - br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile),"UTF-8")); - - while((line = br.readLine()) != null){ - list.add(line); - count++; - } - br.close(); - } catch(final IOException e) { - // list is empty - } finally { - if (br!=null) try { br.close(); } catch (final Exception e) {} - } - return list; - } - - /** - * Write a String to a file (used for string representation of lists). - * - * @param listFile the file to write to - * @param out the String to write - * @return returns true if successful, false otherwise - */ - public static boolean writeList(final File listFile, final String out) { - BufferedWriter bw = null; - try { - bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile))); - bw.write(out); - bw.close(); - return true; - } catch(final IOException e) { - return false; - } finally { - if (bw!=null) try { bw.close(); } catch (final Exception e) {} - } - } - - /** - * Write elements of an Array of Strings to a file (one element per line). - * - * @param listFile the file to write to - * @param list the Array to write - * @return returns true if successful, false otherwise - */ - public static boolean writeList(final File listFile, final String[] list){ - final StringBuilder out = new StringBuilder(); - for(int i=0;i < list.length; i++){ - out - .append(list[i]) - .append(serverCore.CRLF_STRING); - } - return writeList(listFile, new String(out)); //(File, String) - } - - // same as below public static String getListString(final String filename, final boolean withcomments) { final File listFile = new File(listsPath ,filename); - return getListString(listFile, withcomments); - } - - /** - * Read lines of a text file into a String, optionally ignoring comments. - * - * @param listFile the File to read from. - * @param withcomments If false ignore lines starting with '#'. - * @return String representation of the file content. - */ - public static String getListString(final File listFile, final boolean withcomments){ - final StringBuilder temp = new StringBuilder(); - - BufferedReader br = null; - try{ - br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile))); - temp.ensureCapacity((int) listFile.length()); - - // Read the List - String line = ""; - while ((line = br.readLine()) != null) { - if ((!line.startsWith("#") || withcomments) || !line.equals("")) { - //temp += line + serverCore.CRLF_STRING; - temp.append(line) - .append(serverCore.CRLF_STRING); - } - } - br.close(); - } catch (final IOException e) { - } finally { - if (br!=null) try { br.close(); } catch (final Exception e) {} - } - - return new String(temp); + return FileUtils.getListString(listFile, withcomments); } - - /** - * Read content of a directory into a String array of file names. - * @param dirname The directory to get the file listing from. If it doesn't exist yet, - * it will be created. - * @return array of file names - */ - public static List getDirListing(final String dirname){ - return getDirListing(dirname, null); - } - - /** - * Read content of a directory into a String array of file names. - * @param dirname The directory to get the file listing from. If it doesn't exist yet, - * it will be created. - * @param filter String which contains a regular expression which has to be matched by - * file names in order to appear in returned array. All file names will be returned if - * filter is null. - * @return array of file names - */ - public static List getDirListing(final String dirname, final String filter) { - return getDirListing(new File(dirname), filter); - } - - /** - * Read content of a directory into a String array of file names. - * - * @param dir The directory to get the file listing from. If it doesn't exist yet, - * it will be created. - * @return array of file names - */ - public static List getDirListing(final File dir){ - return getDirListing(dir, null); - } - - /** - * Read content of a directory into a String array of file names. - * @param dir The directory to get the file listing from. If it doesn't exist yet, - * it will be created. - * @param filter String which contains a regular expression which has to be matched by - * file names in order to appear in returned array. All file names will be returned if - * filter is null. - * @return array of file names - */ - public static List getDirListing(final File dir, final String filter){ - List ret = new LinkedList(); - File[] fileList; - if (dir != null ) { - if (!dir.exists()) { - dir.mkdir(); - } - fileList = dir.listFiles(); - for (int i=0; i<= fileList.length-1; i++) { - if (filter == null || fileList[i].getName().matches(filter)) { - ret.add(fileList[i].getName()); - } - } - return ret; - } - return null; - } - - // same as below - public static ArrayList getDirsRecursive(final File dir, final String notdir){ - return getDirsRecursive(dir, notdir, true); - } - - /** - * Returns a List of all dirs and subdirs as File Objects - * - * Warning: untested - */ - public static ArrayList getDirsRecursive(final File dir, final String notdir, final boolean excludeDotfiles){ - final File[] dirList = dir.listFiles(); - final ArrayList resultList = new ArrayList(); - ArrayList recursive; - Iterator iter; - for (int i=0;i> lists = new Hashtable>(); //list of translationLists for different files. Hashtable translationList = new Hashtable(); //current Translation Table - final ArrayList list = listManager.getListArray(translationFile); + final ArrayList list = FileUtils.getListArray(translationFile); final Iterator it = list.iterator(); String line = ""; String[] splitted; @@ -204,7 +205,7 @@ public class translator { } public static boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){ - final ArrayList dirList=listManager.getDirsRecursive(sourceDir, notdir); + final ArrayList dirList=FileUtils.getDirsRecursive(sourceDir, notdir); dirList.add(sourceDir); final Iterator it=dirList.iterator(); File file=null; diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index 114a7d094..28a4b3591 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -30,6 +30,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -45,6 +46,8 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.StringTokenizer; @@ -542,6 +545,199 @@ public final class FileUtils { } } + + /** + * Read lines of a file into an ArrayList. + * + * @param listFile the file + * @return the resulting array as an ArrayList + */ + public static ArrayList getListArray(final File listFile){ + String line; + final ArrayList list = new ArrayList(); + int count = 0; + BufferedReader br = null; + try { + br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile),"UTF-8")); + + while((line = br.readLine()) != null){ + list.add(line); + count++; + } + br.close(); + } catch(final IOException e) { + // list is empty + } finally { + if (br!=null) try { br.close(); } catch (final Exception e) {} + } + return list; + } + + + + /** + * Write a String to a file (used for string representation of lists). + * + * @param listFile the file to write to + * @param out the String to write + * @return returns true if successful, false otherwise + */ + public static boolean writeList(final File listFile, final String out) { + BufferedWriter bw = null; + try { + bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile))); + bw.write(out); + bw.close(); + return true; + } catch(final IOException e) { + return false; + } finally { + if (bw!=null) try { bw.close(); } catch (final Exception e) {} + } + } + + public static final byte LF = 10; + public static final byte CR = 13; + + /** + * Read lines of a text file into a String, optionally ignoring comments. + * + * @param listFile the File to read from. + * @param withcomments If false ignore lines starting with '#'. + * @return String representation of the file content. + */ + public static String getListString(final File listFile, final boolean withcomments){ + final StringBuilder temp = new StringBuilder(); + + BufferedReader br = null; + try{ + br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile))); + temp.ensureCapacity((int) listFile.length()); + + // Read the List + String line = ""; + while ((line = br.readLine()) != null) { + if ((!line.startsWith("#") || withcomments) || !line.equals("")) { + //temp += line + serverCore.CRLF_STRING; + temp.append(line).append(CR).append(LF); + } + } + br.close(); + } catch (final IOException e) { + } finally { + if (br!=null) try { br.close(); } catch (final Exception e) {} + } + + return new String(temp); + } + + /** + * Read content of a directory into a String array of file names. + * @param dirname The directory to get the file listing from. If it doesn't exist yet, + * it will be created. + * @return array of file names + */ + public static List getDirListing(final String dirname){ + return getDirListing(dirname, null); + } + + /** + * Read content of a directory into a String array of file names. + * @param dirname The directory to get the file listing from. If it doesn't exist yet, + * it will be created. + * @param filter String which contains a regular expression which has to be matched by + * file names in order to appear in returned array. All file names will be returned if + * filter is null. + * @return array of file names + */ + public static List getDirListing(final String dirname, final String filter) { + return getDirListing(new File(dirname), filter); + } + + /** + * Read content of a directory into a String array of file names. + * + * @param dir The directory to get the file listing from. If it doesn't exist yet, + * it will be created. + * @return array of file names + */ + public static List getDirListing(final File dir){ + return getDirListing(dir, null); + } + + /** + * Read content of a directory into a String array of file names. + * @param dir The directory to get the file listing from. If it doesn't exist yet, + * it will be created. + * @param filter String which contains a regular expression which has to be matched by + * file names in order to appear in returned array. All file names will be returned if + * filter is null. + * @return array of file names + */ + public static List getDirListing(final File dir, final String filter){ + List ret = new LinkedList(); + File[] fileList; + if (dir != null ) { + if (!dir.exists()) { + dir.mkdir(); + } + fileList = dir.listFiles(); + for (int i=0; i<= fileList.length-1; i++) { + if (filter == null || fileList[i].getName().matches(filter)) { + ret.add(fileList[i].getName()); + } + } + return ret; + } + return null; + } + + // same as below + public static ArrayList getDirsRecursive(final File dir, final String notdir){ + return getDirsRecursive(dir, notdir, true); + } + + /** + * Returns a List of all dirs and subdirs as File Objects + * + * Warning: untested + */ + public static ArrayList getDirsRecursive(final File dir, final String notdir, final boolean excludeDotfiles){ + final File[] dirList = dir.listFiles(); + final ArrayList resultList = new ArrayList(); + ArrayList recursive; + Iterator iter; + for (int i=0;itrue if successful, false otherwise + */ + public static boolean writeList(final File listFile, final String[] list){ + final StringBuilder out = new StringBuilder(); + for(int i=0;i < list.length; i++){ + out.append(list[i]).append(CR).append(LF); + } + return FileUtils.writeList(listFile, new String(out)); //(File, String) + } + + /* public static ArrayList strings(byte[] a) { final ArrayList list = new ArrayList(); diff --git a/source/net/yacy/migration.java b/source/net/yacy/migration.java index 19c57b6a2..03d6632ef 100644 --- a/source/net/yacy/migration.java +++ b/source/net/yacy/migration.java @@ -28,7 +28,6 @@ import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.util.FileUtils; -import de.anomic.data.listManager; import de.anomic.http.server.HTTPDemon; import de.anomic.search.Switchboard; import de.anomic.search.SwitchboardConstants; @@ -88,7 +87,7 @@ public class migration { final File skinsPath = sb.getConfigPath("skinPath", "DATA/SKINS"); final File defaultSkinsPath = new File(sb.getRootPath(), "skins"); if (defaultSkinsPath.exists()) { - final List skinFiles = listManager.getDirListing(defaultSkinsPath.getAbsolutePath()); + final List skinFiles = FileUtils.getDirListing(defaultSkinsPath.getAbsolutePath()); mkdirs(skinsPath); for (String skinFile : skinFiles){ if (skinFile.endsWith(".css")){ diff --git a/source/net/yacy/repository/Blacklist.java b/source/net/yacy/repository/Blacklist.java index 105e7b998..28a7f65cd 100644 --- a/source/net/yacy/repository/Blacklist.java +++ b/source/net/yacy/repository/Blacklist.java @@ -34,14 +34,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.Map.Entry; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; - import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.SetTools; public class Blacklist { @@ -53,6 +54,8 @@ public class Blacklist { public static final String BLACKLIST_SEARCH = "search"; public static final String BLACKLIST_SURFTIPS = "surftips"; public static final String BLACKLIST_NEWS = "news"; + + public final static String BLACKLIST_FILENAME_FILTER = "^.*\\.black$"; public static final int ERR_TWO_WILDCARDS_IN_HOST = 1; public static final int ERR_SUBDOMAIN_XOR_WILDCARD = 2; @@ -98,9 +101,7 @@ public class Blacklist { this.cachedUrlHashs.put(blacklistType, Collections.synchronizedSet(new HashSet())); } } - - public void setRootPath(final File rootPath) { if (rootPath == null) throw new NullPointerException("The blacklist root path must not be null."); @@ -458,5 +459,26 @@ public class Blacklist { } return ret; } + + public static final String defaultBlacklist(final File listsPath) { + List dirlist = FileUtils.getDirListing(listsPath, Blacklist.BLACKLIST_FILENAME_FILTER); + if (dirlist.size() == 0) return null; + return dirlist.get(0); + } + /** + * Checks if a blacklist file contains a certain entry. + * @param blacklistToUse The blacklist. + * @param newEntry The Entry. + * @return True if file contains entry, else false. + */ + public static boolean blacklistFileContains(final File listsPath, final String blacklistToUse, String newEntry) { + boolean ret = false; + final HashSet Blacklist = new HashSet(FileUtils.getListArray(new File(listsPath, blacklistToUse))); + if (Blacklist != null) { + ret = Blacklist.contains(newEntry); + } + return ret; + } + } diff --git a/source/net/yacy/repository/BlacklistFile.java b/source/net/yacy/repository/BlacklistFile.java index 9b2ff074e..147de41ed 100644 --- a/source/net/yacy/repository/BlacklistFile.java +++ b/source/net/yacy/repository/BlacklistFile.java @@ -43,7 +43,7 @@ public class BlacklistFile { /** - * Construct a unified array of file names from comma seperated file name + * Construct a unified array of file names from comma separated file name * list. * * @return unified String array of file names