boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring (reason: get more abstraction to use the blacklist class; for integration in other servlets)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6471 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent a97fdb4566
commit 5399d1e2bc
16 changed files with 353 additions and 307 deletions
Show Stats Download Patch File Download Diff File

11
htroot/BlacklistCleaner_p.java
Unescape View File

@ -52,6 +52,7 @@ import de.anomic.server.serverSwitch;
import java.util.Set;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;
public class BlacklistCleaner_p {
@ -94,7 +95,7 @@ public class BlacklistCleaner_p {
}
}
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
putBlacklists(prop, FileUtils.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
if (blacklistToUse != null) {
prop.put("results", "1");
@ -127,7 +128,7 @@ public class BlacklistCleaner_p {
}
} else {
prop.put("results", "0");
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
putBlacklists(prop, FileUtils.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
}
return prop;
@ -241,7 +242,7 @@ public class BlacklistCleaner_p {
final Map<String, Integer> illegalEntries = new HashMap<String, Integer>();
final Set<String> legalEntries = new HashSet<String>();
final List<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
final List<String> list = FileUtils.getListArray(new File(listManager.listsPath, blacklistToUse));
final Map<String, String> properties= new HashMap<String, String>();
properties.put("allowRegex", String.valueOf(allowRegex));
@ -276,7 +277,7 @@ public class BlacklistCleaner_p {
*/
private static int removeEntries(final String blacklistToUse, final String[] supportedBlacklistTypes, final String[] entries) {
// load blacklist data from file
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
final ArrayList<String> list = FileUtils.getListArray(new File(listManager.listsPath, blacklistToUse));
boolean listChanged = false;
@ -315,7 +316,7 @@ public class BlacklistCleaner_p {
SearchEventCache.cleanupEvents(true);
}
if (listChanged){
listManager.writeList(new File(listManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
FileUtils.writeList(new File(listManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
}
return entries.length;
}

4
htroot/BlacklistImpExp_p.java
Unescape View File

@ -41,6 +41,8 @@ import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeed;
import java.util.List;
import net.yacy.kelondro.util.FileUtils;
public class BlacklistImpExp_p {
private final static String DISABLED = "disabled_";
@ -52,7 +54,7 @@ public class BlacklistImpExp_p {
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
// loading all blacklist files located in the directory
final List<String> dirlist = listManager.getDirListing(listManager.listsPath);
final List<String> dirlist = FileUtils.getDirListing(listManager.listsPath);
String blacklistToUse = null;
final serverObjects prop = new serverObjects();

176
htroot/Blacklist_p.java
Unescape View File

@ -36,11 +36,11 @@ import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;
import de.anomic.data.listManager;
@ -57,7 +57,6 @@ public class Blacklist_p {
private final static String BLACKLIST_MOVE = "blackListsMove_";
private final static String BLACKLIST_SHARED = "BlackLists.Shared";
private final static String BLACKLIST_FILENAME_FILTER = "^.*\\.black$";
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
@ -70,7 +69,7 @@ public class Blacklist_p {
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
// load all blacklist files located in the directory
List<String> dirlist = listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER);
List<String> dirlist = FileUtils.getDirListing(listManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
String blacklistToUse = null;
final serverObjects prop = new serverObjects();
@ -152,7 +151,7 @@ public class Blacklist_p {
}
// reload Blacklists
dirlist = listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER);
dirlist = FileUtils.getDirListing(listManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
}
} else if (post.containsKey("deleteList")) {
@ -180,7 +179,7 @@ public class Blacklist_p {
blacklistToUse = null;
// reload Blacklists
dirlist = listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER);
dirlist = FileUtils.getDirListing(listManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
} else if (post.containsKey("activateList")) {
@ -235,8 +234,7 @@ public class Blacklist_p {
if (selectedBlacklistEntries.length > 0) {
for (int i = 0; i < selectedBlacklistEntries.length; i++) {
temp = deleteBlacklistEntry(blacklistToUse,
selectedBlacklistEntries[i], header, supportedBlacklistTypes);
temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, supportedBlacklistTypes);
if (temp != null) {
prop.put("LOCATION", temp);
return prop;
@ -253,8 +251,7 @@ public class Blacklist_p {
blacklistToUse = post.get("currentBlacklist");
final String temp = addBlacklistEntry(post.get("currentBlacklist"),
post.get("newEntry"), header, supportedBlacklistTypes);
final String temp = addBlacklistEntry(post.get("currentBlacklist"), post.get("newEntry"), header, supportedBlacklistTypes);
if (temp != null) {
prop.put("LOCATION", temp);
return prop;
@ -280,15 +277,13 @@ public class Blacklist_p {
!targetBlacklist.equals(blacklistToUse)) {
for (int i = 0; i < selectedBlacklistEntries.length; i++) {
temp = addBlacklistEntry(targetBlacklist,
selectedBlacklistEntries[i], header, supportedBlacklistTypes);
temp = addBlacklistEntry(targetBlacklist, selectedBlacklistEntries[i], header, supportedBlacklistTypes);
if (temp != null) {
prop.put("LOCATION", temp);
return prop;
}
temp = deleteBlacklistEntry(blacklistToUse,
selectedBlacklistEntries[i], header, supportedBlacklistTypes);
temp = deleteBlacklistEntry(blacklistToUse, selectedBlacklistEntries[i], header, supportedBlacklistTypes);
if (temp != null) {
prop.put("LOCATION", temp);
return prop;
@ -366,7 +361,7 @@ public class Blacklist_p {
// Read the blacklist items from file
if (blacklistToUse != null) {
int entryCount = 0;
final List<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
final List<String> list = FileUtils.getListArray(new File(listManager.listsPath, blacklistToUse));
// sort them
final String[] sortedlist = new String[list.size()];
@ -500,6 +495,8 @@ public class Blacklist_p {
return prop;
}
/**
* This method adds a new entry to the chosen blacklist.
* @param blacklistToUse the name of the blacklist the entry is to be added to
@ -508,8 +505,11 @@ public class Blacklist_p {
* @param supportedBlacklistTypes
* @return null if no error occured, else a String to put into LOCATION
*/
private static String addBlacklistEntry(final String blacklistToUse, String newEntry,
final RequestHeader header, final String[] supportedBlacklistTypes) {
private static String addBlacklistEntry(
final String blacklistToUse,
String newEntry,
final RequestHeader header,
final String[] supportedBlacklistTypes) {
if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
return "";
@ -519,53 +519,12 @@ public class Blacklist_p {
return header.get("PATH") + "?selectList=&selectedListName=" + blacklistToUse;
}
// TODO: ignore empty entries
if (newEntry.startsWith("http://") ){
newEntry = newEntry.substring(7);
} else if (newEntry.startsWith("https://")) {
newEntry = newEntry.substring(8);
}
int pos = newEntry.indexOf("/");
if (pos < 0) {
// add default empty path pattern
pos = newEntry.length();
newEntry = newEntry + "/.*";
}
if (!blacklistFileContains(blacklistToUse, newEntry)) {
// append the line to the file
PrintWriter pw = null;
try {
pw = new PrintWriter(new FileWriter(new File(listManager.listsPath, blacklistToUse), true));
pw.println(newEntry);
pw.close();
} catch (final IOException e) {
Log.logException(e);
} finally {
if (pw != null) {
try {
pw.close();
} catch (final Exception e) {
Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage());
}
}
}
// add to blacklist
for (int blTypes = 0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) {
Switchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes], newEntry.substring(0, pos), newEntry.substring(pos + 1));
}
}
SearchEventCache.cleanupEvents(true);
}
addBlacklistEntry(listManager.listsPath, blacklistToUse, newEntry, supportedBlacklistTypes);
SearchEventCache.cleanupEvents(true);
return null;
}
/**
* This method deletes a blacklist entry.
* @param blacklistToUse the name of the blacklist the entry is to be deleted from
@ -574,8 +533,11 @@ public class Blacklist_p {
* @param supportedBlacklistTypes
* @return null if no error occured, else a String to put into LOCATION
*/
private static String deleteBlacklistEntry(final String blacklistToUse, String oldEntry,
final RequestHeader header, final String[] supportedBlacklistTypes) {
private static String deleteBlacklistEntry(
final String blacklistToUse,
String oldEntry,
final RequestHeader header,
final String[] supportedBlacklistTypes) {
if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
return "";
@ -585,8 +547,27 @@ public class Blacklist_p {
return header.get("PATH") + "?selectList=&selectedListName=" + blacklistToUse;
}
deleteBlacklistEntry(listManager.listsPath, blacklistToUse, oldEntry, supportedBlacklistTypes);
SearchEventCache.cleanupEvents(true);
return null;
}
/**
* This method deletes a blacklist entry.
* @param blacklistToUse the name of the blacklist the entry is to be deleted from
* @param oldEntry the entry that is to be deleted
* @param supportedBlacklistTypes
*/
public static void deleteBlacklistEntry(
final File listsPath,
final String blacklistToUse,
String oldEntry,
final String[] supportedBlacklistTypes) {
// load blacklist data from file
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
final ArrayList<String> list = FileUtils.getListArray(new File(listsPath, blacklistToUse));
// delete the old entry from file
if (list != null) {
@ -596,7 +577,7 @@ public class Blacklist_p {
break;
}
}
listManager.writeList(new File(listManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
FileUtils.writeList(new File(listsPath, blacklistToUse), list.toArray(new String[list.size()]));
}
// remove the entry from the running blacklist engine
@ -611,24 +592,63 @@ public class Blacklist_p {
Switchboard.urlBlacklist.remove(supportedBlacklistTypes[blTypes],oldEntry.substring(0, pos), oldEntry.substring(pos + 1));
}
}
SearchEventCache.cleanupEvents(true);
return null;
}
/**
* Checks if a blacklist file contains a certain entry.
* @param blacklistToUse The blacklist.
* @param newEntry The Entry.
* @return True if file contains entry, else false.
* This method adds a new entry to the chosen blacklist.
* @param blacklistToUse the name of the blacklist the entry is to be added to
* @param newEntry the entry that is to be added
* @param supportedBlacklistTypes
*/
private static boolean blacklistFileContains(final String blacklistToUse, String newEntry) {
boolean ret = false;
final HashSet<String> Blacklist = new HashSet<String>(listManager.getListArray(new File(listManager.listsPath, blacklistToUse)));
if (Blacklist != null) {
ret = Blacklist.contains(newEntry);
public static void addBlacklistEntry(
final File listsPath,
final String blacklistToUse,
String newEntry,
final String[] supportedBlacklistTypes) {
// TODO: ignore empty entries
if (newEntry.startsWith("http://") ){
newEntry = newEntry.substring(7);
} else if (newEntry.startsWith("https://")) {
newEntry = newEntry.substring(8);
}
int pos = newEntry.indexOf("/");
if (pos < 0) {
// add default empty path pattern
pos = newEntry.length();
newEntry = newEntry + "/.*";
}
return ret;
}
if (!Blacklist.blacklistFileContains(listsPath, blacklistToUse, newEntry)) {
// append the line to the file
PrintWriter pw = null;
try {
pw = new PrintWriter(new FileWriter(new File(listsPath, blacklistToUse), true));
pw.println(newEntry);
pw.close();
} catch (final IOException e) {
Log.logException(e);
} finally {
if (pw != null) {
try {
pw.close();
} catch (final Exception e) {
Log.logWarning("Blacklist", "could not close stream to " + blacklistToUse + "! " + e.getMessage());
}
}
}
// add to blacklist
for (int blTypes = 0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) {
Switchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes], newEntry.substring(0, pos), newEntry.substring(pos + 1));
}
}
}
}
}

5
htroot/ConfigAppearance_p.java
Unescape View File

@ -41,7 +41,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.data.listManager;
import de.anomic.http.client.Client;
import de.anomic.http.server.HeaderFramework;
import de.anomic.http.server.RequestHeader;
@ -63,7 +62,7 @@ public class ConfigAppearance_p {
prop.put("currentskin", "");
prop.put("status", "0"); // nothing
List<String> skinFiles = listManager.getDirListing(skinPath, SKIN_FILENAME_FILTER);
List<String> skinFiles = FileUtils.getDirListing(skinPath, SKIN_FILENAME_FILTER);
if (skinFiles == null) {
return prop;
}
@ -129,7 +128,7 @@ public class ConfigAppearance_p {
}
// reread skins
skinFiles = listManager.getDirListing(skinPath, SKIN_FILENAME_FILTER);
skinFiles = FileUtils.getDirListing(skinPath, SKIN_FILENAME_FILTER);
Collections.sort(skinFiles);
int count = 0;
for (String skinFile : skinFiles) {

5
htroot/ConfigLanguage_p.java
Unescape View File

@ -42,7 +42,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.data.listManager;
import de.anomic.data.translator;
import de.anomic.http.client.Client;
import de.anomic.http.server.HeaderFramework;
@ -65,7 +64,7 @@ public class ConfigLanguage_p {
//prop.put("currentlang", ""); //is done by Translationtemplate
prop.put("status", "0");//nothing
List<String> langFiles = listManager.getDirListing(langPath, LANG_FILENAME_FILTER);
List<String> langFiles = FileUtils.getDirListing(langPath, LANG_FILENAME_FILTER);
if(langFiles == null){
return prop;
}
@ -128,7 +127,7 @@ public class ConfigLanguage_p {
}
//reread language files
langFiles = listManager.getDirListing(langPath, LANG_FILENAME_FILTER);
langFiles = FileUtils.getDirListing(langPath, LANG_FILENAME_FILTER);
Collections.sort(langFiles);
final HashMap<String, String> langNames = translator.langMap(env);
String langKey, langName;

3
htroot/IndexControlRWIs_p.java
Unescape View File

@ -48,6 +48,7 @@ import net.yacy.kelondro.rwi.Reference;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.rwi.ReferenceContainerCache;
import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;
import de.anomic.data.listManager;
@ -451,7 +452,7 @@ public class IndexControlRWIs_p {
prop.put("genUrlList_urlList", i);
prop.putHTML("genUrlList_keyString", keystring);
prop.put("genUrlList_count", i);
putBlacklists(prop, listManager.getDirListing(listManager.listsPath));
putBlacklists(prop, FileUtils.getDirListing(listManager.listsPath));
}
}

6
htroot/api/blacklists.java
Unescape View File

@ -2,6 +2,8 @@
import java.io.File;
import java.util.List;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.data.listManager;
import de.anomic.http.server.RequestHeader;
import de.anomic.server.serverObjects;
@ -13,7 +15,7 @@ public class blacklists {
final serverObjects prop = new serverObjects();
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
final List<String> dirlist = listManager.getDirListing(listManager.listsPath);
final List<String> dirlist = FileUtils.getDirListing(listManager.listsPath);
int blacklistCount=0;
final String blackListName = (post == null) ? "" : post.get("listname", "");
@ -27,7 +29,7 @@ public class blacklists {
if (listManager.listSetContains("BlackLists.Shared", element)) {
list = listManager.getListArray(new File(listManager.listsPath, element));
list = FileUtils.getListArray(new File(listManager.listsPath, element));
count=0;
for (int j=0;j<list.size();++j){

5
htroot/api/blacklists_p.java
Unescape View File

@ -2,6 +2,7 @@
import java.io.File;
import java.util.List;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;
import de.anomic.data.listManager;
@ -16,7 +17,7 @@ public class blacklists_p {
final serverObjects prop = new serverObjects();
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
final List<String> dirlist = listManager.getDirListing(listManager.listsPath);
final List<String> dirlist = FileUtils.getDirListing(listManager.listsPath);
int blacklistCount=0;
final String blackListName = (post == null) ? "" : post.get("listname", "");
@ -44,7 +45,7 @@ public class blacklists_p {
prop.put("lists_" + blacklistCount + "_types", types.length);
if ( ! (attrOnly.equals("1") || attrOnly.equals("true"))) {
list = listManager.getListArray(new File(listManager.listsPath, element));
list = FileUtils.getListArray(new File(listManager.listsPath, element));
count=0;
for (int j=0;j<list.size();++j){

4
htroot/sharedBlacklist_p.java
Unescape View File

@ -94,7 +94,7 @@ public class sharedBlacklist_p {
// loading all blacklist files located in the directory
final List<String> dirlist = listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER);
final List<String> dirlist = FileUtils.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER);
// List BlackLists
int blacklistCount = 0;
@ -280,7 +280,7 @@ public class sharedBlacklist_p {
// generate the html list
if (otherBlacklist != null) {
// loading the current blacklist content
final HashSet<String> Blacklist = new HashSet<String>(listManager.getListArray(new File(listManager.listsPath, selectedBlacklistName)));
final HashSet<String> Blacklist = new HashSet<String>(FileUtils.getListArray(new File(listManager.listsPath, selectedBlacklistName)));
int count = 0;
while (otherBlacklist.hasNext()) {

5
htroot/yacy/list.java
Unescape View File

@ -28,7 +28,8 @@
import java.io.File;
import de.anomic.data.listManager;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.http.server.HeaderFramework;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
@ -79,7 +80,7 @@ public final class list {
if (blackListName.equals("") || filenamesarray[i].equals(blackListName)) {
final String filename = filenamesarray[i];
final File fileObj = new File(listsPath,filename);
out.append(listManager.getListString(fileObj, false)).append(serverCore.CRLF_STRING);
out.append(FileUtils.getListString(fileObj, false)).append(serverCore.CRLF_STRING);
}
}
}

202
source/de/anomic/data/listManager.java
Unescape View File

@ -24,30 +24,21 @@
package de.anomic.data;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.Vector;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;
import net.yacy.repository.BlacklistFile;
import de.anomic.search.SearchEventCache;
import de.anomic.search.Switchboard;
import de.anomic.server.serverCore;
// The Naming of the functions is a bit strange...
@ -118,199 +109,10 @@ public class listManager {
//================general Lists==================
/**
* Read lines of a file into an ArrayList.
*
* @param listFile the file
* @return the resulting array as an ArrayList
*/
public static ArrayList<String> getListArray(final File listFile){
String line;
final ArrayList<String> list = new ArrayList<String>();
int count = 0;
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile),"UTF-8"));
while((line = br.readLine()) != null){
list.add(line);
count++;
}
br.close();
} catch(final IOException e) {
// list is empty
} finally {
if (br!=null) try { br.close(); } catch (final Exception e) {}
}
return list;
}
/**
* Write a String to a file (used for string representation of lists).
*
* @param listFile the file to write to
* @param out the String to write
* @return returns <code>true</code> if successful, <code>false</code> otherwise
*/
public static boolean writeList(final File listFile, final String out) {
BufferedWriter bw = null;
try {
bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile)));
bw.write(out);
bw.close();
return true;
} catch(final IOException e) {
return false;
} finally {
if (bw!=null) try { bw.close(); } catch (final Exception e) {}
}
}
/**
* Write elements of an Array of Strings to a file (one element per line).
*
* @param listFile the file to write to
* @param list the Array to write
* @return returns <code>true</code> if successful, <code>false</code> otherwise
*/
public static boolean writeList(final File listFile, final String[] list){
final StringBuilder out = new StringBuilder();
for(int i=0;i < list.length; i++){
out
.append(list[i])
.append(serverCore.CRLF_STRING);
}
return writeList(listFile, new String(out)); //(File, String)
}
// same as below
public static String getListString(final String filename, final boolean withcomments) {
final File listFile = new File(listsPath ,filename);
return getListString(listFile, withcomments);
}
/**
* Read lines of a text file into a String, optionally ignoring comments.
*
* @param listFile the File to read from.
* @param withcomments If <code>false</code> ignore lines starting with '#'.
* @return String representation of the file content.
*/
public static String getListString(final File listFile, final boolean withcomments){
final StringBuilder temp = new StringBuilder();
BufferedReader br = null;
try{
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile)));
temp.ensureCapacity((int) listFile.length());
// Read the List
String line = "";
while ((line = br.readLine()) != null) {
if ((!line.startsWith("#") || withcomments) || !line.equals("")) {
//temp += line + serverCore.CRLF_STRING;
temp.append(line)
.append(serverCore.CRLF_STRING);
}
}
br.close();
} catch (final IOException e) {
} finally {
if (br!=null) try { br.close(); } catch (final Exception e) {}
}
return new String(temp);
return FileUtils.getListString(listFile, withcomments);
}
/**
* Read content of a directory into a String array of file names.
* @param dirname The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @return array of file names
*/
public static List<String> getDirListing(final String dirname){
return getDirListing(dirname, null);
}
/**
* Read content of a directory into a String array of file names.
* @param dirname The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @param filter String which contains a regular expression which has to be matched by
* file names in order to appear in returned array. All file names will be returned if
* filter is null.
* @return array of file names
*/
public static List<String> getDirListing(final String dirname, final String filter) {
return getDirListing(new File(dirname), filter);
}
/**
* Read content of a directory into a String array of file names.
*
* @param dir The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @return array of file names
*/
public static List<String> getDirListing(final File dir){
return getDirListing(dir, null);
}
/**
* Read content of a directory into a String array of file names.
* @param dir The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @param filter String which contains a regular expression which has to be matched by
* file names in order to appear in returned array. All file names will be returned if
* filter is null.
* @return array of file names
*/
public static List<String> getDirListing(final File dir, final String filter){
List<String> ret = new LinkedList<String>();
File[] fileList;
if (dir != null ) {
if (!dir.exists()) {
dir.mkdir();
}
fileList = dir.listFiles();
for (int i=0; i<= fileList.length-1; i++) {
if (filter == null || fileList[i].getName().matches(filter)) {
ret.add(fileList[i].getName());
}
}
return ret;
}
return null;
}
// same as below
public static ArrayList<File> getDirsRecursive(final File dir, final String notdir){
return getDirsRecursive(dir, notdir, true);
}
/**
* Returns a List of all dirs and subdirs as File Objects
*
* Warning: untested
*/
public static ArrayList<File> getDirsRecursive(final File dir, final String notdir, final boolean excludeDotfiles){
final File[] dirList = dir.listFiles();
final ArrayList<File> resultList = new ArrayList<File>();
ArrayList<File> recursive;
Iterator<File> iter;
for (int i=0;i<dirList.length;i++) {
if (dirList[i].isDirectory() && (!excludeDotfiles || !dirList[i].getName().startsWith(".")) && !dirList[i].getName().equals(notdir)) {
resultList.add(dirList[i]);
recursive = getDirsRecursive(dirList[i], notdir, excludeDotfiles);
iter=recursive.iterator();
while (iter.hasNext()) {
resultList.add(iter.next());
}
}
}
return resultList;
}
//================Helper functions for collection conversion==================

5
source/de/anomic/data/translator.java
Unescape View File

@ -49,6 +49,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.Formatter;
import de.anomic.server.serverSwitch;
@ -86,7 +87,7 @@ public class translator {
final Hashtable<String, Hashtable<String, String>> lists = new Hashtable<String, Hashtable<String, String>>(); //list of translationLists for different files.
Hashtable<String, String> translationList = new Hashtable<String, String>(); //current Translation Table
final ArrayList<String> list = listManager.getListArray(translationFile);
final ArrayList<String> list = FileUtils.getListArray(translationFile);
final Iterator<String> it = list.iterator();
String line = "";
String[] splitted;
@ -204,7 +205,7 @@ public class translator {
}
public static boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){
final ArrayList<File> dirList=listManager.getDirsRecursive(sourceDir, notdir);
final ArrayList<File> dirList=FileUtils.getDirsRecursive(sourceDir, notdir);
dirList.add(sourceDir);
final Iterator<File> it=dirList.iterator();
File file=null;

196
source/net/yacy/kelondro/util/FileUtils.java
Unescape View File

@ -30,6 +30,7 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@ -45,6 +46,8 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
@ -542,6 +545,199 @@ public final class FileUtils {
}
}
/**
* Read lines of a file into an ArrayList.
*
* @param listFile the file
* @return the resulting array as an ArrayList
*/
public static ArrayList<String> getListArray(final File listFile){
String line;
final ArrayList<String> list = new ArrayList<String>();
int count = 0;
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile),"UTF-8"));
while((line = br.readLine()) != null){
list.add(line);
count++;
}
br.close();
} catch(final IOException e) {
// list is empty
} finally {
if (br!=null) try { br.close(); } catch (final Exception e) {}
}
return list;
}
/**
* Write a String to a file (used for string representation of lists).
*
* @param listFile the file to write to
* @param out the String to write
* @return returns <code>true</code> if successful, <code>false</code> otherwise
*/
public static boolean writeList(final File listFile, final String out) {
BufferedWriter bw = null;
try {
bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile)));
bw.write(out);
bw.close();
return true;
} catch(final IOException e) {
return false;
} finally {
if (bw!=null) try { bw.close(); } catch (final Exception e) {}
}
}
public static final byte LF = 10;
public static final byte CR = 13;
/**
* Read lines of a text file into a String, optionally ignoring comments.
*
* @param listFile the File to read from.
* @param withcomments If <code>false</code> ignore lines starting with '#'.
* @return String representation of the file content.
*/
public static String getListString(final File listFile, final boolean withcomments){
final StringBuilder temp = new StringBuilder();
BufferedReader br = null;
try{
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile)));
temp.ensureCapacity((int) listFile.length());
// Read the List
String line = "";
while ((line = br.readLine()) != null) {
if ((!line.startsWith("#") || withcomments) || !line.equals("")) {
//temp += line + serverCore.CRLF_STRING;
temp.append(line).append(CR).append(LF);
}
}
br.close();
} catch (final IOException e) {
} finally {
if (br!=null) try { br.close(); } catch (final Exception e) {}
}
return new String(temp);
}
/**
* Read content of a directory into a String array of file names.
* @param dirname The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @return array of file names
*/
public static List<String> getDirListing(final String dirname){
return getDirListing(dirname, null);
}
/**
* Read content of a directory into a String array of file names.
* @param dirname The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @param filter String which contains a regular expression which has to be matched by
* file names in order to appear in returned array. All file names will be returned if
* filter is null.
* @return array of file names
*/
public static List<String> getDirListing(final String dirname, final String filter) {
return getDirListing(new File(dirname), filter);
}
/**
* Read content of a directory into a String array of file names.
*
* @param dir The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @return array of file names
*/
public static List<String> getDirListing(final File dir){
return getDirListing(dir, null);
}
/**
* Read content of a directory into a String array of file names.
* @param dir The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @param filter String which contains a regular expression which has to be matched by
* file names in order to appear in returned array. All file names will be returned if
* filter is null.
* @return array of file names
*/
public static List<String> getDirListing(final File dir, final String filter){
List<String> ret = new LinkedList<String>();
File[] fileList;
if (dir != null ) {
if (!dir.exists()) {
dir.mkdir();
}
fileList = dir.listFiles();
for (int i=0; i<= fileList.length-1; i++) {
if (filter == null || fileList[i].getName().matches(filter)) {
ret.add(fileList[i].getName());
}
}
return ret;
}
return null;
}
// same as below
public static ArrayList<File> getDirsRecursive(final File dir, final String notdir){
return getDirsRecursive(dir, notdir, true);
}
/**
* Returns a List of all dirs and subdirs as File Objects
*
* Warning: untested
*/
public static ArrayList<File> getDirsRecursive(final File dir, final String notdir, final boolean excludeDotfiles){
final File[] dirList = dir.listFiles();
final ArrayList<File> resultList = new ArrayList<File>();
ArrayList<File> recursive;
Iterator<File> iter;
for (int i=0;i<dirList.length;i++) {
if (dirList[i].isDirectory() && (!excludeDotfiles || !dirList[i].getName().startsWith(".")) && !dirList[i].getName().equals(notdir)) {
resultList.add(dirList[i]);
recursive = getDirsRecursive(dirList[i], notdir, excludeDotfiles);
iter=recursive.iterator();
while (iter.hasNext()) {
resultList.add(iter.next());
}
}
}
return resultList;
}
/**
* Write elements of an Array of Strings to a file (one element per line).
*
* @param listFile the file to write to
* @param list the Array to write
* @return returns <code>true</code> if successful, <code>false</code> otherwise
*/
public static boolean writeList(final File listFile, final String[] list){
final StringBuilder out = new StringBuilder();
for(int i=0;i < list.length; i++){
out.append(list[i]).append(CR).append(LF);
}
return FileUtils.writeList(listFile, new String(out)); //(File, String)
}
/*
public static ArrayList<String> strings(byte[] a) {
final ArrayList<String> list = new ArrayList<String>();

3
source/net/yacy/migration.java
Unescape View File

@ -28,7 +28,6 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.data.listManager;
import de.anomic.http.server.HTTPDemon;
import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
@ -88,7 +87,7 @@ public class migration {
final File skinsPath = sb.getConfigPath("skinPath", "DATA/SKINS");
final File defaultSkinsPath = new File(sb.getRootPath(), "skins");
if (defaultSkinsPath.exists()) {
final List<String> skinFiles = listManager.getDirListing(defaultSkinsPath.getAbsolutePath());
final List<String> skinFiles = FileUtils.getDirListing(defaultSkinsPath.getAbsolutePath());
mkdirs(skinsPath);
for (String skinFile : skinFiles){
if (skinFile.endsWith(".css")){

28
source/net/yacy/repository/Blacklist.java
Unescape View File

@ -34,14 +34,15 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.SetTools;
public class Blacklist {
@ -53,6 +54,8 @@ public class Blacklist {
public static final String BLACKLIST_SEARCH = "search";
public static final String BLACKLIST_SURFTIPS = "surftips";
public static final String BLACKLIST_NEWS = "news";
public final static String BLACKLIST_FILENAME_FILTER = "^.*\\.black$";
public static final int ERR_TWO_WILDCARDS_IN_HOST = 1;
public static final int ERR_SUBDOMAIN_XOR_WILDCARD = 2;
@ -98,9 +101,7 @@ public class Blacklist {
this.cachedUrlHashs.put(blacklistType, Collections.synchronizedSet(new HashSet<String>()));
}
}
public void setRootPath(final File rootPath) {
if (rootPath == null)
throw new NullPointerException("The blacklist root path must not be null.");
@ -458,5 +459,26 @@ public class Blacklist {
}
return ret;
}
public static final String defaultBlacklist(final File listsPath) {
List<String> dirlist = FileUtils.getDirListing(listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
if (dirlist.size() == 0) return null;
return dirlist.get(0);
}
/**
* Checks if a blacklist file contains a certain entry.
* @param blacklistToUse The blacklist.
* @param newEntry The Entry.
* @return True if file contains entry, else false.
*/
public static boolean blacklistFileContains(final File listsPath, final String blacklistToUse, String newEntry) {
boolean ret = false;
final HashSet<String> Blacklist = new HashSet<String>(FileUtils.getListArray(new File(listsPath, blacklistToUse)));
if (Blacklist != null) {
ret = Blacklist.contains(newEntry);
}
return ret;
}
}

2
source/net/yacy/repository/BlacklistFile.java
Unescape View File

@ -43,7 +43,7 @@ public class BlacklistFile {
/**
* Construct a unified array of file names from comma seperated file name
* Construct a unified array of file names from comma separated file name
* list.
*
* @return unified String array of file names

Write Preview
Loading…
Cancel
Save