yacy_search_server/htroot/Blacklist_p.java

// Blacklist_p.java 
// -----------------------
// part of YaCy
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
//
// This File is contributed by Alexander Schier
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.

// You must compile this file with
// javac -classpath .:../classes Blacklist_p.java
// if the shell's current path is HTROOT

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.TreeMap;

import de.anomic.data.listManager;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.urlPattern.abstractURLPattern;
import de.anomic.plasma.urlPattern.plasmaURLPattern;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;

public class Blacklist_p {
	private final static String DISABLED         = "disabled_";
    private final static String BLACKLIST        = "blackLists_";
    private final static String BLACKLIST_SHARED = "BlackLists.Shared";

    public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {

        // initialize the list manager
        listManager.switchboard = (plasmaSwitchboard) env;
        listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
        
        // getting the list of supported blacklist types
        String supportedBlacklistTypesStr = abstractURLPattern.BLACKLIST_TYPES_STRING;
        String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");        
        
        String blacklistToUse = null;
        serverObjects prop = new serverObjects();
        prop.put("blacklistEngine", plasmaSwitchboard.urlBlacklist.getEngineInfo());
        
        // do all post operations
        if (post != null) {
            
            if(post.containsKey("testList")) {
            	prop.put("testlist",1);
            	String urlstring = post.get("testurl", "");
            	if(!urlstring.startsWith("http://")) urlstring = "http://"+urlstring;
                yacyURL testurl = null;
				try {
					testurl = new yacyURL(urlstring, null);
				} catch (MalformedURLException e) { }
				if(testurl != null) {
					prop.put("testlist_url",testurl.toString());
					if(plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, testurl))
						prop.put("testlist_listedincrawler",1);
					if(plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_DHT, testurl))
						prop.put("testlist_listedindht",1);
					if(plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_NEWS, testurl))
						prop.put("testlist_listedinnews",1);
					if(plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_PROXY, testurl))
						prop.put("testlist_listedinproxy",1);
					if(plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, testurl))
						prop.put("testlist_listedinsearch",1);
					if(plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_SURFTIPS, testurl))
						prop.put("testlist_listedinsurftips",1);
				}
				else prop.put("testlist_url","not valid");
            }
        	if (post.containsKey("selectList")) {
                blacklistToUse = (String)post.get("selectedListName"); 
                if (blacklistToUse != null && blacklistToUse.length() == 0) blacklistToUse = null;
            }
            if (post.containsKey("createNewList")) {
                /* ===========================================================
                 * Creation of a new blacklist
                 * =========================================================== */
                
                blacklistToUse = (String)post.get("newListName");
                if (blacklistToUse.trim().length() == 0) {
                    prop.put("LOCATION","");
                    return prop;
                }   
                
                if (!blacklistToUse.endsWith(".black")) blacklistToUse += ".black";

                try {
                    final File newFile = new File(listManager.listsPath, blacklistToUse);
                    newFile.createNewFile();
                    
                    // share the newly created blacklist
                    listManager.updateListSet(BLACKLIST_SHARED, blacklistToUse);
                    
                    // activate it for all known blacklist types
                    for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
                        listManager.updateListSet(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
                    }                                 
                } catch (IOException e) {/* */}
                
            } else if (post.containsKey("deleteList")) {
                /* ===========================================================
                 * Delete a blacklist
                 * =========================================================== */                
                
                blacklistToUse = (String)post.get("selectedListName");
                if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
                    prop.put("LOCATION","");
                    return prop;
                }                   
                
                File BlackListFile = new File(listManager.listsPath, blacklistToUse);
                BlackListFile.delete();

                for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
                    listManager.removeFromListSet(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
                }                
                
                // remove it from the shared list
                listManager.removeFromListSet(BLACKLIST_SHARED, blacklistToUse);
                blacklistToUse = null;
                
                // reload Blacklists
                listManager.reloadBlacklists();

            } else if (post.containsKey("activateList")) {

                /* ===========================================================
                 * Activate/Deactivate a blacklist
                 * =========================================================== */                   
                
                blacklistToUse = (String)post.get("selectedListName");
                if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
                    prop.put("LOCATION","");
                    return prop;
                }                   
                
                for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {                    
                    if (post.containsKey("activateList4" + supportedBlacklistTypes[blTypes])) {
                        listManager.updateListSet(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);                        
                    } else {
                        listManager.removeFromListSet(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);                        
                    }                    
                }                     

                listManager.reloadBlacklists();                
                
            } else if (post.containsKey("shareList")) {

                /* ===========================================================
                 * Share a blacklist
                 * =========================================================== */                   
                
                blacklistToUse = (String)post.get("selectedListName");
                if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
                    prop.put("LOCATION","");
                    return prop;
                }                   
                
                if (listManager.listSetContains(BLACKLIST_SHARED, blacklistToUse)) { 
                    // Remove from shared BlackLists
                    listManager.removeFromListSet(BLACKLIST_SHARED, blacklistToUse);
                } else { // inactive list -> enable
                    listManager.updateListSet(BLACKLIST_SHARED, blacklistToUse);
                }                                
            } else if (post.containsKey("deleteBlacklistEntry")) {
                
                /* ===========================================================
                 * Delete a blacklist entry
                 * =========================================================== */                     
                
                // get the current selected blacklist name
                blacklistToUse = (String)post.get("currentBlacklist");
                if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
                    prop.put("LOCATION","");
                    return prop;
                }                 
                
                // get the entry that should be deleted
                String oldEntry = (String)post.get("selectedEntry");
                if (oldEntry == null || oldEntry.trim().length() == 0) {
                    prop.put("LOCATION",header.get("PATH") + "?selectList=&selectedListName=" + blacklistToUse);
                    return prop;
                }                
                
                // load blacklist data from file
                ArrayList list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
                
                // delete the old entry from file
                if (list != null) {
                    for (int i=0; i < list.size(); i++) {
                        if (((String)list.get(i)).equals(oldEntry)) {
                            list.remove(i);
                            break;
                        }
                    }
                    listManager.writeList(new File(listManager.listsPath, blacklistToUse), (String[])list.toArray(new String[list.size()]));
                }
                
                // remove the entry from the running blacklist engine
                int pos = oldEntry.indexOf("/");
                if (pos < 0) {
                    // add default empty path pattern
                    pos = oldEntry.length();
                    oldEntry = oldEntry + "/.*";
                }                
                for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
                    if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
                        plasmaSwitchboard.urlBlacklist.remove(supportedBlacklistTypes[blTypes],oldEntry.substring(0, pos), oldEntry.substring(pos + 1));
                    }                
                }                    
                
            } else if (post.containsKey("addBlacklistEntry")) {
                
                /* ===========================================================
                 * Add a new blacklist entry
                 * =========================================================== */                     
                
                blacklistToUse = (String)post.get("currentBlacklist");   
                if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
                    prop.put("LOCATION","");
                    return prop;
                }                  
                
                String newEntry = (String)post.get("newEntry");
                if (newEntry.trim().length() == 0) {
                    prop.put("LOCATION",header.get("PATH") + "?selectList=&selectedListName=" + blacklistToUse);
                    return prop;
                }
                
                // TODO: ignore empty entries
                
                if (newEntry.startsWith("http://") ){
                    newEntry = newEntry.substring(7);
                }

                int pos = newEntry.indexOf("/");
                if (pos < 0) {
                    // add default empty path pattern
                    pos = newEntry.length();
                    newEntry = newEntry + "/.*";
                }

                // append the line to the file
                PrintWriter pw = null;
                try {
                    pw = new PrintWriter(new FileWriter(new File(listManager.listsPath, blacklistToUse), true));
                    pw.println(newEntry);
                    pw.close();
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    if (pw != null) try { pw.close(); } catch (Exception e){ /* */}
                }

                // add to blacklist
                for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
                    if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
                        plasmaSwitchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes],newEntry.substring(0, pos), newEntry.substring(pos + 1));
                    }
                }
            } else if (post.containsKey("moveBlacklistEntry")) {

                /* ===========================================================
                 * First add entry to blacklist item moves to
                 * =========================================================== */

                blacklistToUse = (String)post.get("targetBlacklist");
                if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
                    prop.put("LOCATION","");
                    return prop;
                }

                String entry = (String)post.get("selectedEntry");
                if (entry == null || entry.trim().length() == 0) {
                    prop.put("LOCATION",header.get("PATH") + "?selectList=&selectedListName=" + blacklistToUse);
                    return prop;
                }

                // TODO: ignore empty entries

                if (entry.startsWith("http://") ){
                    entry = entry.substring(7);
                }

                int pos = entry.indexOf("/");
                if (pos < 0) {
                    // add default empty path pattern
                    pos = entry.length();
                    entry = entry + "/.*";
                }

                // append the line to the file
                PrintWriter pw = null;
                try {
                    pw = new PrintWriter(new FileWriter(new File(listManager.listsPath, blacklistToUse), true));
                    pw.println(entry);
                    pw.close();
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    if (pw != null) try { pw.close(); } catch (Exception e){ /* */}
                }

                // add to blacklist
                for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
                    if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
                        plasmaSwitchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes],entry.substring(0, pos), entry.substring(pos + 1));
                    }
                }

                /* ===========================================================
                 * Then delete item from blacklist it is moved away from
                 * =========================================================== */

                // get the current selected blacklist name
                blacklistToUse = (String)post.get("currentBlacklist");
                if (blacklistToUse == null || blacklistToUse.trim().length() == 0) {
                    prop.put("LOCATION","");
                    return prop;
                }

                // load blacklist data from file
                ArrayList list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));

                // delete the old entry from file
                if (list != null) {
                    for (int i=0; i < list.size(); i++) {
                        if (((String)list.get(i)).equals(entry)) {
                            list.remove(i);
                            break;
                        }
                    }
                    listManager.writeList(new File(listManager.listsPath, blacklistToUse), (String[])list.toArray(new String[list.size()]));
                }

                // remove the entry from the running blacklist engine
                pos = entry.indexOf("/");
                if (pos < 0) {
                    // add default empty path pattern
                    pos = entry.length();
                    entry = entry + "/.*";
                }
                for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
                    if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
                        plasmaSwitchboard.urlBlacklist.remove(supportedBlacklistTypes[blTypes],entry.substring(0, pos), entry.substring(pos + 1));
                    }
                }



            }

        }

        // loading all blacklist files located in the directory
        String[] dirlist = listManager.getDirListing(listManager.listsPath);
        
        // if we have not chosen a blacklist until yet we use the first file
        if (blacklistToUse == null && dirlist != null && dirlist.length > 0) {
            blacklistToUse = dirlist[0];
        }
        

        // Read the blacklist items from file
        if (blacklistToUse != null) {
            int entryCount = 0;
            final ArrayList list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
            
            // sort them
            String[] sortedlist = new String[list.size()];
            Arrays.sort(list.toArray(sortedlist));
            
            // display them
            for (int j=0;j<sortedlist.length;++j){
                String nextEntry = sortedlist[j];
                
                if (nextEntry.length() == 0) continue;
                if (nextEntry.startsWith("#")) continue;
    
                prop.put(DISABLED + "Itemlist_" + entryCount + "_item", nextEntry);
                entryCount++;
            }
        	prop.put(DISABLED + "Itemlist", entryCount);            


	        // List known hosts for BlackList retrieval
	        if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { // no nullpointer error
	            int peerCount = 0;
	            try {
	                TreeMap hostList = new TreeMap();
	                final Iterator e = yacyCore.seedDB.seedsConnected(true, false, null, (float) 0.0);
	                while (e.hasNext()) {
	                    yacySeed seed = (yacySeed) e.next();
	                    if (seed != null) hostList.put(seed.get(yacySeed.NAME, "nameless"),seed.hash);
	                }
	
	                String peername;
	                while ((peername = (String) hostList.firstKey()) != null) {
	                    final String Hash = (String) hostList.get(peername);
	                    prop.put(DISABLED + "otherHosts_" + peerCount + "_hash", Hash);
	                    prop.put(DISABLED + "otherHosts_" + peerCount + "_name", peername);
	                    hostList.remove(peername);
	                    peerCount++;
	                }
	            } catch (Exception e) {/* */}
	            prop.put(DISABLED + "otherHosts", peerCount);
	        }
        }
        
        // List BlackLists
        int blacklistCount = 0;
        if (dirlist != null) {
            for (int i = 0; i <= dirlist.length - 1; i++) {
                prop.put(DISABLED + BLACKLIST + blacklistCount + "_name", dirlist[i]);
                prop.put(DISABLED + BLACKLIST + blacklistCount + "_selected", 0);

                if (dirlist[i].equals(blacklistToUse)) { //current List
                    prop.put(DISABLED + BLACKLIST + blacklistCount + "_selected", 1);

                    for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
                        prop.put(DISABLED + "currentActiveFor_" + blTypes + "_blTypeName",supportedBlacklistTypes[blTypes]);
                        prop.put(DISABLED + "currentActiveFor_" + blTypes + "_checked",
                                listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",dirlist[i])?0:1);
                    }
                    prop.put(DISABLED + "currentActiveFor",supportedBlacklistTypes.length);

                }
                
                if (listManager.listSetContains(BLACKLIST_SHARED, dirlist[i])) {
                    prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", 1);
                } else {
                    prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", 0);
                }

                int activeCount = 0;
                for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
                    if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",dirlist[i])) {
                        prop.put(DISABLED + BLACKLIST + blacklistCount + "_active_" + activeCount + "_blTypeName",supportedBlacklistTypes[blTypes]);
                        activeCount++;
                    }                
                }          
                prop.put(DISABLED + BLACKLIST + blacklistCount + "_active",activeCount);
                blacklistCount++;
            }
        }
        prop.put(DISABLED + "blackLists", blacklistCount);
        
        prop.put(DISABLED + "currentBlacklist", (blacklistToUse==null)?"":blacklistToUse);
        prop.put("disabled", (blacklistToUse == null) ? 1 : 0);
        return prop;
    }

}