yacy_search_server/htroot/WatchCrawler_p.java

// WatchCrawler_p.java
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 18.12.2006 on http://www.anomic.de
// this file was created using the an implementation from IndexCreate_p.java, published 02.12.2004
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
// 
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

import java.io.File;
import java.io.Writer;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterWriter;
import de.anomic.http.httpHeader;
import de.anomic.net.URL;
import de.anomic.plasma.plasmaCrawlProfile;
import de.anomic.plasma.plasmaCrawlZURL;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaURL;
import de.anomic.plasma.dbImport.dbImporter;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;

public class WatchCrawler_p {
	public static final String CRAWLING_MODE_URL = "url";
	public static final String CRAWLING_MODE_FILE = "file";
	public static final String CRAWLING_MODE_SITEMAP = "sitemap";
	

    // this servlet does NOT create the WatchCrawler page content!
    // this servlet starts a web crawl. The interface for entering the web crawl parameters is in IndexCreate_p.html
    
    public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
        // return variable that accumulates replacements
        plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
        serverObjects prop = new serverObjects();
        
        if (post == null) {
            // not a crawl start, only monitoring
            prop.put("info", 0);
        } else {
            prop.put("info", 0);
            
            if (post.containsKey("deleteprofile")) {
                // deletion of a crawl
                String handle = (String) post.get("handle");
                if (handle != null) switchboard.profiles.removeEntry(handle);
            }
            
            if (post.containsKey("continue")) {
                // continue queue
                String queue = post.get("continue", "");
                if (queue.equals("localcrawler")) {
                    switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
                } else if (queue.equals("remotecrawler")) {
                    switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
                }
            }

            if (post.containsKey("pause")) {
                // pause queue
                String queue = post.get("pause", "");
                if (queue.equals("localcrawler")) {
                    switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
                } else if (queue.equals("remotecrawler")) {
                    switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
                }
            }
            
            if (post.containsKey("crawlingstart")) {
                // init crawl
                if (yacyCore.seedDB == null) {
                    prop.put("info", 3);
                } else {
                    // set new properties
                    String newcrawlingfilter = post.get("crawlingFilter", ".*");
                    env.setConfig("crawlingFilter", newcrawlingfilter);
                    
                    int newcrawlingdepth = Integer.parseInt(post.get("crawlingDepth", "0"));
                    env.setConfig("crawlingDepth", Integer.toString(newcrawlingdepth));
                    
                    boolean crawlingIfOlderCheck = post.get("crawlingIfOlderCheck", "off").equals("on");
                    int crawlingIfOlderNumber = Integer.parseInt(post.get("crawlingIfOlderNumber", "-1"));
                    String crawlingIfOlderUnit = post.get("crawlingIfOlderUnit","year");
                    int crawlingIfOlder = recrawlIfOlderC(crawlingIfOlderCheck, crawlingIfOlderNumber, crawlingIfOlderUnit);                    
                    env.setConfig("crawlingIfOlder", crawlingIfOlder);
                    
                    boolean crawlingDomFilterCheck = post.get("crawlingDomFilterCheck", "off").equals("on");
                    int crawlingDomFilterDepth = (crawlingDomFilterCheck) ? Integer.parseInt(post.get("crawlingDomFilterDepth", "-1")) : -1;
                    env.setConfig("crawlingDomFilterDepth", Integer.toString(crawlingDomFilterDepth));
                    
                    boolean crawlingDomMaxCheck = post.get("crawlingDomMaxCheck", "off").equals("on");
                    int crawlingDomMaxPages = (crawlingDomMaxCheck) ? Integer.parseInt(post.get("crawlingDomMaxPages", "-1")) : -1;
                    env.setConfig("crawlingDomMaxPages", Integer.toString(crawlingDomMaxPages));
                    
                    boolean crawlingQ = post.get("crawlingQ", "off").equals("on");
                    env.setConfig("crawlingQ", (crawlingQ) ? "true" : "false");
                    
                    boolean indexText = post.get("indexText", "off").equals("on");
                    env.setConfig("indexText", (indexText) ? "true" : "false");
                    
                    boolean indexMedia = post.get("indexMedia", "off").equals("on");
                    env.setConfig("indexMedia", (indexMedia) ? "true" : "false");
                    
                    boolean storeHTCache = post.get("storeHTCache", "off").equals("on");
                    env.setConfig("storeHTCache", (storeHTCache) ? "true" : "false");
                    
                    boolean crawlOrder = post.get("crawlOrder", "off").equals("on");
                    env.setConfig("crawlOrder", (crawlOrder) ? "true" : "false");
                    
                    boolean xsstopw = post.get("xsstopw", "off").equals("on");
                    env.setConfig("xsstopw", (xsstopw) ? "true" : "false");
                    
                    boolean xdstopw = post.get("xdstopw", "off").equals("on");
                    env.setConfig("xdstopw", (xdstopw) ? "true" : "false");
                    
                    boolean xpstopw = post.get("xpstopw", "off").equals("on");
                    env.setConfig("xpstopw", (xpstopw) ? "true" : "false");
                    
                    String crawlingMode = post.get("crawlingMode","url");
                    if (crawlingMode.equals(CRAWLING_MODE_URL)) {
                        // getting the crawljob start url
                        String crawlingStart = post.get("crawlingURL","");
                        crawlingStart = crawlingStart.trim();
                        
                        // adding the prefix http:// if necessary
                        int pos = crawlingStart.indexOf("://");
                        if (pos == -1) crawlingStart = "http://" + crawlingStart;

                        // normalizing URL
                        try {crawlingStart = new URL(crawlingStart).toNormalform();} catch (MalformedURLException e1) {}
                        
                        // check if url is proper
                        URL crawlingStartURL = null;
                        try {
                            crawlingStartURL = new URL(crawlingStart);
                        } catch (MalformedURLException e) {
                            crawlingStartURL = null;
                        }
                        
                        // check if pattern matches
                        if ((crawlingStartURL == null) /* || (!(crawlingStart.matches(newcrawlingfilter))) */) {
                            // print error message
                            prop.put("info", 4); //crawlfilter does not match url
                            prop.put("info_newcrawlingfilter", newcrawlingfilter);
                            prop.put("info_crawlingStart", crawlingStart);
                        } else try {
                            
                            // check if the crawl filter works correctly
                            Pattern.compile(newcrawlingfilter);
                            
                            // stack request
                            // first delete old entry, if exists
                            String urlhash = plasmaURL.urlHash(crawlingStart);
                            switchboard.wordIndex.loadedURL.remove(urlhash);
                            switchboard.noticeURL.remove(urlhash);
                            switchboard.errorURL.remove(urlhash);
                            
                            // stack url
                            plasmaCrawlProfile.entry pe = switchboard.profiles.newEntry(
                                    crawlingStartURL.getHost(), crawlingStart, newcrawlingfilter, newcrawlingfilter,
                                    newcrawlingdepth, newcrawlingdepth,
                                    crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages,
                                    crawlingQ,
                                    indexText, indexMedia,
                                    storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw);
                            String reasonString = switchboard.sbStackCrawlThread.stackCrawl(crawlingStart, null, yacyCore.seedDB.mySeed.hash, "CRAWLING-ROOT", new Date(), 0, pe);
                            
                            if (reasonString == null) {
                                // liftoff!
                                prop.put("info", 8);//start msg
                                prop.put("info_crawlingURL", ((String) post.get("crawlingURL")));
                                
                                // generate a YaCyNews if the global flag was set
                                if (crawlOrder) {
                                    Map m = new HashMap(pe.map()); // must be cloned
                                    m.remove("specificDepth");
                                    m.remove("indexText");
                                    m.remove("indexMedia");
                                    m.remove("remoteIndexing");
                                    m.remove("xsstopw");
                                    m.remove("xpstopw");
                                    m.remove("xdstopw");
                                    m.remove("storeTXCache");
                                    m.remove("storeHTCache");
                                    m.remove("generalFilter");
                                    m.remove("specificFilter");
                                    m.put("intention", post.get("intention", "").replace(',', '/'));
                                    yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsPool.CATEGORY_CRAWL_START, m));
                                }
                                
                            } else {
                                prop.put("info", 5); //Crawling failed
                                prop.put("info_crawlingURL", ((String) post.get("crawlingURL")));
                                prop.put("info_reasonString", reasonString);
                                
                                plasmaCrawlZURL.Entry ee = switchboard.errorURL.newEntry(crawlingStartURL, reasonString);
                                ee.store();
                                switchboard.errorURL.stackPushEntry(ee);
                            }
                        } catch (PatternSyntaxException e) {
                            prop.put("info", 4); //crawlfilter does not match url
                            prop.put("info_newcrawlingfilter", newcrawlingfilter);
                            prop.put("info_error", e.getMessage());                                 
                        } catch (Exception e) {
                            // mist
                            prop.put("info", 6);//Error with url
                            prop.put("info_crawlingStart", crawlingStart);
                            prop.put("info_error", e.getMessage());
                            e.printStackTrace();
                        }                        
                        
                    } else if (crawlingMode.equals(CRAWLING_MODE_FILE)) {                        
                        if (post.containsKey("crawlingFile")) {
                            // getting the name of the uploaded file
                            String fileName = (String) post.get("crawlingFile");  
                            try {                     
                                // check if the crawl filter works correctly
                                Pattern.compile(newcrawlingfilter);                              
                                
                                // loading the file content
                                File file = new File(fileName);
                                
                                // getting the content of the bookmark file
                                byte[] fileContent = (byte[]) post.get("crawlingFile$file");
                                
                                // TODO: determine the real charset here ....
                                String fileString = new String(fileContent,"UTF-8");
                                
                                // parsing the bookmark file and fetching the headline and contained links
                                htmlFilterContentScraper scraper = new htmlFilterContentScraper(new URL(file));
                                //OutputStream os = new htmlFilterOutputStream(null, scraper, null, false);
                                Writer writer = new htmlFilterWriter(null,null,scraper,null,false);
                                serverFileUtils.write(fileString,writer);
                                writer.close();
                                
                                //String headline = scraper.getHeadline();
                                HashMap hyperlinks = (HashMap) scraper.getAnchors();
                                
                                // creating a crawler profile
                                plasmaCrawlProfile.entry profile = switchboard.profiles.newEntry(fileName, "file://" + file.toString(), newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw);
                                
                                // loop through the contained links
                                Iterator interator = hyperlinks.entrySet().iterator();
                                int c = 0;
                                while (interator.hasNext()) {
                                    Map.Entry e = (Map.Entry) interator.next();
                                    String nexturlstring = (String) e.getKey();
                                    
                                    if (nexturlstring == null) continue;
                                    
                                    nexturlstring = nexturlstring.trim();
                                    
                                    // normalizing URL
                                    nexturlstring = new URL(nexturlstring).toNormalform();                                    
                                    
                                    // generating an url object
                                    URL nexturlURL = null;
                                    try {
                                        nexturlURL = new URL(nexturlstring);
                                    } catch (MalformedURLException ex) {
                                        nexturlURL = null;
                                        c++;
                                        continue;
                                    }                                    
                                    
                                    // enqueuing the url for crawling
                                    String rejectReason = switchboard.sbStackCrawlThread.stackCrawl(nexturlstring, null, yacyCore.seedDB.mySeed.hash, (String)e.getValue(), new Date(), 1, profile);                                    
                                    
                                    // if something failed add the url into the errorURL list
                                    if (rejectReason == null) {
                                        c++;
                                    } else {
                                        plasmaCrawlZURL.Entry ee = switchboard.errorURL.newEntry(nexturlURL, rejectReason);
                                        ee.store();
                                        switchboard.errorURL.stackPushEntry(ee);
                                    }
                                }                             
                               
                            } catch (PatternSyntaxException e) {
                                // print error message
                                prop.put("info", 4); //crawlfilter does not match url
                                prop.put("info_newcrawlingfilter", newcrawlingfilter);
                                prop.put("info_error", e.getMessage());                            
                            } catch (Exception e) {
                                // mist
                                prop.put("info", 7);//Error with file
                                prop.put("info_crawlingStart", fileName);
                                prop.put("info_error", e.getMessage());
                                e.printStackTrace();                                
                            }
                        }                        
                    } else if (crawlingMode.equals(CRAWLING_MODE_SITEMAP)) { 
                    	String sitemapURLStr = null;
                    	try {
                    		// getting the sitemap URL
                    		sitemapURLStr = post.get("sitemapURL","");
                    		
                    		// create a new profile
                    		plasmaCrawlProfile.entry pe = switchboard.profiles.newEntry(
                    				sitemapURLStr, sitemapURLStr, newcrawlingfilter, newcrawlingfilter,
                    				newcrawlingdepth, newcrawlingdepth,
                    				crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages,
                    				crawlingQ,
                    				indexText, indexMedia,
                    				storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw);
                    		
                    		// create a new sitemap importer             
                    		dbImporter importerThread = switchboard.dbImportManager.getNewImporter("sitemap");
                    		if (importerThread != null) {
                    			HashMap initParams = new HashMap();
                    			initParams.put("sitemapURL",sitemapURLStr);
                    			initParams.put("crawlingProfile",pe.handle());
                    			
                    			importerThread.init(initParams);
                    			importerThread.startIt();                            
                    		}              
                    	} catch (Exception e) {
                    		// mist
                    		prop.put("info", 6);//Error with url
                    		prop.put("info_crawlingStart", sitemapURLStr);
                    		prop.put("info_error", e.getMessage());
                    		e.printStackTrace();                    		
                    	}
                    }
                }
            }
            
            if (post.containsKey("crawlingPerformance")) {
                setPerformance(switchboard, post);
            }
        }
        
        // crawl profiles
        int count = 0;
        int domlistlength = (post == null) ? 160 : post.getInt("domlistlength", 160);
        Iterator it = switchboard.profiles.profiles(true);
        plasmaCrawlProfile.entry profile;
        boolean dark = true;
        while (it.hasNext()) {
            profile = (plasmaCrawlProfile.entry) it.next();
            prop.put("crawlProfiles_"+count+"_dark", ((dark) ? 1 : 0));
            prop.put("crawlProfiles_"+count+"_name", profile.name());
            prop.put("crawlProfiles_"+count+"_startURL", profile.startURL());
            prop.put("crawlProfiles_"+count+"_handle", profile.handle());
            prop.put("crawlProfiles_"+count+"_depth", profile.generalDepth());
            prop.put("crawlProfiles_"+count+"_filter", profile.generalFilter());
            prop.put("crawlProfiles_"+count+"_crawlingIfOlder", (profile.recrawlIfOlder() == Long.MAX_VALUE) ? "no re-crawl" : ""+profile.recrawlIfOlder());
            prop.put("crawlProfiles_"+count+"_crawlingDomFilterDepth", (profile.domFilterDepth() == Integer.MAX_VALUE) ? "inactive" : Integer.toString(profile.domFilterDepth()));

            //start contrib [MN]
            int i = 0;
            String item;
            while((i <= domlistlength) && !((item = profile.domName(true, i)).equals(""))){
                if(i == domlistlength){
                    item = item + " ...";
                }
                prop.put("crawlProfiles_"+count+"_crawlingDomFilterContent_"+i+"_item", item);
                i++;
            }

            prop.put("crawlProfiles_"+count+"_crawlingDomFilterContent", i);
            //end contrib [MN]

            prop.put("crawlProfiles_"+count+"_crawlingDomMaxPages", (profile.domMaxPages() == Integer.MAX_VALUE) ? "unlimited" : ""+profile.domMaxPages());
            prop.put("crawlProfiles_"+count+"_withQuery", ((profile.crawlingQ()) ? 1 : 0));
            prop.put("crawlProfiles_"+count+"_storeCache", ((profile.storeHTCache()) ? 1 : 0));
            prop.put("crawlProfiles_"+count+"_indexText", ((profile.indexText()) ? 1 : 0));
            prop.put("crawlProfiles_"+count+"_indexMedia", ((profile.indexMedia()) ? 1 : 0));
            prop.put("crawlProfiles_"+count+"_remoteIndexing", ((profile.remoteIndexing()) ? 1 : 0));
            prop.put("crawlProfiles_"+count+"_deleteButton", (((profile.name().equals("remote")) ||
                                                               (profile.name().equals("proxy")) ||
                                                               (profile.name().equals("snippetText")) ||
                                                               (profile.name().equals("snippetMedia")) ? 0 : 1)));
            prop.put("crawlProfiles_"+count+"_deleteButton_handle", profile.handle());
            
            dark = !dark;
            count++;
        }
        prop.put("crawlProfiles", count);
        
        // performance settings
        long LCbusySleep = Integer.parseInt(env.getConfig(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP, "100"));
        int LCppm = (int) (60000L / LCbusySleep);
        prop.put("crawlingSpeedMaxChecked", (LCppm >= 1000) ? 1 : 0);
        prop.put("crawlingSpeedCustChecked", ((LCppm > 10) && (LCppm < 1000)) ? 1 : 0);
        prop.put("crawlingSpeedMinChecked", (LCppm <= 10) ? 1 : 0);
        prop.put("customPPMdefault", ((LCppm > 10) && (LCppm < 1000)) ? Integer.toString(LCppm) : "");
        
        // return rewrite properties
        return prop;
    }

    private static int recrawlIfOlderC(boolean recrawlIfOlderCheck, int recrawlIfOlderNumber, String crawlingIfOlderUnit) {
        if (!recrawlIfOlderCheck) return -1;
        if (crawlingIfOlderUnit.equals("year")) return recrawlIfOlderNumber * 60 * 24 * 365;
        if (crawlingIfOlderUnit.equals("month")) return recrawlIfOlderNumber * 60 * 24 * 30;
        if (crawlingIfOlderUnit.equals("day")) return recrawlIfOlderNumber * 60 * 24;
        if (crawlingIfOlderUnit.equals("hour")) return recrawlIfOlderNumber * 60;
        if (crawlingIfOlderUnit.equals("minute")) return recrawlIfOlderNumber;
        return -1;
    }
    
    private static void setPerformance(plasmaSwitchboard sb, serverObjects post) {
        String crawlingPerformance = post.get("crawlingPerformance","custom");
        long LCbusySleep = Integer.parseInt(sb.getConfig(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP, "100"));
        int wantedPPM = (int) (60000L / LCbusySleep);
        try {
            wantedPPM = Integer.parseInt(post.get("customPPM",Integer.toString(wantedPPM)));
        } catch (NumberFormatException e) {}
        if (crawlingPerformance.equals("minimum")) wantedPPM = 10;
        if (crawlingPerformance.equals("maximum")) wantedPPM = 1000;
        sb.setPerformance(wantedPPM);
    }
    
}