// plasmaSwitchboard.java // (C) 2004-2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // first published 2004 on http://yacy.net // // This is a part of YaCy, a peer-to-peer based web search engine // // $LastChangedDate$ // $LastChangedRevision$ // $LastChangedBy$ // // LICENSE // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA /* This class holds the run-time environment of the plasma Search Engine. It's data forms a blackboard which can be used to organize running jobs around the indexing algorithm. The blackboard consist of the following entities: - storage: one plasmaStore object with the url-based database - configuration: initialized by properties once, then by external functions - job queues: for parsing, condensing, indexing - black/blue/whitelists: controls input and output to the index this class is also the core of the http crawling. There are some items that need to be respected when crawling the web: 1) respect robots.txt 2) do not access one domain too frequently, wait between accesses 3) remember crawled URL's and do not access again too early 4) priorization of specific links should be possible (hot-lists) 5) attributes for crawling (depth, filters, hot/black-lists, priority) 6) different crawling jobs with different attributes ('Orders') simultanoulsy We implement some specific tasks and use different database to archieve these goals: - a database 'crawlerDisallow.db' contains all url's that shall not be crawled - a database 'crawlerDomain.db' holds all domains and access times, where we loaded the disallow tables this table contains the following entities: - four databases for scheduled access: crawlerScheduledHotText.db, crawlerScheduledColdText.db, crawlerScheduledHotMedia.db and crawlerScheduledColdMedia.db - two stacks for new URLS: newText.stack and newMedia.stack - two databases for URL double-check: knownText.db and knownMedia.db - one database with crawling orders: crawlerOrders.db The Information flow of a single URL that is crawled is as follows: - a html file is loaded from a specific URL within the module httpdProxyServlet as a process of the proxy. - the file is passed to httpdProxyCache. Here it's processing is delayed until the proxy is idle. - The cache entry is passed on to the plasmaSwitchboard. There the URL is stored into plasmaLURL where the URL is stored under a specific hash. The URL's from the content are stripped off, stored in plasmaLURL with a 'wrong' date (the date of the URL's are not known at this time, only after fetching) and stacked with plasmaCrawlerTextStack. The content is read and splitted into rated words in plasmaCondenser. The splitted words are then integrated into the index with plasmaSearch. - In plasmaSearch the words are indexed by reversing the relation between URL and words: one URL points to many words, the words within the document at the URL. After reversing, one word points to many URL's, all the URL's where the word occurrs. One single word->URL-hash relation is stored in plasmaIndexEntry. A set of plasmaIndexEntries is a reverse word index. This reverse word index is stored temporarly in plasmaIndexCache. - In plasmaIndexCache the single plasmaIndexEntry'ies are collected and stored into a plasmaIndex - entry These plasmaIndex - Objects are the true reverse words indexes. - in plasmaIndex the plasmaIndexEntry - objects are stored in a kelondroTree; an indexed file in the file system. The information flow of a search request is as follows: - in httpdFileServlet the user enters a search query, which is passed to plasmaSwitchboard - in plasmaSwitchboard, the query is passed to plasmaSearch. - in plasmaSearch, the plasmaSearch.result object is generated by simultanous enumeration of URL hases in the reverse word indexes plasmaIndex - (future: the plasmaSearch.result - object is used to identify more key words for a new search) */ package de.anomic.plasma; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.net.MalformedURLException; import java.security.NoSuchAlgorithmException; import java.security.PublicKey; import java.security.spec.InvalidKeySpecException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.Hashtable; import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Pattern; import de.anomic.content.DCEntry; import de.anomic.content.RSSMessage; import de.anomic.content.file.SurrogateReader; import de.anomic.crawler.CrawlEntry; import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.CrawlQueues; import de.anomic.crawler.CrawlStacker; import de.anomic.crawler.CrawlSwitchboard; import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.ImporterManager; import de.anomic.crawler.IndexingStack; import de.anomic.crawler.NoticedURL; import de.anomic.crawler.ProtocolLoader; import de.anomic.crawler.ResourceObserver; import de.anomic.crawler.ResultImages; import de.anomic.crawler.ResultURLs; import de.anomic.crawler.RobotsTxt; import de.anomic.crawler.ZURL; import de.anomic.crawler.CrawlProfile.entry; import de.anomic.crawler.IndexingStack.QueueEntry; import de.anomic.data.Blacklist; import de.anomic.data.URLLicense; import de.anomic.data.blogBoard; import de.anomic.data.blogBoardComments; import de.anomic.data.bookmarksDB; import de.anomic.data.listManager; import de.anomic.data.messageBoard; import de.anomic.data.userDB; import de.anomic.data.wiki.wikiBoard; import de.anomic.data.wiki.wikiCode; import de.anomic.data.wiki.wikiParser; import de.anomic.http.httpClient; import de.anomic.http.httpRemoteProxyConfig; import de.anomic.http.httpRequestHeader; import de.anomic.http.httpResponseHeader; import de.anomic.http.httpd; import de.anomic.http.httpdRobotsTxtConfig; import de.anomic.kelondro.order.Digest; import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.text.Segment; import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow; import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.Log; import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.SetTools; import de.anomic.net.UPnP; import de.anomic.plasma.parser.Document; import de.anomic.plasma.parser.ParserException; import de.anomic.plasma.parser.Word; import de.anomic.plasma.parser.Condenser; import de.anomic.server.serverAbstractSwitch; import de.anomic.server.serverBusyThread; import de.anomic.server.serverCore; import de.anomic.server.serverDomains; import de.anomic.server.serverInstantBusyThread; import de.anomic.server.serverObjects; import de.anomic.server.serverProcessor; import de.anomic.server.serverProcessorJob; import de.anomic.server.serverProfiling; import de.anomic.server.serverSemaphore; import de.anomic.server.serverSwitch; import de.anomic.server.serverSystem; import de.anomic.server.serverThread; import de.anomic.tools.crypt; import de.anomic.tools.CryptoLib; import de.anomic.xml.RSSFeed; import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; import de.anomic.yacy.Tray; import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacyURL; import de.anomic.yacy.yacyUpdateLocation; import de.anomic.yacy.yacyRelease; import de.anomic.yacy.yacyVersion; import de.anomic.yacy.dht.Dispatcher; import de.anomic.yacy.dht.PeerSelection; public final class plasmaSwitchboard extends serverAbstractSwitch implements serverSwitch { // load slots public static int xstackCrawlSlots = 2000; private int dhtMaxContainerCount = 500; private int dhtMaxReferenceCount = 1000; public static long lastPPMUpdate = System.currentTimeMillis()- 30000; // colored list management public static TreeSet badwords = null; public static TreeSet blueList = null; public static TreeSet stopwords = null; public static TreeSet badwordHashes = null; public static TreeSet blueListHashes = null; public static TreeSet stopwordHashes = null; public static Blacklist urlBlacklist = null; public static wikiParser wikiParser = null; // storage management public File htCachePath; public File plasmaPath; public File listsPath; public File htDocsPath; public File rankingPath; public File workPath; public File releasePath; public File surrogatesInPath; public File surrogatesOutPath; public Map rankingPermissions; public Segment indexSegment; public CrawlSwitchboard crawler; public CrawlQueues crawlQueues; public ResultURLs crawlResults; public CrawlStacker crawlStacker; public messageBoard messageDB; public wikiBoard wikiDB; public blogBoard blogDB; public blogBoardComments blogCommentDB; public RobotsTxt robots; public boolean rankingOn; public plasmaRankingDistribution rankingOwnDistribution; public plasmaRankingDistribution rankingOtherDistribution; public HashMap outgoingCookies, incomingCookies; public plasmaParser parser; public volatile long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess; public yacyCore yc; public ResourceObserver observer; public userDB userDB; public bookmarksDB bookmarksDB; public plasmaWebStructure webStructure; public ImporterManager dbImportManager; public ArrayList localSearches; // array of search result properties as HashMaps public ArrayList remoteSearches; // array of search result properties as HashMaps public HashMap> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time) public long indexedPages = 0; public double requestedQueries = 0d; public double totalQPM = 0d; public TreeMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used public URLLicense licensedURLs; public List networkWhitelist, networkBlacklist; public Dispatcher dhtDispatcher; public List trail; public yacySeedDB peers; public serverProcessor indexingDocumentProcessor; public serverProcessor indexingCondensementProcessor; public serverProcessor indexingAnalysisProcessor; public serverProcessor indexingStorageProcessor; public httpdRobotsTxtConfig robotstxtConfig = null; private final serverSemaphore shutdownSync = new serverSemaphore(0); private boolean terminate = false; //private Object crawlingPausedSync = new Object(); //private boolean crawlingIsPaused = false; public Hashtable crawlJobsStatus = new Hashtable(); private static plasmaSwitchboard sb = null; public plasmaSwitchboard(final File rootPath, final String initPath, final String configPath, final boolean applyPro) throws IOException { super(rootPath, initPath, configPath, applyPro); serverProfiling.startSystemProfiling(); sb=this; // set loglevel and log setLog(new Log("PLASMA")); if (applyPro) this.log.logInfo("This is the pro-version of YaCy"); // UPnP port mapping if (getConfigBool(plasmaSwitchboardConstants.UPNP_ENABLED, false)) serverInstantBusyThread.oneTimeJob(UPnP.class, "addPortMapping", UPnP.log, 0); // init TrayIcon if possible Tray.init(this); // remote proxy configuration httpRemoteProxyConfig.init(this); // load values from configs this.plasmaPath = getConfigPath(plasmaSwitchboardConstants.PLASMA_PATH, plasmaSwitchboardConstants.PLASMA_PATH_DEFAULT); this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString()); final File indexPrimaryPath = getConfigPath(plasmaSwitchboardConstants.INDEX_PRIMARY_PATH, plasmaSwitchboardConstants.INDEX_PATH_DEFAULT); this.log.logConfig("Index Primary Path: " + indexPrimaryPath.toString()); this.listsPath = getConfigPath(plasmaSwitchboardConstants.LISTS_PATH, plasmaSwitchboardConstants.LISTS_PATH_DEFAULT); this.log.logConfig("Lists Path: " + this.listsPath.toString()); this.htDocsPath = getConfigPath(plasmaSwitchboardConstants.HTDOCS_PATH, plasmaSwitchboardConstants.HTDOCS_PATH_DEFAULT); this.log.logConfig("HTDOCS Path: " + this.htDocsPath.toString()); this.rankingPath = getConfigPath(plasmaSwitchboardConstants.RANKING_PATH, plasmaSwitchboardConstants.RANKING_PATH_DEFAULT); this.log.logConfig("Ranking Path: " + this.rankingPath.toString()); this.rankingPermissions = new HashMap(); // mapping of permission - to filename. this.workPath = getConfigPath(plasmaSwitchboardConstants.WORK_PATH, plasmaSwitchboardConstants.WORK_PATH_DEFAULT); this.log.logConfig("Work Path: " + this.workPath.toString()); // set a high maximum cache size to current size; this is adopted later automatically final int wordCacheMaxCount = (int) getConfigLong(plasmaSwitchboardConstants.WORDCACHE_MAX_COUNT, 20000); setConfig(plasmaSwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount)); // set network-specific performance attributes if (this.firstInit) { setRemotecrawlPPM(Math.max(1, (int) getConfigLong("network.unit.remotecrawl.speed", 60))); } // load the network definition overwriteNetworkDefinition(); // start indexing management log.logConfig("Starting Indexing Management"); final String networkName = getConfig(plasmaSwitchboardConstants.NETWORK_NAME, ""); final long fileSizeMax = (serverSystem.isWindows) ? sb.getConfigLong("filesize.max.win", (long) Integer.MAX_VALUE) : sb.getConfigLong("filesize.max.other", (long) Integer.MAX_VALUE); final int redundancy = (int) sb.getConfigLong("network.unit.dhtredundancy.senior", 1); final int partitionExponent = (int) sb.getConfigLong("network.unit.dht.partitionExponent", 0); try { final File networkRoot = new File(new File(indexPrimaryPath, networkName), "NETWORK"); networkRoot.mkdirs(); final File mySeedFile = new File(networkRoot, yacySeedDB.DBFILE_OWN_SEED); peers = new yacySeedDB( networkRoot, "seed.new.heap", "seed.old.heap", "seed.pot.heap", mySeedFile, redundancy, partitionExponent); indexSegment = new Segment( log, new File(new File(indexPrimaryPath, networkName), "TEXT"), wordCacheMaxCount, fileSizeMax); crawler = new CrawlSwitchboard( peers, networkName, log, new File(new File(indexPrimaryPath, networkName), "QUEUES")); } catch (IOException e1) { e1.printStackTrace(); indexSegment = null; } crawlResults = new ResultURLs(); // start yacy core log.logConfig("Starting YaCy Protocol Core"); this.yc = new yacyCore(this); serverInstantBusyThread.oneTimeJob(this, "loadSeedLists", yacyCore.log, 0); //final long startedSeedListAquisition = System.currentTimeMillis(); // init a DHT transmission dispatcher this.dhtDispatcher = new Dispatcher( indexSegment.termIndex(), indexSegment.urlMetadata(), peers, true, 30000); // set up local robots.txt this.robotstxtConfig = httpdRobotsTxtConfig.init(this); // setting timestamp of last proxy access this.proxyLastAccess = System.currentTimeMillis() - 10000; this.localSearchLastAccess = System.currentTimeMillis() - 10000; this.remoteSearchLastAccess = System.currentTimeMillis() - 10000; this.webStructure = new plasmaWebStructure(log, rankingPath, "LOCAL/010_cr/", getConfig("CRDist0Path", plasmaRankingDistribution.CR_OWN), new File(plasmaPath, "webStructure.map")); // configuring list path if (!(listsPath.exists())) listsPath.mkdirs(); // load coloured lists if (blueList == null) { // read only once upon first instantiation of this class final String f = getConfig(plasmaSwitchboardConstants.LIST_BLUE, plasmaSwitchboardConstants.LIST_BLUE_DEFAULT); final File plasmaBlueListFile = new File(f); if (f != null) blueList = SetTools.loadList(plasmaBlueListFile, NaturalOrder.naturalComparator); else blueList= new TreeSet(); blueListHashes = Word.words2hashes(blueList); this.log.logConfig("loaded blue-list from file " + plasmaBlueListFile.getName() + ", " + blueList.size() + " entries, " + ppRamString(plasmaBlueListFile.length()/1024)); } // load the black-list / inspired by [AS] final File blacklistsPath = getConfigPath(plasmaSwitchboardConstants.LISTS_PATH, plasmaSwitchboardConstants.LISTS_PATH_DEFAULT); String[] blacklistClassName = new String[] { getConfig(plasmaSwitchboardConstants.BLACKLIST_CLASS, plasmaSwitchboardConstants.BLACKLIST_CLASS_DEFAULT), plasmaSwitchboardConstants.BLACKLIST_CLASS_DEFAULT }; this.log.logConfig("Starting blacklist engine ..."); urlBlacklist = null; for (int i = 0; i < blacklistClassName.length; i++) { try { final Class blacklistClass = Class.forName(blacklistClassName[i]); final Constructor blacklistClassConstr = blacklistClass.getConstructor( new Class[] { File.class } ); urlBlacklist = (Blacklist) blacklistClassConstr.newInstance(new Object[] { blacklistsPath }); this.log.logFine("Used blacklist engine class: " + blacklistClassName); this.log.logConfig("Using blacklist engine: " + urlBlacklist.getEngineInfo()); break; } catch (final Exception e) { continue; // try next } catch (final Error e) { continue; // try next } } if (urlBlacklist == null) { this.log.logSevere("Unable to load the blacklist engine"); System.exit(-1); } this.log.logConfig("Loading backlist data ..."); listManager.switchboard = this; listManager.listsPath = blacklistsPath; listManager.reloadBlacklists(); // load badwords (to filter the topwords) if (badwords == null) { final File badwordsFile = new File(rootPath, plasmaSwitchboardConstants.LIST_BADWORDS_DEFAULT); badwords = SetTools.loadList(badwordsFile, NaturalOrder.naturalComparator); badwordHashes = Word.words2hashes(badwords); this.log.logConfig("loaded badwords from file " + badwordsFile.getName() + ", " + badwords.size() + " entries, " + ppRamString(badwordsFile.length()/1024)); } // load stopwords if (stopwords == null) { final File stopwordsFile = new File(rootPath, plasmaSwitchboardConstants.LIST_STOPWORDS_DEFAULT); stopwords = SetTools.loadList(stopwordsFile, NaturalOrder.naturalComparator); stopwordHashes = Word.words2hashes(stopwords); this.log.logConfig("loaded stopwords from file " + stopwordsFile.getName() + ", " + stopwords.size() + " entries, " + ppRamString(stopwordsFile.length()/1024)); } // load ranking tables final File YBRPath = new File(rootPath, "ranking/YBR"); if (YBRPath.exists()) { plasmaSearchRankingProcess.loadYBR(YBRPath, 15); } // loading the robots.txt db this.log.logConfig("Initializing robots.txt DB"); final File robotsDBFile = new File(this.plasmaPath, "crawlRobotsTxt.heap"); robots = new RobotsTxt(robotsDBFile); this.log.logConfig("Loaded robots.txt DB from file " + robotsDBFile.getName() + ", " + robots.size() + " entries" + ", " + ppRamString(robotsDBFile.length()/1024)); // start a cache manager log.logConfig("Starting HT Cache Manager"); // create the cache directory htCachePath = getConfigPath(plasmaSwitchboardConstants.HTCACHE_PATH, plasmaSwitchboardConstants.HTCACHE_PATH_DEFAULT); this.log.logInfo("HTCACHE Path = " + htCachePath.getAbsolutePath()); final long maxCacheSize = 1024 * 1024 * Long.parseLong(getConfig(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte plasmaHTCache.init(htCachePath, peers.mySeed().hash, maxCacheSize); // create the surrogates directories surrogatesInPath = getConfigPath(plasmaSwitchboardConstants.SURROGATES_IN_PATH, plasmaSwitchboardConstants.SURROGATES_IN_PATH_DEFAULT); this.log.logInfo("surrogates.in Path = " + surrogatesInPath.getAbsolutePath()); surrogatesInPath.mkdirs(); surrogatesOutPath = getConfigPath(plasmaSwitchboardConstants.SURROGATES_OUT_PATH, plasmaSwitchboardConstants.SURROGATES_OUT_PATH_DEFAULT); this.log.logInfo("surrogates.out Path = " + surrogatesOutPath.getAbsolutePath()); surrogatesOutPath.mkdirs(); // create the release download directory releasePath = getConfigPath(plasmaSwitchboardConstants.RELEASE_PATH, plasmaSwitchboardConstants.RELEASE_PATH_DEFAULT); releasePath.mkdirs(); this.log.logInfo("RELEASE Path = " + releasePath.getAbsolutePath()); // starting message board initMessages(); // starting wiki initWiki(); //starting blog initBlog(); // Init User DB this.log.logConfig("Loading User DB"); final File userDbFileOld = new File(getRootPath(), "DATA/SETTINGS/user.db"); final File userDbFile = new File(getRootPath(), "DATA/SETTINGS/user.heap"); this.userDB = new userDB(userDbFileOld, userDbFile); this.log.logConfig("Loaded User DB from file " + userDbFile.getName() + ", " + this.userDB.size() + " entries" + ", " + ppRamString(userDbFile.length()/1024)); //Init bookmarks DB initBookmarks(); // make parser log.logConfig("Starting Parser"); this.parser = new plasmaParser(); // define an extension-blacklist log.logConfig("Parser: Initializing Extension Mappings for Media/Parser"); plasmaParser.initMediaExt(plasmaParser.extString2extList(getConfig(plasmaSwitchboardConstants.PARSER_MEDIA_EXT,""))); plasmaParser.initSupportedHTMLFileExt(plasmaParser.extString2extList(getConfig(plasmaSwitchboardConstants.PARSER_MEDIA_EXT_PARSEABLE,""))); // define a realtime parsable mimetype list log.logConfig("Parser: Initializing Mime Types"); plasmaParser.initHTMLParsableMimeTypes(getConfig(plasmaSwitchboardConstants.PARSER_MIMETYPES_HTML, "application/xhtml+xml,text/html,text/plain")); plasmaParser.initParseableMimeTypes(plasmaParser.PARSER_MODE_PROXY, getConfig(plasmaSwitchboardConstants.PARSER_MIMETYPES_PROXY, null)); plasmaParser.initParseableMimeTypes(plasmaParser.PARSER_MODE_CRAWLER, getConfig(plasmaSwitchboardConstants.PARSER_MIMETYPES_CRAWLER, null)); plasmaParser.initParseableMimeTypes(plasmaParser.PARSER_MODE_ICAP, getConfig(plasmaSwitchboardConstants.PARSER_MIMETYPES_ICAP, null)); plasmaParser.initParseableMimeTypes(plasmaParser.PARSER_MODE_URLREDIRECTOR, getConfig(plasmaSwitchboardConstants.PARSER_MIMETYPES_URLREDIRECTOR, null)); plasmaParser.initParseableMimeTypes(plasmaParser.PARSER_MODE_IMAGE, getConfig(plasmaSwitchboardConstants.PARSER_MIMETYPES_IMAGE, null)); // start a loader log.logConfig("Starting Crawl Loader"); this.crawlQueues = new CrawlQueues(this, plasmaPath); this.crawlQueues.noticeURL.setMinimumDelta( this.getConfigLong("minimumLocalDelta", this.crawlQueues.noticeURL.getMinimumLocalDelta()), this.getConfigLong("minimumGlobalDelta", this.crawlQueues.noticeURL.getMinimumGlobalDelta())); /* * Creating sync objects and loading status for the crawl jobs * a) local crawl * b) remote triggered crawl * c) global crawl trigger */ this.crawlJobsStatus.put(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, new Object[]{ new Object(), Boolean.valueOf(getConfig(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL + "_isPaused", "false"))}); this.crawlJobsStatus.put(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, new Object[]{ new Object(), Boolean.valueOf(getConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL + "_isPaused", "false"))}); this.crawlJobsStatus.put(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, new Object[]{ new Object(), Boolean.valueOf(getConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER + "_isPaused", "false"))}); // init cookie-Monitor this.log.logConfig("Starting Cookie Monitor"); this.outgoingCookies = new HashMap(); this.incomingCookies = new HashMap(); // init search history trackers this.localSearchTracker = new HashMap>(); // String:TreeSet - IP:set of Long(accessTime) this.remoteSearchTracker = new HashMap>(); this.localSearches = new ArrayList(); // contains search result properties as HashMaps this.remoteSearches = new ArrayList(); // init messages: clean up message symbol final File notifierSource = new File(getRootPath(), getConfig(plasmaSwitchboardConstants.HTROOT_PATH, plasmaSwitchboardConstants.HTROOT_PATH_DEFAULT) + "/env/grafics/empty.gif"); final File notifierDest = new File(getConfigPath(plasmaSwitchboardConstants.HTDOCS_PATH, plasmaSwitchboardConstants.HTDOCS_PATH_DEFAULT), "notifier.gif"); try { FileUtils.copy(notifierSource, notifierDest); } catch (final IOException e) { } // clean up profiles this.log.logConfig("Cleaning Profiles"); try { cleanProfiles(); } catch (final InterruptedException e) { /* Ignore this here */ } // init ranking transmission /* CRDistOn = true/false CRDist0Path = GLOBAL/010_owncr CRDist0Method = 1 CRDist0Percent = 0 CRDist0Target = CRDist1Path = GLOBAL/014_othercr/1 CRDist1Method = 9 CRDist1Percent = 30 CRDist1Target = kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080 **/ rankingOn = getConfig(plasmaSwitchboardConstants.RANKING_DIST_ON, "true").equals("true") && networkName.equals("freeworld"); rankingOwnDistribution = new plasmaRankingDistribution(log, peers, new File(rankingPath, getConfig(plasmaSwitchboardConstants.RANKING_DIST_0_PATH, plasmaRankingDistribution.CR_OWN)), (int) getConfigLong(plasmaSwitchboardConstants.RANKING_DIST_0_METHOD, plasmaRankingDistribution.METHOD_ANYSENIOR), (int) getConfigLong(plasmaSwitchboardConstants.RANKING_DIST_0_METHOD, 0), getConfig(plasmaSwitchboardConstants.RANKING_DIST_0_TARGET, "")); rankingOtherDistribution = new plasmaRankingDistribution(log, peers, new File(rankingPath, getConfig(plasmaSwitchboardConstants.RANKING_DIST_1_PATH, plasmaRankingDistribution.CR_OTHER)), (int) getConfigLong(plasmaSwitchboardConstants.RANKING_DIST_1_METHOD, plasmaRankingDistribution.METHOD_MIXEDSENIOR), (int) getConfigLong(plasmaSwitchboardConstants.RANKING_DIST_1_METHOD, 30), getConfig(plasmaSwitchboardConstants.RANKING_DIST_1_TARGET, "kaskelix.de:8080,yacy.dyndns.org:8000")); // init nameCacheNoCachingList serverDomains.setNoCachingPatterns(getConfig(plasmaSwitchboardConstants.HTTPC_NAME_CACHE_CACHING_PATTERNS_NO,"")); // generate snippets cache log.logConfig("Initializing Snippet Cache"); plasmaSnippetCache.init(parser, log, this); // init the wiki wikiParser = new wikiCode(this.peers.mySeed().getClusterAddress()); // initializing the resourceObserver serverInstantBusyThread.oneTimeJob(ResourceObserver.class, "initThread", ResourceObserver.log, 0); // initializing the stackCrawlThread this.crawlStacker = new CrawlStacker( this.crawlQueues, this.crawler, this.indexSegment, this.peers, "local.any".indexOf(getConfig("network.unit.domain", "global")) >= 0, "global.any".indexOf(getConfig("network.unit.domain", "global")) >= 0); // initializing dht chunk generation this.dhtMaxReferenceCount = (int) getConfigLong(plasmaSwitchboardConstants.INDEX_DIST_CHUNK_SIZE_START, 50); // init robinson cluster // before we do that, we wait some time until the seed list is loaded. //while (((System.currentTimeMillis() - startedSeedListAquisition) < 8000) && (this.webIndex.seedDB.sizeConnected() == 0)) try {Thread.sleep(1000);} catch (final InterruptedException e) {} //try {Thread.sleep(1000);} catch (final InterruptedException e) {} this.clusterhashes = this.peers.clusterHashes(getConfig("cluster.peers.yacydomain", "")); // deploy blocking threads int indexerThreads = Math.max(1, serverProcessor.useCPU / 2); indexingStorageProcessor = new serverProcessor( "storeDocumentIndex", "This is the sequencing step of the indexing queue: no concurrency is wanted here, because the access of the indexer works better if it is not concurrent. Files are written as streams, councurrency would destroy IO performance. In this process the words are written to the RWI cache, which flushes if it is full.", new String[]{"RWI/Cache/Collections"}, this, "storeDocumentIndex", serverProcessor.useCPU + 40, null, indexerThreads); indexingAnalysisProcessor = new serverProcessor( "webStructureAnalysis", "This just stores the link structure of the document into a web structure database.", new String[]{"storeDocumentIndex"}, this, "webStructureAnalysis", serverProcessor.useCPU + 20, indexingStorageProcessor, serverProcessor.useCPU + 1); indexingCondensementProcessor = new serverProcessor( "condenseDocument", "This does a structural analysis of plain texts: markup of headlines, slicing into phrases (i.e. sentences), markup with position, counting of words, calculation of term frequency.", new String[]{"webStructureAnalysis"}, this, "condenseDocument", serverProcessor.useCPU + 10, indexingAnalysisProcessor, serverProcessor.useCPU + 1); indexingDocumentProcessor = new serverProcessor( "parseDocument", "This does the parsing of the newly loaded documents from the web. The result is not only a plain text document, but also a list of URLs that are embedded into the document. The urls are handed over to the CrawlStacker. This process has two child process queues!", new String[]{"condenseDocument", "CrawlStacker"}, this, "parseDocument", 2 * serverProcessor.useCPU + 1, indexingCondensementProcessor, 2 * serverProcessor.useCPU + 1); // deploy busy threads log.logConfig("Starting Threads"); MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq deployThread(plasmaSwitchboardConstants.CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null, new serverInstantBusyThread(this, plasmaSwitchboardConstants.CLEANUP_METHOD_START, plasmaSwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CLEANUP_METHOD_FREEMEM), 600000); // all 5 Minutes, wait 10 minutes until first run deployThread(plasmaSwitchboardConstants.INDEXER, "Indexing", "thread that either initiates a parsing/indexing queue, distributes the index into the DHT, stores parsed documents", "/IndexCreateIndexingQueue_p.html", new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEXER_METHOD_START, plasmaSwitchboardConstants.INDEXER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEXER_METHOD_FREEMEM), 10000); deployThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null, new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000); deployThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, "Remote Crawl URL Loader", "thread that loads remote crawl lists from other peers", "", new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM), 30000); // error here? deployThread(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html", new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000); deployThread(plasmaSwitchboardConstants.SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null, new serverInstantBusyThread(yc, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_START, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM), 180000); deployThread(plasmaSwitchboardConstants.PEER_PING, "YaCy Core", "this is the p2p-control and peer-ping task", null, new serverInstantBusyThread(yc, plasmaSwitchboardConstants.PEER_PING_METHOD_START, plasmaSwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, plasmaSwitchboardConstants.PEER_PING_METHOD_FREEMEM), 2000); deployThread(plasmaSwitchboardConstants.INDEX_DIST, "DHT Distribution", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null, new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEX_DIST_METHOD_START, plasmaSwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEX_DIST_METHOD_FREEMEM), 5000, Long.parseLong(getConfig(plasmaSwitchboardConstants.INDEX_DIST_IDLESLEEP , "5000")), Long.parseLong(getConfig(plasmaSwitchboardConstants.INDEX_DIST_BUSYSLEEP , "0")), Long.parseLong(getConfig(plasmaSwitchboardConstants.INDEX_DIST_MEMPREREQ , "1000000"))); // test routine for snippet fetch //Set query = new HashSet(); //query.add(CrawlSwitchboardEntry.word2hash("Weitergabe")); //query.add(CrawlSwitchboardEntry.word2hash("Zahl")); //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/mobil/newsticker/meldung/mail/54980"), query, true); //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/security/news/foren/go.shtml?read=1&msg_id=7301419&forum_id=72721"), query, true); //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/kiosk/archiv/ct/2003/4/20"), query, true, 260); this.dbImportManager = new ImporterManager(); this.trail = new ArrayList(); log.logConfig("Finished Switchboard Initialization"); } public void overwriteNetworkDefinition() { // load network configuration into settings String networkUnitDefinition = getConfig("network.unit.definition", "defaults/yacy.network.freeworld.unit"); final String networkGroupDefinition = getConfig("network.group.definition", "yacy.network.group"); // patch old values if (networkUnitDefinition.equals("yacy.network.unit")) { networkUnitDefinition = "defaults/yacy.network.freeworld.unit"; setConfig("network.unit.definition", networkUnitDefinition); } // remove old release and bootstrap locations Iterator ki = configKeys(); ArrayList d = new ArrayList(); String k; while (ki.hasNext()) { k = ki.next(); if (k.startsWith("network.unit.update.location")) d.add(k); if (k.startsWith("network.unit.bootstrap")) d.add(k); } for (String s:d) this.removeConfig(s); // must be removed afterwards othervise a ki.remove() would not remove the property on file // include additional network definition properties into our settings // note that these properties cannot be set in the application because they are // _always_ overwritten each time with the default values. This is done so on purpose. // the network definition should be made either consistent for all peers, // or independently using a bootstrap URL Map initProps; if (networkUnitDefinition.startsWith("http://")) { try { setConfig(plasmaSwitchboard.loadHashMap(new yacyURL(networkUnitDefinition, null))); } catch (final MalformedURLException e) { } } else { final File networkUnitDefinitionFile = (networkUnitDefinition.startsWith("/")) ? new File(networkUnitDefinition) : new File(getRootPath(), networkUnitDefinition); if (networkUnitDefinitionFile.exists()) { initProps = FileUtils.loadMap(networkUnitDefinitionFile); setConfig(initProps); } } if (networkGroupDefinition.startsWith("http://")) { try { setConfig(plasmaSwitchboard.loadHashMap(new yacyURL(networkGroupDefinition, null))); } catch (final MalformedURLException e) { } } else { final File networkGroupDefinitionFile = new File(getRootPath(), networkGroupDefinition); if (networkGroupDefinitionFile.exists()) { initProps = FileUtils.loadMap(networkGroupDefinitionFile); setConfig(initProps); } } // set release locations int i = 0; CryptoLib cryptoLib; try { cryptoLib = new CryptoLib(); while (true) { String location = getConfig("network.unit.update.location" + i, ""); if (location.length() == 0) break; yacyURL locationURL; try { // try to parse url locationURL = new yacyURL(location, null); } catch (final MalformedURLException e) { break; } PublicKey publicKey = null; // get public key if it's in config try { String publicKeyString = getConfig("network.unit.update.location" + i + ".key", null); if(publicKeyString != null) { byte[] publicKeyBytes = Base64Order.standardCoder.decode(publicKeyString.trim(), "decode public Key"); publicKey = cryptoLib.getPublicKeyFromBytes(publicKeyBytes); } } catch (InvalidKeySpecException e) { e.printStackTrace(); } yacyUpdateLocation updateLocation = new yacyUpdateLocation(locationURL, publicKey); yacyRelease.latestReleaseLocations.add(updateLocation); i++; } } catch (NoSuchAlgorithmException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } // initiate url license object licensedURLs = new URLLicense(8); // set white/blacklists this.networkWhitelist = serverDomains.makePatterns(getConfig(plasmaSwitchboardConstants.NETWORK_WHITELIST, "")); this.networkBlacklist = serverDomains.makePatterns(getConfig(plasmaSwitchboardConstants.NETWORK_BLACKLIST, "")); /* // in intranet and portal network set robinson mode if (networkUnitDefinition.equals("defaults/yacy.network.webportal.unit") || networkUnitDefinition.equals("defaults/yacy.network.intranet.unit")) { // switch to robinson mode setConfig("crawlResponse", "false"); setConfig(plasmaSwitchboardConstants.INDEX_DIST_ALLOW, false); setConfig(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, false); } // in freeworld network set full p2p mode if (networkUnitDefinition.equals("defaults/yacy.network.freeworld.unit")) { // switch to robinson mode setConfig("crawlResponse", "true"); setConfig(plasmaSwitchboardConstants.INDEX_DIST_ALLOW, true); setConfig(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, true); } */ } public void switchNetwork(final String networkDefinition) { // pause crawls final boolean lcp = crawlJobIsPaused(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); if (!lcp) pauseCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); final boolean rcp = crawlJobIsPaused(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); if (!rcp) pauseCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); // trigger online caution proxyLastAccess = System.currentTimeMillis() + 10000; // at least 10 seconds online caution to prevent unnecessary action on database meanwhile // clean search events which have cached relations to the old index plasmaSearchEvent.cleanupEvents(true); // switch the networks synchronized (this) { // shut down synchronized (this.indexSegment) { this.indexSegment.close(); } this.crawlStacker.announceClose(); this.crawlStacker.close(); // start up setConfig("network.unit.definition", networkDefinition); overwriteNetworkDefinition(); final File indexPrimaryPath = getConfigPath(plasmaSwitchboardConstants.INDEX_PRIMARY_PATH, plasmaSwitchboardConstants.INDEX_PATH_DEFAULT); final int wordCacheMaxCount = (int) getConfigLong(plasmaSwitchboardConstants.WORDCACHE_MAX_COUNT, 20000); final long fileSizeMax = (serverSystem.isWindows) ? sb.getConfigLong("filesize.max.win", (long) Integer.MAX_VALUE) : sb.getConfigLong("filesize.max.other", (long) Integer.MAX_VALUE); final int redundancy = (int) sb.getConfigLong("network.unit.dhtredundancy.senior", 1); final int partitionExponent = (int) sb.getConfigLong("network.unit.dht.partitionExponent", 0); try { String networkName = getConfig(plasmaSwitchboardConstants.NETWORK_NAME, ""); final File networkRoot = new File(new File(indexPrimaryPath, networkName), "NETWORK"); networkRoot.mkdirs(); final File mySeedFile = new File(networkRoot, yacySeedDB.DBFILE_OWN_SEED); peers = new yacySeedDB( networkRoot, "seed.new.heap", "seed.old.heap", "seed.pot.heap", mySeedFile, redundancy, partitionExponent); indexSegment = new Segment( log, new File(new File(indexPrimaryPath, networkName), "TEXT"), wordCacheMaxCount, fileSizeMax); crawler = new CrawlSwitchboard( peers, networkName, log, new File(new File(indexPrimaryPath, networkName), "QUEUES")); } catch (IOException e) { e.printStackTrace(); this.indexSegment = null; } // we need a new stacker, because this uses network-specific attributes to sort out urls (local, global) this.crawlStacker = new CrawlStacker( this.crawlQueues, this.crawler, this.indexSegment, this.peers, "local.any".indexOf(getConfig("network.unit.domain", "global")) >= 0, "global.any".indexOf(getConfig("network.unit.domain", "global")) >= 0); } // start up crawl jobs continueCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); continueCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); this.log.logInfo("switched network to " + networkDefinition); // check status of account configuration: when local url crawling is allowed, it is not allowed // that an automatic authorization of localhost is done, because in this case crawls from local // addresses are blocked to prevent attack szenarios where remote pages contain links to localhost // addresses that can steer a YaCy peer if ((crawlStacker.acceptLocalURLs()) && (getConfigBool("adminAccountForLocalhost", false))) { setConfig("adminAccountForLocalhost", false); if (getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").startsWith("0000")) { // the password was set automatically with a random value. // We must remove that here to prevent that a user cannot log in any more setConfig(httpd.ADMIN_ACCOUNT_B64MD5, ""); // after this a message must be generated to alert the user to set a new password log.logInfo("RANDOM PASSWORD REMOVED! User must set a new password"); } } // set the network-specific remote crawl ppm setRemotecrawlPPM(Math.max(1, (int) getConfigLong("network.unit.remotecrawl.speed", 60))); } public void setRemotecrawlPPM(int ppm) { setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, 60000 / ppm); setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP, Math.max(10000, 180000 / ppm)); setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_BUSYSLEEP, Math.max(15000, 1800000 / ppm)); setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_IDLESLEEP, Math.max(30000, 3600000 / ppm)); } public void initMessages() throws IOException { this.log.logConfig("Starting Message Board"); final File messageDbFileOld = new File(workPath, "message.db"); final File messageDbFile = new File(workPath, "message.heap"); this.messageDB = new messageBoard(messageDbFileOld, messageDbFile); this.log.logConfig("Loaded Message Board DB from file " + messageDbFile.getName() + ", " + this.messageDB.size() + " entries" + ", " + ppRamString(messageDbFile.length()/1024)); } public void initWiki() throws IOException { this.log.logConfig("Starting Wiki Board"); final File wikiDbFileOld = new File(workPath, "wiki.db"); final File wikiDbFile = new File(workPath, "wiki.heap"); this.wikiDB = new wikiBoard(wikiDbFileOld, wikiDbFile, new File(workPath, "wiki-bkp.db"), new File(workPath, "wiki-bkp.heap")); this.log.logConfig("Loaded Wiki Board DB from file " + wikiDbFile.getName() + ", " + this.wikiDB.size() + " entries" + ", " + ppRamString(wikiDbFile.length()/1024)); } public void initBlog() throws IOException { this.log.logConfig("Starting Blog"); final File blogDbFileOld = new File(workPath, "blog.db"); final File blogDbFile = new File(workPath, "blog.heap"); this.blogDB = new blogBoard(blogDbFileOld, blogDbFile); this.log.logConfig("Loaded Blog DB from file " + blogDbFile.getName() + ", " + this.blogDB.size() + " entries" + ", " + ppRamString(blogDbFile.length()/1024)); final File blogCommentDbFileOld = new File(workPath, "blogComment.db"); final File blogCommentDbFile = new File(workPath, "blogComment.heap"); this.blogCommentDB = new blogBoardComments(blogCommentDbFileOld, blogCommentDbFile); this.log.logConfig("Loaded Blog-Comment DB from file " + blogCommentDbFile.getName() + ", " + this.blogCommentDB.size() + " entries" + ", " + ppRamString(blogCommentDbFile.length()/1024)); } public void initBookmarks() throws IOException{ this.log.logConfig("Loading Bookmarks DB"); final File bookmarksFileOld = new File(workPath, "bookmarks.db"); final File tagsFileOld = new File(workPath, "bookmarkTags.db"); final File datesFileOld = new File(workPath, "bookmarkDates.db"); final File bookmarksFile = new File(workPath, "bookmarks.heap"); final File tagsFile = new File(workPath, "bookmarkTags.heap"); final File datesFile = new File(workPath, "bookmarkDates.heap"); this.bookmarksDB = new bookmarksDB( bookmarksFileOld, bookmarksFile, tagsFileOld, tagsFile, datesFileOld, datesFile); this.log.logConfig("Loaded Bookmarks DB from files "+ bookmarksFile.getName()+ ", "+tagsFile.getName()); this.log.logConfig(this.bookmarksDB.tagsSize()+" Tag, "+this.bookmarksDB.bookmarksSize()+" Bookmarks"); } public static plasmaSwitchboard getSwitchboard(){ return sb; } public boolean isRobinsonMode() { // we are in robinson mode, if we do not exchange index by dht distribution // we need to take care that search requests and remote indexing requests go only // to the peers in the same cluster, if we run a robinson cluster. return !getConfigBool(plasmaSwitchboardConstants.INDEX_DIST_ALLOW, false) && !getConfigBool(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, false); } public boolean isPublicRobinson() { // robinson peers may be member of robinson clusters, which can be public or private // this does not check the robinson attribute, only the specific subtype of the cluster final String clustermode = getConfig(plasmaSwitchboardConstants.CLUSTER_MODE, plasmaSwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); return (clustermode.equals(plasmaSwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER)) || (clustermode.equals(plasmaSwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER)); } public boolean isInMyCluster(final String peer) { // check if the given peer is in the own network, if this is a robinson cluster // depending on the robinson cluster type, the peer String may be a peerhash (b64-hash) // or a ip:port String or simply a ip String // if this robinson mode does not define a cluster membership, false is returned if (peer == null) return false; if (!isRobinsonMode()) return false; final String clustermode = getConfig(plasmaSwitchboardConstants.CLUSTER_MODE, plasmaSwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); if (clustermode.equals(plasmaSwitchboardConstants.CLUSTER_MODE_PRIVATE_CLUSTER)) { // check if we got the request from a peer in the private cluster final String network = getConfig(plasmaSwitchboardConstants.CLUSTER_PEERS_IPPORT, ""); return network.indexOf(peer) >= 0; } else if (clustermode.equals(plasmaSwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER)) { // check if we got the request from a peer in the public cluster return this.clusterhashes.containsKey(peer.getBytes()); } else { return false; } } public boolean isInMyCluster(final yacySeed seed) { // check if the given peer is in the own network, if this is a robinson cluster // if this robinson mode does not define a cluster membership, false is returned if (seed == null) return false; if (!isRobinsonMode()) return false; final String clustermode = getConfig(plasmaSwitchboardConstants.CLUSTER_MODE, plasmaSwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); if (clustermode.equals(plasmaSwitchboardConstants.CLUSTER_MODE_PRIVATE_CLUSTER)) { // check if we got the request from a peer in the private cluster final String network = getConfig(plasmaSwitchboardConstants.CLUSTER_PEERS_IPPORT, ""); return network.indexOf(seed.getPublicAddress()) >= 0; } else if (clustermode.equals(plasmaSwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER)) { // check if we got the request from a peer in the public cluster return this.clusterhashes.containsKey(seed.hash.getBytes()); } else { return false; } } public String urlExists(final String hash) { // tests if hash occurrs in any database // if it exists, the name of the database is returned, // if it not exists, null is returned if (indexSegment.urlMetadata().exists(hash)) return "loaded"; return this.crawlQueues.urlExists(hash); } public void urlRemove(final String hash) { indexSegment.urlMetadata().remove(hash); crawlResults.remove(hash); crawlQueues.urlRemove(hash); } public yacyURL getURL(final String urlhash) { if (urlhash == null) return null; if (urlhash.length() == 0) return null; final yacyURL ne = crawlQueues.getURL(urlhash); if (ne != null) return ne; final URLMetadataRow le = indexSegment.urlMetadata().load(urlhash, null, 0); if (le != null) return le.metadata().url(); return null; } public plasmaSearchRankingProfile getRanking() { return (getConfig("rankingProfile", "").length() == 0) ? new plasmaSearchRankingProfile(plasmaSearchQuery.CONTENTDOM_TEXT) : new plasmaSearchRankingProfile("", crypt.simpleDecode(sb.getConfig("rankingProfile", ""), null)); } public boolean onlineCaution() { return (System.currentTimeMillis() - this.proxyLastAccess < Integer.parseInt(getConfig(plasmaSwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, "30000"))) || (System.currentTimeMillis() - this.localSearchLastAccess < Integer.parseInt(getConfig(plasmaSwitchboardConstants.LOCALSEACH_ONLINE_CAUTION_DELAY, "30000"))) || (System.currentTimeMillis() - this.remoteSearchLastAccess < Integer.parseInt(getConfig(plasmaSwitchboardConstants.REMOTESEARCH_ONLINE_CAUTION_DELAY, "30000"))); } private static String ppRamString(long bytes) { if (bytes < 1024) return bytes + " KByte"; bytes = bytes / 1024; if (bytes < 1024) return bytes + " MByte"; bytes = bytes / 1024; if (bytes < 1024) return bytes + " GByte"; return (bytes / 1024) + "TByte"; } /** * {@link CrawlProfile Crawl Profiles} are saved independantly from the queues themselves * and therefore have to be cleaned up from time to time. This method only performs the clean-up * if - and only if - the {@link IndexingStack switchboard}, * {@link ProtocolLoader loader} and {@link plasmaCrawlNURL local crawl} queues are all empty. *

* Then it iterates through all existing {@link CrawlProfile crawl profiles} and removes * all profiles which are not hardcoded. *

*

* If this method encounters DB-failures, the profile DB will be resetted and * true will be returned *

* @see #CRAWL_PROFILE_PROXY hardcoded * @see #CRAWL_PROFILE_REMOTE hardcoded * @see #CRAWL_PROFILE_SNIPPET_TEXT hardcoded * @see #CRAWL_PROFILE_SNIPPET_MEDIA hardcoded * @return whether this method has done something or not (i.e. because the queues have been filled * or there are no profiles left to clean up) * @throws InterruptedException if the current thread has been interrupted, i.e. by the * shutdown procedure */ public boolean cleanProfiles() throws InterruptedException { if ((crawlQueues.size() > 0) || (crawlStacker != null && crawlStacker.size() > 0) || (crawlQueues.noticeURL.notEmpty())) return false; return this.crawler.cleanProfiles(); } public boolean htEntryStoreProcess(final Document entry) { if (entry == null) return false; /* ========================================================================= * PARSER SUPPORT * * Testing if the content type is supported by the available parsers * ========================================================================= */ final boolean isSupportedContent = plasmaParser.supportedContent(entry.url(),entry.getMimeType()); if (log.isFinest()) log.logFinest("STORE "+ entry.url() +" content of type "+ entry.getMimeType() +" is supported: "+ isSupportedContent); /* ========================================================================= * INDEX CONTROL HEADER * * With the X-YACY-Index-Control header set to "no-index" a client could disallow * yacy to index the response returned as answer to a request * ========================================================================= */ boolean doIndexing = true; if (entry.requestProhibitsIndexing()) { doIndexing = false; if (this.log.isFine()) this.log.logFine("Crawling of " + entry.url() + " prohibited by request."); } /* ========================================================================= * LOCAL IP ADDRESS CHECK * * check if ip is local ip address // TODO: remove this procotol specific code here * ========================================================================= */ final String urlRejectReason = crawlStacker.urlInAcceptedDomain(entry.url()); if (urlRejectReason != null) { if (this.log.isFine()) this.log.logFine("Rejected URL '" + entry.url() + "': " + urlRejectReason); doIndexing = false; } synchronized (crawler.queuePreStack) { /* ========================================================================= * STORING DATA * * Now we store the response header and response content if * a) the user has configured to use the htcache or * b) the content should be indexed * ========================================================================= */ if (((entry.profile() != null) && (entry.profile().storeHTCache())) || (doIndexing && isSupportedContent)) { // store response header /* if (entry.writeResourceInfo()) { this.log.logInfo("WROTE HEADER for " + entry.cacheFile()); } */ // work off unwritten files if (entry.cacheArray() != null) { final String error = (entry.initiator() == null) ? entry.shallStoreCacheForProxy() : null; if (error == null) { plasmaHTCache.storeFile(entry.url(), entry.cacheArray()); if (this.log.isFine()) this.log.logFine("WROTE FILE (" + entry.cacheArray().length + " bytes) for " + entry.url()); } else { if (this.log.isWarning()) this.log.logWarning("WRITE OF FILE " + entry.url() + " FORBIDDEN: " + error); } //} else { //this.log.logFine("EXISTING FILE (" + entry.cacheFile.length() + " bytes) for " + entry.cacheFile); } } /* ========================================================================= * INDEXING * ========================================================================= */ if (doIndexing && isSupportedContent) { // enqueue for further crawling enQueue(this.crawler.queuePreStack.newEntry( entry.url(), (entry.referrerURL() == null) ? null : entry.referrerURL().hash(), entry.ifModifiedSince(), entry.requestWithCookie(), entry.initiator(), entry.depth(), entry.profile().handle(), entry.name() )); } else { if (!entry.profile().storeHTCache()) { try { plasmaHTCache.deleteFromCache(entry.url()); } catch (IOException e) { e.printStackTrace(); } } } } return true; } public void close() { log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:"); serverProfiling.stopSystemProfiling(); terminateAllThreads(true); log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing"); // closing all still running db importer jobs dhtDispatcher.close(); indexingDocumentProcessor.announceShutdown(); indexingDocumentProcessor.awaitShutdown(12000); crawlStacker.announceClose(); indexingCondensementProcessor.announceShutdown(); indexingAnalysisProcessor.announceShutdown(); indexingStorageProcessor.announceShutdown(); indexingCondensementProcessor.awaitShutdown(12000); indexingAnalysisProcessor.awaitShutdown(12000); indexingStorageProcessor.awaitShutdown(12000); crawlStacker.close(); this.dbImportManager.close(); httpClient.closeAllConnections(); wikiDB.close(); blogDB.close(); blogCommentDB.close(); userDB.close(); bookmarksDB.close(); messageDB.close(); robots.close(); parser.close(); webStructure.flushCitationReference("crg"); webStructure.close(); crawlQueues.close(); crawler.close(); log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager (stand by...)"); indexSegment.close(); peers.close(); plasmaHTCache.close(); UPnP.deletePortMapping(); Tray.removeTray(); log.logConfig("SWITCHBOARD SHUTDOWN TERMINATED"); } public int queueSize() { return crawler.queuePreStack.size(); } public void enQueue(final IndexingStack.QueueEntry job) { assert job != null; try { crawler.queuePreStack.push(job); } catch (final IOException e) { log.logSevere("IOError in plasmaSwitchboard.enQueue: " + e.getMessage(), e); } } public void deQueueFreeMem() { // empty some caches indexSegment.urlMetadata().clearCache(); plasmaSearchEvent.cleanupEvents(true); } public IndexingStack.QueueEntry deQueue() { // getting the next entry from the indexing queue IndexingStack.QueueEntry nextentry = null; synchronized (crawler.queuePreStack) { // do one processing step if (this.log.isFine()) log.logFine("DEQUEUE: sbQueueSize=" + crawler.queuePreStack.size() + ", coreStackSize=" + crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE) + ", limitStackSize=" + crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT) + ", overhangStackSize=" + crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_OVERHANG) + ", remoteStackSize=" + crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE)); try { final int sizeBefore = crawler.queuePreStack.size(); if (sizeBefore == 0) return null; nextentry = crawler.queuePreStack.pop(); if (nextentry == null) { log.logWarning("deQueue: null entry on queue stack."); if (crawler.queuePreStack.size() == sizeBefore) { // this is a severe problem: because this time a null is returned, it means that this status will last forever // to re-enable use of the sbQueue, it must be emptied completely log.logSevere("deQueue: does not shrink after pop() == null. Emergency reset of " + crawler.queuePreStack.file()); crawler.queuePreStack.clear(); } return null; } } catch (final IOException e) { log.logSevere("IOError in plasmaSwitchboard.deQueue: " + e.getMessage(), e); return null; } return nextentry; } } public boolean processSurrogate(String s) { File surrogateFile = new File(this.surrogatesInPath, s); File outfile = new File(this.surrogatesOutPath, s); if (!surrogateFile.exists() || !surrogateFile.canWrite() || !surrogateFile.canRead()) return false; if (outfile.exists()) return false; boolean moved = false; try { SurrogateReader reader = new SurrogateReader(new BufferedInputStream(new FileInputStream(surrogateFile)), 3); Thread readerThread = new Thread(reader, "Surrogate-Reader " + surrogateFile.getAbsolutePath()); readerThread.start(); DCEntry surrogate; QueueEntry queueentry; while ((surrogate = reader.take()) != DCEntry.poison) { // check if url is in accepted domain final String urlRejectReason = crawlStacker.urlInAcceptedDomain(surrogate.url()); if (urlRejectReason != null) { if (this.log.isFine()) this.log.logInfo("Rejected URL '" + surrogate.url() + "': " + urlRejectReason); continue; } // create a queue entry plasmaParserDocument document = surrogate.document(); queueentry = this.crawler.queuePreStack.newEntry(surrogate.url(), null, null, false, null, 0, this.crawler.defaultSurrogateProfile.handle(), null); /* * public QueueEntry newEntry(final yacyURL url, final String referrer, final Date ifModifiedSince, final boolean requestWithCookie, final String initiator, final int depth, final String profilehandle, final String anchorName) */ indexingQueueEntry queueEntry = new indexingQueueEntry(queueentry, document, null); // place the queue entry into the concurrent process of the condenser (document analysis) try { indexingCondensementProcessor.enQueue(queueEntry); } catch (InterruptedException e) { e.printStackTrace(); break; } } } catch (IOException e) { e.printStackTrace(); } finally { moved = surrogateFile.renameTo(outfile); } return moved; } public boolean deQueueProcess() { try { // work off fresh entries from the proxy or from the crawler if (onlineCaution()) { if (this.log.isFine()) log.logFine("deQueue: online caution, omitting resource stack processing"); return false; } boolean doneSomething = false; // check for interruption checkInterruption(); // check surrogates String[] surrogatelist = this.surrogatesInPath.list(); if (surrogatelist.length > 0) { // look if the is any xml inside for (int i = 0; i < surrogatelist.length; i++) { if (surrogatelist[i].endsWith(".xml")) { // read the surrogate file and store entry in index if (processSurrogate(surrogatelist[i])) return true; } } } // getting the next entry from the indexing queue if (crawler.queuePreStack.size() == 0) { //log.logFine("deQueue: nothing to do, queue is emtpy"); return doneSomething; // nothing to do } // if we were interrupted we should return now if (Thread.currentThread().isInterrupted()) { if (this.log.isFine()) log.logFine("deQueue: thread was interrupted"); return false; } // get next queue entry and start a queue processing final IndexingStack.QueueEntry queueEntry = deQueue(); if (queueEntry == null) return true; if (queueEntry.profile() == null) { queueEntry.close(); return true; } // check if the document should be indexed String noIndexReason = "unspecified indexing error"; if (queueEntry.processCase() == plasmaSwitchboardConstants.PROCESSCASE_4_PROXY_LOAD) { // proxy-load noIndexReason = queueEntry.shallIndexCacheForProxy(); } else { // normal crawling noIndexReason = queueEntry.shallIndexCacheForCrawler(); } if (noIndexReason != null) { // this document should not be indexed. log cause and close queue final yacyURL referrerURL = queueEntry.referrerURL(this.indexSegment.urlMetadata()); log.logFine("Not indexed any word in URL " + queueEntry.url() + "; cause: " + noIndexReason); addURLtoErrorDB(queueEntry.url(), (referrerURL == null) ? "" : referrerURL.hash(), queueEntry.initiator(), queueEntry.anchorName(), noIndexReason); // finish this entry return true; } // put document into the concurrent processing queue crawler.queuePreStack.enQueueToActive(queueEntry); // check for interruption checkInterruption(); // enqueue to indexing queue this.indexingDocumentProcessor.enQueue(new indexingQueueEntry(queueEntry, null, null)); return true; } catch (final InterruptedException e) { log.logInfo("DEQUEUE: Shutdown detected."); return false; } } public static class indexingQueueEntry extends serverProcessorJob { public IndexingStack.QueueEntry queueEntry; public plasmaParserDocument document; public Condenser condenser; public indexingQueueEntry( final IndexingStack.QueueEntry queueEntry, final plasmaParserDocument document, final Condenser condenser) { super(); this.queueEntry = queueEntry; this.document = document; this.condenser = condenser; } } public int cleanupJobSize() { int c = 0; if ((crawlQueues.delegatedURL.stackSize() > 1000)) c++; if ((crawlQueues.errorURL.stackSize() > 1000)) c++; for (int i = 1; i <= 6; i++) { if (crawlResults.getStackSize(i) > 1000) c++; } return c; } public boolean cleanupJob() { try { boolean hasDoneSomething = false; // clear caches if necessary if (!MemoryControl.request(8000000L, false)) { indexSegment.urlMetadata().clearCache(); plasmaSearchEvent.cleanupEvents(true); } // set a random password if no password is configured if (!crawlStacker.acceptLocalURLs() && getConfigBool("adminAccountForLocalhost", false) && getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() == 0) { // make a 'random' password setConfig(httpd.ADMIN_ACCOUNT_B64MD5, "0000" + Digest.encodeMD5Hex(System.getProperties().toString() + System.currentTimeMillis())); setConfig("adminAccount", ""); } // refresh recrawl dates try{ Iterator it = crawler.profilesActiveCrawls.profiles(true); entry selentry; while (it.hasNext()) { selentry = it.next(); if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_PROXY)) crawler.profilesActiveCrawls.changeEntry(selentry, CrawlProfile.entry.RECRAWL_IF_OLDER, Long.toString(crawler.profilesActiveCrawls.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_PROXY_RECRAWL_CYCLE))); // if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_REMOTE)); if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_TEXT)) crawler.profilesActiveCrawls.changeEntry(selentry, CrawlProfile.entry.RECRAWL_IF_OLDER, Long.toString(crawler.profilesActiveCrawls.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_TEXT_RECRAWL_CYCLE))); if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT)) crawler.profilesActiveCrawls.changeEntry(selentry, CrawlProfile.entry.RECRAWL_IF_OLDER, Long.toString(crawler.profilesActiveCrawls.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT_RECRAWL_CYCLE))); if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA)) crawler.profilesActiveCrawls.changeEntry(selentry, CrawlProfile.entry.RECRAWL_IF_OLDER, Long.toString(crawler.profilesActiveCrawls.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE))); if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA)) crawler.profilesActiveCrawls.changeEntry(selentry, CrawlProfile.entry.RECRAWL_IF_OLDER, Long.toString(crawler.profilesActiveCrawls.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE))); if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_SURROGATE)) crawler.profilesActiveCrawls.changeEntry(selentry, CrawlProfile.entry.RECRAWL_IF_OLDER, Long.toString(crawler.profilesActiveCrawls.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE))); } } catch (final IOException e) {}; // close unused connections httpClient.cleanup(); // do transmission of CR-files checkInterruption(); int count = rankingOwnDistribution.size() / 100; if (count == 0) count = 1; if (count > 5) count = 5; if (rankingOn && !isRobinsonMode()) { rankingOwnDistribution.transferRanking(count); rankingOtherDistribution.transferRanking(1); } // clean up delegated stack checkInterruption(); if ((crawlQueues.delegatedURL.stackSize() > 1000)) { if (this.log.isFine()) log.logFine("Cleaning Delegated-URLs report stack, " + crawlQueues.delegatedURL.stackSize() + " entries on stack"); crawlQueues.delegatedURL.clearStack(); hasDoneSomething = true; } // clean up error stack checkInterruption(); if ((crawlQueues.errorURL.stackSize() > 1000)) { if (this.log.isFine()) log.logFine("Cleaning Error-URLs report stack, " + crawlQueues.errorURL.stackSize() + " entries on stack"); crawlQueues.errorURL.clearStack(); hasDoneSomething = true; } // clean up loadedURL stack for (int i = 1; i <= 6; i++) { checkInterruption(); if (crawlResults.getStackSize(i) > 1000) { if (this.log.isFine()) log.logFine("Cleaning Loaded-URLs report stack, " + crawlResults.getStackSize(i) + " entries on stack " + i); crawlResults.clearStack(i); hasDoneSomething = true; } } // clean up image stack ResultImages.clearQueues(); // clean up profiles checkInterruption(); if (cleanProfiles()) hasDoneSomething = true; // clean up news checkInterruption(); try { if (this.log.isFine()) log.logFine("Cleaning Incoming News, " + this.peers.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack"); if (this.peers.newsPool.automaticProcess(peers) > 0) hasDoneSomething = true; } catch (final IOException e) {} if (getConfigBool("cleanup.deletionProcessedNews", true)) { this.peers.newsPool.clear(yacyNewsPool.PROCESSED_DB); } if (getConfigBool("cleanup.deletionPublishedNews", true)) { this.peers.newsPool.clear(yacyNewsPool.PUBLISHED_DB); } // clean up seed-dbs if(getConfigBool("routing.deleteOldSeeds.permission",true)) { final long deleteOldSeedsTime = getConfigLong("routing.deleteOldSeeds.time",7)*24*3600000; Iterator e = this.peers.seedsSortedDisconnected(true,yacySeed.LASTSEEN); yacySeed seed = null; final ArrayList deleteQueue = new ArrayList(); checkInterruption(); //clean passive seeds while(e.hasNext()) { seed = e.next(); if(seed != null) { //list is sorted -> break when peers are too young to delete if(seed.getLastSeenUTC() > (System.currentTimeMillis()-deleteOldSeedsTime)) break; deleteQueue.add(seed.hash); } } for(int i=0;i break when peers are too young to delete if(seed.getLastSeenUTC() > (System.currentTimeMillis()-deleteOldSeedsTime)) break; deleteQueue.add(seed.hash); } } for (int i = 0; i < deleteQueue.size(); ++i) this.peers.removePotential(deleteQueue.get(i)); } // check if update is available and // if auto-update is activated perform an automatic installation and restart final yacyRelease updateVersion = yacyRelease.rulebasedUpdateInfo(false); if (updateVersion != null) { // there is a version that is more recent. Load it and re-start with it log.logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.getUrl()); final File downloaded = updateVersion.downloadRelease(); final boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev"); if (devenvironment) { log.logInfo("AUTO-UPDATE: omiting update because this is a development environment"); } else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) { log.logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found, is too small or signature is bad)"); } else { yacyRelease.deployRelease(downloaded); terminate(5000); log.logInfo("AUTO-UPDATE: deploy and restart initiated"); } } // initiate broadcast about peer startup to spread supporter url if (this.peers.newsPool.size(yacyNewsPool.OUTGOING_DB) == 0) { // read profile final Properties profile = new Properties(); FileInputStream fileIn = null; try { fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt")); profile.load(fileIn); } catch(final IOException e) { } finally { if (fileIn != null) try { fileIn.close(); } catch (final Exception e) {} } final String homepage = (String) profile.get("homepage"); if ((homepage != null) && (homepage.length() > 10)) { final Properties news = new Properties(); news.put("homepage", profile.get("homepage")); this.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news)); } } // update the cluster set this.clusterhashes = this.peers.clusterHashes(getConfig("cluster.peers.yacydomain", "")); // after all clean up is done, check the resource usage observer.resourceObserverJob(); return hasDoneSomething; } catch (final InterruptedException e) { this.log.logInfo("cleanupJob: Shutdown detected"); return false; } } /** * With this function the crawling process can be paused * @param jobType */ public void pauseCrawlJob(final String jobType) { final Object[] status = this.crawlJobsStatus.get(jobType); synchronized(status[plasmaSwitchboardConstants.CRAWLJOB_SYNC]) { status[plasmaSwitchboardConstants.CRAWLJOB_STATUS] = Boolean.TRUE; } setConfig(jobType + "_isPaused", "true"); } /** * Continue the previously paused crawling * @param jobType */ public void continueCrawlJob(final String jobType) { final Object[] status = this.crawlJobsStatus.get(jobType); synchronized(status[plasmaSwitchboardConstants.CRAWLJOB_SYNC]) { if (((Boolean)status[plasmaSwitchboardConstants.CRAWLJOB_STATUS]).booleanValue()) { status[plasmaSwitchboardConstants.CRAWLJOB_STATUS] = Boolean.FALSE; status[plasmaSwitchboardConstants.CRAWLJOB_SYNC].notifyAll(); } } setConfig(jobType + "_isPaused", "false"); } /** * @param jobType * @return true if crawling was paused or false otherwise */ public boolean crawlJobIsPaused(final String jobType) { final Object[] status = this.crawlJobsStatus.get(jobType); synchronized(status[plasmaSwitchboardConstants.CRAWLJOB_SYNC]) { return ((Boolean)status[plasmaSwitchboardConstants.CRAWLJOB_STATUS]).booleanValue(); } } public indexingQueueEntry parseDocument(final indexingQueueEntry in) { in.queueEntry.updateStatus(IndexingStack.QUEUE_STATE_PARSING); // debug if (log.isFinest()) log.logFinest("PARSE "+ in.queueEntry.toString()); plasmaParserDocument document = null; try { document = parseDocument(in.queueEntry); } catch (final InterruptedException e) { document = null; } catch (final Exception e) { document = null; } if (document == null) { in.queueEntry.close(); return null; } return new indexingQueueEntry(in.queueEntry, document, null); } private plasmaParserDocument parseDocument(final IndexingStack.QueueEntry entry) throws InterruptedException { plasmaParserDocument document = null; final int processCase = entry.processCase(); if (this.log.isFine()) log.logFine("processResourceStack processCase=" + processCase + ", depth=" + entry.depth() + ", maxDepth=" + ((entry.profile() == null) ? "null" : Integer.toString(entry.profile().depth())) + ", must-match=" + ((entry.profile() == null) ? "null" : entry.profile().mustMatchPattern().toString()) + ", must-not-match=" + ((entry.profile() == null) ? "null" : entry.profile().mustNotMatchPattern().toString()) + ", initiatorHash=" + entry.initiator() + //", responseHeader=" + ((entry.responseHeader() == null) ? "null" : entry.responseHeader().toString()) + ", url=" + entry.url()); // DEBUG // PARSE CONTENT final long parsingStartTime = System.currentTimeMillis(); try { // parse the document document = parser.parseSource(entry.url(), entry.getMimeType(), entry.getCharacterEncoding(), plasmaHTCache.getResourceContent(entry.url())); assert(document != null) : "Unexpected error. Parser returned null."; if (document == null) return null; } catch (final ParserException e) { this.log.logInfo("Unable to parse the resource '" + entry.url() + "'. " + e.getMessage()); addURLtoErrorDB(entry.url(), entry.referrerHash(), entry.initiator(), entry.anchorName(), e.getErrorCode()); if (document != null) { document.close(); document = null; } return null; } final long parsingEndTime = System.currentTimeMillis(); // get the document date final Date docDate = entry.getModificationDate(); // put anchors on crawl stack final long stackStartTime = System.currentTimeMillis(); if ( ((processCase == plasmaSwitchboardConstants.PROCESSCASE_4_PROXY_LOAD) || (processCase == plasmaSwitchboardConstants.PROCESSCASE_5_LOCAL_CRAWLING)) && ((entry.profile() == null) || (entry.depth() < entry.profile().depth())) ) { final Map hl = document.getHyperlinks(); final Iterator> i = hl.entrySet().iterator(); yacyURL nextUrl; Map.Entry nextEntry; while (i.hasNext()) { // check for interruption checkInterruption(); // fetching the next hyperlink nextEntry = i.next(); nextUrl = nextEntry.getKey(); String u = nextUrl.toNormalform(true, true); if (!(u.startsWith("http") || u.startsWith("ftp"))) continue; // enqueue the hyperlink into the pre-notice-url db crawlStacker.enqueueEntry(new CrawlEntry( entry.initiator(), nextUrl, entry.url().hash(), nextEntry.getValue(), null, docDate, entry.profile().handle(), entry.depth() + 1, 0, 0 )); } final long stackEndTime = System.currentTimeMillis(); if (log.isInfo()) log.logInfo("CRAWL: ADDED " + hl.size() + " LINKS FROM " + entry.url().toNormalform(false, true) + ", NEW CRAWL STACK SIZE IS " + crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE) + ", STACKING TIME = " + (stackEndTime-stackStartTime) + ", PARSING TIME = " + (parsingEndTime-parsingStartTime)); } return document; } public indexingQueueEntry condenseDocument(final indexingQueueEntry in) { in.queueEntry.updateStatus(IndexingStack.QUEUE_STATE_CONDENSING); // debug if (log.isFinest()) log.logFinest("CONDENSE "+ in.queueEntry.toString()); // strip out words and generate statistics if (this.log.isFine()) log.logFine("Condensing for '" + in.queueEntry.url().toNormalform(false, true) + "'"); try { Condenser condenser = new Condenser(in.document, in.queueEntry.profile().indexText(), in.queueEntry.profile().indexMedia()); // update image result list statistics // its good to do this concurrently here, because it needs a DNS lookup // to compute a URL hash which is necessary for a double-check final CrawlProfile.entry profile = in.queueEntry.profile(); ResultImages.registerImages(in.document, (profile == null) ? true : !profile.remoteIndexing()); return new indexingQueueEntry(in.queueEntry, in.document, condenser); } catch (final UnsupportedEncodingException e) { in.queueEntry.close(); return null; } } public indexingQueueEntry webStructureAnalysis(final indexingQueueEntry in) { in.queueEntry.updateStatus(IndexingStack.QUEUE_STATE_STRUCTUREANALYSIS); in.document.notifyWebStructure(webStructure, in.condenser, in.queueEntry.getModificationDate()); return in; } public void storeDocumentIndex(final indexingQueueEntry in) { in.queueEntry.updateStatus(IndexingStack.QUEUE_STATE_INDEXSTORAGE); storeDocumentIndex(in.queueEntry, in.document, in.condenser); in.queueEntry.updateStatus(IndexingStack.QUEUE_STATE_FINISHED); in.queueEntry.close(); } private void storeDocumentIndex(final IndexingStack.QueueEntry queueEntry, final plasmaParserDocument document, final Condenser condenser) { // CREATE INDEX final String dc_title = document.dc_title(); final yacyURL referrerURL = queueEntry.referrerURL(this.indexSegment.urlMetadata()); final int processCase = queueEntry.processCase(); // remove stopwords log.logInfo("Excluded " + condenser.excludeWords(stopwords) + " words in URL " + queueEntry.url()); // STORE URL TO LOADED-URL-DB URLMetadataRow newEntry = null; try { newEntry = indexSegment.storeDocument( queueEntry.url(), referrerURL, queueEntry.getModificationDate(), queueEntry.size(), document, condenser); RSSFeed.channels((queueEntry.initiator().equals(peers.mySeed().hash)) ? RSSFeed.LOCALINDEXING : RSSFeed.REMOTEINDEXING).addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url().toNormalform(true, false))); } catch (final IOException e) { if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase); addURLtoErrorDB(queueEntry.url(), referrerURL.hash(), queueEntry.initiator(), dc_title, "error storing url: " + e.getMessage()); return; } // update url result list statistics crawlResults.stack( newEntry, // loaded url db entry queueEntry.initiator(), // initiator peer hash this.peers.mySeed().hash, // executor peer hash processCase // process case ); // STORE WORD INDEX if ((!queueEntry.profile().indexText()) && (!queueEntry.profile().indexMedia())) { if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase); addURLtoErrorDB(queueEntry.url(), referrerURL.hash(), queueEntry.initiator(), dc_title, "unknown indexing process case" + processCase); return; } // increment number of indexed urls indexedPages++; // update profiling info if (System.currentTimeMillis() - lastPPMUpdate > 20000) { // we don't want to do this too often updateMySeed(); serverProfiling.update("ppm", Long.valueOf(currentPPM()), true); lastPPMUpdate = System.currentTimeMillis(); } serverProfiling.update("indexed", queueEntry.url().toNormalform(true, false), false); // if this was performed for a remote crawl request, notify requester final yacySeed initiatorPeer = queueEntry.initiatorPeer(); if ((processCase == plasmaSwitchboardConstants.PROCESSCASE_6_GLOBAL_CRAWLING) && (initiatorPeer != null)) { log.logInfo("Sending crawl receipt for '" + queueEntry.url().toNormalform(false, true) + "' to " + initiatorPeer.getName()); if (clusterhashes != null) initiatorPeer.setAlternativeAddress(clusterhashes.get(initiatorPeer.hash.getBytes())); // start a thread for receipt sending to avoid a blocking here new Thread(new receiptSending(initiatorPeer, newEntry), "sending receipt to " + initiatorPeer.hash).start(); } } public class receiptSending implements Runnable { yacySeed initiatorPeer; URLMetadataRow reference; public receiptSending(final yacySeed initiatorPeer, final URLMetadataRow reference) { this.initiatorPeer = initiatorPeer; this.reference = reference; } public void run() { yacyClient.crawlReceipt(peers.mySeed(), initiatorPeer, "crawl", "fill", "indexed", reference, ""); } } private static SimpleDateFormat DateFormat1 = new SimpleDateFormat("EEE, dd MMM yyyy"); public static String dateString(final Date date) { if (date == null) return ""; return DateFormat1.format(date); } // we need locale independent RFC-822 dates at some places private static SimpleDateFormat DateFormatter822 = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z", Locale.US); public static String dateString822(final Date date) { if (date == null) return ""; return DateFormatter822.format(date); } public serverObjects action(final String actionName, final serverObjects actionInput) { // perform an action. (not used) return null; } public String toString() { // it is possible to use this method in the cgi pages. // actually it is used there for testing purpose return "PROPS: " + super.toString() + "; QUEUE: " + crawler.queuePreStack.toString(); } // method for index deletion public int removeAllUrlReferences(final yacyURL url, final boolean fetchOnline) { return removeAllUrlReferences(url.hash(), fetchOnline); } public int removeAllUrlReferences(final String urlhash, final boolean fetchOnline) { // find all the words in a specific resource and remove the url reference from every word index // finally, delete the url entry if (urlhash == null) return 0; // determine the url string final URLMetadataRow entry = indexSegment.urlMetadata().load(urlhash, null, 0); if (entry == null) return 0; final URLMetadataRow.Components metadata = entry.metadata(); if (metadata.url() == null) return 0; InputStream resourceContent = null; try { // get the resource content Object[] resource = null; try { resource = plasmaSnippetCache.getResource(metadata.url(), fetchOnline, 10000, true, false); } catch (IOException e) { Log.logWarning("removeAllUrlReferences", "cannot load: " + e.getMessage()); } if (resource == null) { // delete just the url entry indexSegment.urlMetadata().remove(urlhash); return 0; } else { resourceContent = (InputStream) resource[0]; final Long resourceContentLength = (Long) resource[1]; // parse the resource final plasmaParserDocument document = plasmaSnippetCache.parseDocument(metadata.url(), resourceContentLength.longValue(), resourceContent); // get the word set Set words = null; try { words = new Condenser(document, true, true).words().keySet(); } catch (final UnsupportedEncodingException e) { e.printStackTrace(); } // delete all word references int count = 0; if (words != null) count = indexSegment.termIndex().remove(Word.words2hashes(words), urlhash); // finally delete the url entry itself indexSegment.urlMetadata().remove(urlhash); return count; } } catch (final ParserException e) { return 0; } catch (IOException e) { e.printStackTrace(); return 0; } finally { if (resourceContent != null) try { resourceContent.close(); } catch (final Exception e) {/* ignore this */} } } public int adminAuthenticated(final httpRequestHeader requestHeader) { // authorization for localhost, only if flag is set to grant localhost access as admin final String clientIP = (String) requestHeader.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, ""); final String refererHost = requestHeader.refererHost(); final boolean accessFromLocalhost = serverCore.isLocalhost(clientIP) && (refererHost.length() == 0 || serverCore.isLocalhost(refererHost)); if (getConfigBool("adminAccountForLocalhost", false) && accessFromLocalhost) return 3; // soft-authenticated for localhost // get the authorization string from the header final String authorization = ((String) requestHeader.get(httpRequestHeader.AUTHORIZATION, "xxxxxx")).trim().substring(6); // security check against too long authorization strings if (authorization.length() > 256) return 0; // authorization by encoded password, only for localhost access final String adminAccountBase64MD5 = getConfig(httpd.ADMIN_ACCOUNT_B64MD5, ""); if (accessFromLocalhost && (adminAccountBase64MD5.equals(authorization))) return 3; // soft-authenticated for localhost // authorization by hit in userDB if (userDB.hasAdminRight((String) requestHeader.get(httpRequestHeader.AUTHORIZATION, "xxxxxx"), ((String) requestHeader.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "")), requestHeader.getHeaderCookies())) return 4; //return, because 4=max // authorization with admin keyword in configuration return httpd.staticAdminAuthenticated(authorization, this); } public boolean verifyAuthentication(final httpRequestHeader header, final boolean strict) { // handle access rights switch (adminAuthenticated(header)) { case 0: // wrong password given try { Thread.sleep(3000); } catch (final InterruptedException e) { } // prevent brute-force return false; case 1: // no password given return false; case 2: // no password stored return !strict; case 3: // soft-authenticated for localhost only return true; case 4: // hard-authenticated, all ok return true; } return false; } public void setPerformance(int wantedPPM) { // we consider 3 cases here // wantedPPM <= 10: low performance // 10 < wantedPPM < 1000: custom performance // 1000 <= wantedPPM : maximum performance if (wantedPPM <= 10) wantedPPM = 10; if (wantedPPM >= 6000) wantedPPM = 6000; final int newBusySleep = 60000 / wantedPPM; // for wantedPPM = 10: 6000; for wantedPPM = 1000: 60 serverBusyThread thread; thread = getThread(plasmaSwitchboardConstants.INDEX_DIST); if (thread != null) { setConfig(plasmaSwitchboardConstants.INDEX_DIST_BUSYSLEEP , thread.setBusySleep(Math.max(2000, thread.setBusySleep(newBusySleep * 2)))); thread.setIdleSleep(30000); } thread = getThread(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); if (thread != null) { setConfig(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP , thread.setBusySleep(newBusySleep)); thread.setIdleSleep(2000); } thread = getThread(plasmaSwitchboardConstants.INDEXER); if (thread != null) { setConfig(plasmaSwitchboardConstants.INDEXER_BUSYSLEEP , thread.setBusySleep(newBusySleep / 8)); thread.setIdleSleep(2000); } } public static int accessFrequency(final HashMap> tracker, final String host) { // returns the access frequency in queries per hour for a given host and a specific tracker final long timeInterval = 1000 * 60 * 60; final TreeSet accessSet = tracker.get(host); if (accessSet == null) return 0; return accessSet.tailSet(Long.valueOf(System.currentTimeMillis() - timeInterval)).size(); } public String dhtShallTransfer() { if (this.peers == null) { return "no DHT distribution: seedDB == null"; } if (this.peers.mySeed() == null) { return "no DHT distribution: mySeed == null"; } if (this.peers.mySeed().isVirgin()) { return "no DHT distribution: status is virgin"; } if (this.peers.noDHTActivity()) { return "no DHT distribution: network too small"; } if (!this.getConfigBool("network.unit.dht", true)) { return "no DHT distribution: disabled by network.unit.dht"; } if (getConfig(plasmaSwitchboardConstants.INDEX_DIST_ALLOW, "false").equalsIgnoreCase("false")) { return "no DHT distribution: not enabled (ser setting)"; } if (indexSegment.urlMetadata().size() < 10) { return "no DHT distribution: loadedURL.size() = " + indexSegment.urlMetadata().size(); } if (indexSegment.termIndex().size() < 100) { return "no DHT distribution: not enough words - wordIndex.size() = " + indexSegment.termIndex().size(); } if ((getConfig(plasmaSwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, "false").equalsIgnoreCase("false")) && (crawlQueues.noticeURL.notEmptyLocal())) { return "no DHT distribution: crawl in progress: noticeURL.stackSize() = " + crawlQueues.noticeURL.size() + ", sbQueue.size() = " + crawler.queuePreStack.size(); } if ((getConfig(plasmaSwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, "false").equalsIgnoreCase("false")) && (crawler.queuePreStack.size() > 1)) { return "no DHT distribution: indexing in progress: noticeURL.stackSize() = " + crawlQueues.noticeURL.size() + ", sbQueue.size() = " + crawler.queuePreStack.size(); } return null; // this means; yes, please do dht transfer } public boolean dhtTransferJob() { final String rejectReason = dhtShallTransfer(); if (rejectReason != null) { if (this.log.isFine()) log.logFine(rejectReason); return false; } boolean hasDoneSomething = false; if (this.dhtDispatcher.cloudSize() > this.peers.scheme.verticalPartitions() * 4) { log.logInfo("dhtTransferJob: no selection, too many entries in transmission cloud: " + this.dhtDispatcher.cloudSize()); } else if (MemoryControl.available() < 1024*1024*25) { log.logInfo("dhtTransferJob: no selection, too less memory available : " + (MemoryControl.available() / 1024 / 1024) + " MB"); } else { byte[] startHash = null, limitHash = null; int tries = 10; while (tries-- > 0) { startHash = PeerSelection.selectTransferStart(); assert startHash != null; limitHash = PeerSelection.limitOver(this.peers, startHash); if (limitHash != null) break; } if (limitHash == null || startHash == null) { log.logInfo("dhtTransferJob: approaching full DHT dispersion."); return false; } log.logInfo("dhtTransferJob: selected " + new String(startHash) + " as start hash"); log.logInfo("dhtTransferJob: selected " + new String(limitHash) + " as limit hash"); try { boolean enqueued = this.dhtDispatcher.selectContainersEnqueueToCloud( startHash, limitHash, dhtMaxContainerCount, dhtMaxReferenceCount, 5000); hasDoneSomething = hasDoneSomething | enqueued; log.logInfo("dhtTransferJob: result from enqueueing: " + ((enqueued) ? "true" : "false")); } catch (IOException e) { log.logSevere("dhtTransferJob: interrupted with exception: " + e.getMessage(), e); return false; } } if (this.dhtDispatcher.transmissionSize() >= 10) { log.logInfo("dhtTransferJob: no dequeueing from cloud to transmission: too many concurrent sessions: " + this.dhtDispatcher.transmissionSize()); } else { boolean dequeued = this.dhtDispatcher.dequeueContainer(); hasDoneSomething = hasDoneSomething | dequeued; log.logInfo("dhtTransferJob: result from dequeueing: " + ((dequeued) ? "true" : "false")); } return hasDoneSomething; } private void addURLtoErrorDB( final yacyURL url, final String referrerHash, final String initiator, final String name, final String failreason ) { assert initiator != null; // create a new errorURL DB entry final CrawlEntry bentry = new CrawlEntry( initiator, url, referrerHash, (name == null) ? "" : name, new Date(), null, null, 0, 0, 0); final ZURL.Entry ee = crawlQueues.errorURL.newEntry( bentry, initiator, new Date(), 0, failreason); // store the entry ee.store(); // push it onto the stack crawlQueues.errorURL.push(ee); } public int currentPPM() { return serverProfiling.countEvents("indexed", 20000) * 3; } public void updateMySeed() { if (getConfig("peerName", "anomic").equals("anomic")) { // generate new peer name setConfig("peerName", yacySeed.makeDefaultPeerName()); } peers.mySeed().put(yacySeed.NAME, getConfig("peerName", "nameless")); peers.mySeed().put(yacySeed.PORT, Integer.toString(serverCore.getPortNr(getConfig("port", "8080")))); //the speed of indexing (pages/minute) of the peer final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000; peers.mySeed().put(yacySeed.ISPEED, Integer.toString(currentPPM())); totalQPM = requestedQueries * 60d / Math.max(uptime, 1d); peers.mySeed().put(yacySeed.RSPEED, Double.toString(totalQPM /*Math.max((float) requestcdiff, 0f) * 60f / Math.max((float) uptimediff, 1f)*/ )); peers.mySeed().put(yacySeed.UPTIME, Long.toString(uptime/60)); // the number of minutes that the peer is up in minutes/day (moving average MA30) peers.mySeed().put(yacySeed.LCOUNT, Integer.toString(indexSegment.urlMetadata().size())); // the number of links that the peer has stored (LURL's) peers.mySeed().put(yacySeed.NCOUNT, Integer.toString(crawlQueues.noticeURL.size())); // the number of links that the peer has noticed, but not loaded (NURL's) peers.mySeed().put(yacySeed.RCOUNT, Integer.toString(crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT))); // the number of links that the peer provides for remote crawling (ZURL's) peers.mySeed().put(yacySeed.ICOUNT, Integer.toString(indexSegment.termIndex().size())); // the minimum number of words that the peer has indexed (as it says) peers.mySeed().put(yacySeed.SCOUNT, Integer.toString(peers.sizeConnected())); // the number of seeds that the peer has stored peers.mySeed().put(yacySeed.CCOUNT, Double.toString(((int) ((peers.sizeConnected() + peers.sizeDisconnected() + peers.sizePotential()) * 60.0 / (uptime + 1.01)) * 100) / 100.0)); // the number of clients that the peer connects (as connects/hour) peers.mySeed().put(yacySeed.VERSION, getConfig("version", "")); peers.mySeed().setFlagDirectConnect(true); peers.mySeed().setLastSeenUTC(); peers.mySeed().put(yacySeed.UTC, DateFormatter.UTCDiffString()); peers.mySeed().setFlagAcceptRemoteCrawl(getConfig("crawlResponse", "").equals("true")); peers.mySeed().setFlagAcceptRemoteIndex(getConfig("allowReceiveIndex", "").equals("true")); //mySeed.setFlagAcceptRemoteIndex(true); } public void loadSeedLists() { // uses the superseed to initialize the database with known seeds yacySeed ys; String seedListFileURL; yacyURL url; ArrayList seedList; Iterator enu; int lc; final int sc = peers.sizeConnected(); httpResponseHeader header; yacyCore.log.logInfo("BOOTSTRAP: " + sc + " seeds known from previous run"); // - use the superseed to further fill up the seedDB int ssc = 0, c = 0; while (true) { if (Thread.currentThread().isInterrupted()) break; seedListFileURL = sb.getConfig("network.unit.bootstrap.seedlist" + c, ""); if (seedListFileURL.length() == 0) break; c++; if ( seedListFileURL.startsWith("http://") || seedListFileURL.startsWith("https://") ) { // load the seed list try { final httpRequestHeader reqHeader = new httpRequestHeader(); reqHeader.put(httpRequestHeader.PRAGMA, "no-cache"); reqHeader.put(httpRequestHeader.CACHE_CONTROL, "no-cache"); reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); url = new yacyURL(seedListFileURL, null); final long start = System.currentTimeMillis(); header = httpClient.whead(url.toString(), reqHeader); final long loadtime = System.currentTimeMillis() - start; if (header == null) { if (loadtime > getConfigLong("bootstrapLoadTimeout", 6000)) { yacyCore.log.logWarning("BOOTSTRAP: seed-list URL " + seedListFileURL + " not available, time-out after " + loadtime + " milliseconds"); } else { yacyCore.log.logWarning("BOOTSTRAP: seed-list URL " + seedListFileURL + " not available, no content"); } } else if (header.lastModified() == null) { yacyCore.log.logWarning("BOOTSTRAP: seed-list URL " + seedListFileURL + " not usable, last-modified is missing"); } else if ((header.age() > 86400000) && (ssc > 0)) { yacyCore.log.logInfo("BOOTSTRAP: seed-list URL " + seedListFileURL + " too old (" + (header.age() / 86400000) + " days)"); } else { ssc++; final byte[] content = httpClient.wget(url.toString(), reqHeader, (int) getConfigLong("bootstrapLoadTimeout", 20000)); seedList = FileUtils.strings(content, "UTF-8"); enu = seedList.iterator(); lc = 0; while (enu.hasNext()) { ys = yacySeed.genRemoteSeed(enu.next(), null, false); if ((ys != null) && ((!peers.mySeedIsDefined()) || !peers.mySeed().hash.equals(ys.hash))) { if (peers.peerActions.connectPeer(ys, false)) lc++; //seedDB.writeMap(ys.hash, ys.getMap(), "init"); //System.out.println("BOOTSTRAP: received peer " + ys.get(yacySeed.NAME, "anonymous") + "/" + ys.getAddress()); //lc++; } } yacyCore.log.logInfo("BOOTSTRAP: " + lc + " seeds from seed-list URL " + seedListFileURL + ", AGE=" + (header.age() / 3600000) + "h"); } } catch (final IOException e) { // this is when wget fails, commonly because of timeout yacyCore.log.logWarning("BOOTSTRAP: failed (1) to load seeds from seed-list URL " + seedListFileURL + ": " + e.getMessage()); } catch (final Exception e) { // this is when wget fails; may be because of missing internet connection yacyCore.log.logSevere("BOOTSTRAP: failed (2) to load seeds from seed-list URL " + seedListFileURL + ": " + e.getMessage(), e); } } } yacyCore.log.logInfo("BOOTSTRAP: " + (peers.sizeConnected() - sc) + " new seeds while bootstraping."); } public void checkInterruption() throws InterruptedException { final Thread curThread = Thread.currentThread(); if ((curThread instanceof serverThread) && ((serverThread)curThread).shutdownInProgress()) throw new InterruptedException("Shutdown in progress ..."); else if (this.terminate || curThread.isInterrupted()) throw new InterruptedException("Shutdown in progress ..."); } public void terminate(final long delay) { if (delay <= 0) throw new IllegalArgumentException("The shutdown delay must be greater than 0."); (new delayedShutdown(this,delay)).start(); } public void terminate() { this.terminate = true; this.shutdownSync.V(); } public boolean isTerminated() { return this.terminate; } public boolean waitForShutdown() throws InterruptedException { this.shutdownSync.P(); dhtDispatcher.close(); return this.terminate; } /** * loads the url as Map * * Strings like abc=123 are parsed as pair: abc => 123 * * @param url * @return */ public static Map loadHashMap(final yacyURL url) { try { // sending request final httpRequestHeader reqHeader = new httpRequestHeader(); reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); final HashMap result = FileUtils.table(httpClient.wget(url.toString(), reqHeader, 10000), "UTF-8"); if (result == null) return new HashMap(); return result; } catch (final Exception e) { return new HashMap(); } } } class delayedShutdown extends Thread { private final plasmaSwitchboard sb; private final long delay; public delayedShutdown(final plasmaSwitchboard sb, final long delay) { this.sb = sb; this.delay = delay; } public void run() { try { Thread.sleep(delay); } catch (final InterruptedException e) { sb.getLog().logInfo("interrupted delayed shutdown"); } catch (final Exception e) { e.printStackTrace(); } this.sb.terminate(); } }