diff --git a/htroot/api/termlist_p.java b/htroot/api/termlist_p.java index 4d420a650..db299e2ec 100644 --- a/htroot/api/termlist_p.java +++ b/htroot/api/termlist_p.java @@ -37,9 +37,10 @@ import net.yacy.server.serverSwitch; public class termlist_p { + private final static ConcurrentLog log = new ConcurrentLog("TERMLIST"); + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { - final ConcurrentLog log = new ConcurrentLog("TERMLIST"); final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; Segment segment = sb.index; diff --git a/source/net/yacy/cora/language/synonyms/SynonymLibrary.java b/source/net/yacy/cora/language/synonyms/SynonymLibrary.java index 125269705..e55fbe15c 100644 --- a/source/net/yacy/cora/language/synonyms/SynonymLibrary.java +++ b/source/net/yacy/cora/language/synonyms/SynonymLibrary.java @@ -42,7 +42,7 @@ import net.yacy.cora.util.ConcurrentLog; */ public class SynonymLibrary { - ConcurrentLog log = new ConcurrentLog(SynonymLibrary.class.getName()); + private final static ConcurrentLog log = new ConcurrentLog(SynonymLibrary.class.getName()); private Map>> lib; public SynonymLibrary(final File path) { diff --git a/source/net/yacy/cora/protocol/ResponseHeader.java b/source/net/yacy/cora/protocol/ResponseHeader.java index 5d5df4fe4..8f7b90b93 100644 --- a/source/net/yacy/cora/protocol/ResponseHeader.java +++ b/source/net/yacy/cora/protocol/ResponseHeader.java @@ -36,7 +36,7 @@ public class ResponseHeader extends HeaderFramework { // response header properties private static final long serialVersionUID = 0L; - private static ConcurrentLog log = new ConcurrentLog(ResponseHeader.class.getName()); + private static final ConcurrentLog log = new ConcurrentLog(ResponseHeader.class.getName()); public ResponseHeader(final int statusCode) { super(); diff --git a/source/net/yacy/cora/protocol/ftp/FTPClient.java b/source/net/yacy/cora/protocol/ftp/FTPClient.java index 8cbbdd211..47448191b 100644 --- a/source/net/yacy/cora/protocol/ftp/FTPClient.java +++ b/source/net/yacy/cora/protocol/ftp/FTPClient.java @@ -71,7 +71,7 @@ import net.yacy.cora.util.ConcurrentLog; public class FTPClient { - private static ConcurrentLog log = new ConcurrentLog("FTPClient"); + private static final ConcurrentLog log = new ConcurrentLog("FTPClient"); private static final String vDATE = "20100823"; diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index 211fa2e50..7f5fedc30 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -63,7 +63,6 @@ import net.yacy.peers.SeedDB; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.repository.FilterEngine; import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; import net.yacy.search.schema.CollectionConfiguration; @@ -72,8 +71,8 @@ public final class CrawlStacker { public static String ERROR_NO_MATCH_MUST_MATCH_FILTER = "url does not match must-match filter "; public static String ERROR_MATCH_WITH_MUST_NOT_MATCH_FILTER = "url matches must-not-match filter "; - - private final ConcurrentLog log = new ConcurrentLog("STACKCRAWL"); + private final static ConcurrentLog log = new ConcurrentLog("STACKCRAWL"); + private final RobotsTxt robots; private final WorkflowProcessor requestQueue; public final CrawlQueues nextQueue; @@ -103,13 +102,13 @@ public final class CrawlStacker { this.acceptGlobalURLs = acceptGlobalURLs; this.domainList = domainList; this.requestQueue = new WorkflowProcessor("CrawlStacker", "This process checks new urls before they are enqueued into the balancer (proper, double-check, correct domain, filter)", new String[]{"Balancer"}, this, "job", 10000, null, WorkflowProcessor.availableCPU); - this.log.info("STACKCRAWL thread initialized."); + CrawlStacker.log.info("STACKCRAWL thread initialized."); } - public int size() { return this.requestQueue.getQueueSize(); } + public boolean isEmpty() { if (!this.requestQueue.queueIsEmpty()) return false; return true; @@ -120,15 +119,15 @@ public final class CrawlStacker { } public void announceClose() { - this.log.info("Flushing remaining " + size() + " crawl stacker job entries."); + CrawlStacker.log.info("Flushing remaining " + size() + " crawl stacker job entries."); this.requestQueue.shutdown(); } public synchronized void close() { - this.log.info("Shutdown. waiting for remaining " + size() + " crawl stacker job entries. please wait."); + CrawlStacker.log.info("Shutdown. waiting for remaining " + size() + " crawl stacker job entries. please wait."); this.requestQueue.shutdown(); - this.log.info("Shutdown. Closing stackCrawl queue."); + CrawlStacker.log.info("Shutdown. Closing stackCrawl queue."); clear(); } @@ -155,7 +154,7 @@ public final class CrawlStacker { this.nextQueue.errorURL.push(entry.url(), profile, FailCategory.FINAL_LOAD_CONTEXT, rejectReason, -1); } } catch (final Exception e) { - CrawlStacker.this.log.warn("Error while processing stackCrawl entry.\n" + "Entry: " + entry.toString() + "Error: " + e.toString(), e); + CrawlStacker.log.warn("Error while processing stackCrawl entry.\n" + "Entry: " + entry.toString() + "Error: " + e.toString(), e); return null; } return null; @@ -164,7 +163,7 @@ public final class CrawlStacker { public void enqueueEntry(final Request entry) { // DEBUG - if (this.log.isFinest()) this.log.finest("ENQUEUE " + entry.url() + ", referer=" + entry.referrerhash() + ", initiator=" + ((entry.initiator() == null) ? "" : ASCII.String(entry.initiator())) + ", name=" + entry.name() + ", appdate=" + entry.appdate() + ", depth=" + entry.depth()); + if (CrawlStacker.log.isFinest()) CrawlStacker.log.finest("ENQUEUE " + entry.url() + ", referer=" + entry.referrerhash() + ", initiator=" + ((entry.initiator() == null) ? "" : ASCII.String(entry.initiator())) + ", name=" + entry.name() + ", appdate=" + entry.appdate() + ", depth=" + entry.depth()); this.requestQueue.enQueue(entry); } public void enqueueEntriesAsynchronous(final byte[] initiator, final String profileHandle, final List hyperlinks) { @@ -310,7 +309,7 @@ public final class CrawlStacker { String error; if (profile == null) { error = "LOST STACKER PROFILE HANDLE '" + entry.profileHandle() + "' for URL " + entry.url(); - this.log.warn(error); + CrawlStacker.log.warn(error); return error; } @@ -334,7 +333,7 @@ public final class CrawlStacker { if (!local && !global && !remote && !proxy) { error = "URL '" + entry.url().toString() + "' cannot be crawled. initiator = " + ((entry.initiator() == null) ? "" : ASCII.String(entry.initiator())) + ", profile.handle = " + profile.handle(); - this.log.severe(error); + CrawlStacker.log.severe(error); return error; } @@ -362,20 +361,20 @@ public final class CrawlStacker { if (global) { // it may be possible that global == true and local == true, so do not check an error case against it - if (proxy) this.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = proxy" + ", profile.handle = " + profile.handle()); - if (remote) this.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, remote = true, initiator = " + ASCII.String(entry.initiator()) + ", profile.handle = " + profile.handle()); + if (proxy) CrawlStacker.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = proxy" + ", profile.handle = " + profile.handle()); + if (remote) CrawlStacker.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, remote = true, initiator = " + ASCII.String(entry.initiator()) + ", profile.handle = " + profile.handle()); warning = this.nextQueue.noticeURL.push(NoticedURL.StackType.GLOBAL, entry, profile, this.robots); } else if (local) { - if (proxy) this.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, proxy = true, initiator = proxy" + ", profile.handle = " + profile.handle()); - if (remote) this.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, remote = true, initiator = " + ASCII.String(entry.initiator()) + ", profile.handle = " + profile.handle()); + if (proxy) CrawlStacker.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, proxy = true, initiator = proxy" + ", profile.handle = " + profile.handle()); + if (remote) CrawlStacker.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, remote = true, initiator = " + ASCII.String(entry.initiator()) + ", profile.handle = " + profile.handle()); warning = this.nextQueue.noticeURL.push(NoticedURL.StackType.LOCAL, entry, profile, this.robots); } else if (proxy) { - if (remote) this.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: proxy = true, remote = true, initiator = " + ASCII.String(entry.initiator()) + ", profile.handle = " + profile.handle()); + if (remote) CrawlStacker.log.warn("URL '" + entry.url().toString() + "' has conflicting initiator properties: proxy = true, remote = true, initiator = " + ASCII.String(entry.initiator()) + ", profile.handle = " + profile.handle()); warning = this.nextQueue.noticeURL.push(NoticedURL.StackType.LOCAL, entry, profile, this.robots); } else if (remote) { warning = this.nextQueue.noticeURL.push(NoticedURL.StackType.REMOTE, entry, profile, this.robots); } - if (warning != null && this.log.isFine()) this.log.fine("CrawlStacker.stackCrawl of URL " + entry.url().toNormalform(true) + " - not pushed: " + warning); + if (warning != null && CrawlStacker.log.isFine()) CrawlStacker.log.fine("CrawlStacker.stackCrawl of URL " + entry.url().toNormalform(true) + " - not pushed: " + warning); return null; } @@ -405,8 +404,8 @@ public final class CrawlStacker { } else { final boolean recrawl = profile.recrawlIfOlder() > oldDate.getTime(); if (recrawl) { - if (this.log.isInfo()) - this.log.info("RE-CRAWL of URL '" + urlstring + "': this url was crawled " + + if (CrawlStacker.log.isInfo()) + CrawlStacker.log.info("RE-CRAWL of URL '" + urlstring + "': this url was crawled " + ((System.currentTimeMillis() - oldDate.getTime()) / 60000 / 60 / 24) + " days ago."); } else { if (dbocc == null) { @@ -414,10 +413,10 @@ public final class CrawlStacker { } if (dbocc == HarvestProcess.ERRORS) { final CollectionConfiguration.FailDoc errorEntry = this.nextQueue.errorURL.get(ASCII.String(url.hash())); - if (this.log.isInfo()) this.log.info("URL '" + urlstring + "' is double registered in '" + dbocc.toString() + "', previous cause: " + errorEntry.getFailReason()); + if (CrawlStacker.log.isInfo()) CrawlStacker.log.info("URL '" + urlstring + "' is double registered in '" + dbocc.toString() + "', previous cause: " + errorEntry.getFailReason()); return "double in: errors (" + errorEntry.getFailReason() + "), oldDate = " + oldDate.toString(); } - if (this.log.isInfo()) this.log.info("URL '" + urlstring + "' is double registered in '" + dbocc.toString() + "'. "); + if (CrawlStacker.log.isInfo()) CrawlStacker.log.info("URL '" + urlstring + "' is double registered in '" + dbocc.toString() + "'. "); return "double in: " + dbocc.toString() + ", oldDate = " + oldDate.toString(); } } @@ -427,7 +426,7 @@ public final class CrawlStacker { if (maxAllowedPagesPerDomain < Integer.MAX_VALUE && maxAllowedPagesPerDomain > 0) { final AtomicInteger dp = profile.getCount(url.getHost()); if (dp != null && dp.get() >= maxAllowedPagesPerDomain) { - if (this.log.isFine()) this.log.fine("URL '" + urlstring + "' appeared too often in crawl stack, a maximum of " + maxAllowedPagesPerDomain + " is allowed."); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("URL '" + urlstring + "' appeared too often in crawl stack, a maximum of " + maxAllowedPagesPerDomain + " is allowed."); return "crawl stack domain counter exceeded (test by profile)"; } @@ -455,44 +454,44 @@ public final class CrawlStacker { final String urlProtocol = url.getProtocol(); final String urlstring = url.toString(); if (!Switchboard.getSwitchboard().loader.isSupportedProtocol(urlProtocol)) { - this.log.severe("Unsupported protocol in URL '" + urlstring + "'."); + CrawlStacker.log.severe("Unsupported protocol in URL '" + urlstring + "'."); return "unsupported protocol"; } // check if ip is local ip address final String urlRejectReason = urlInAcceptedDomain(url); if (urlRejectReason != null) { - if (this.log.isFine()) this.log.fine("denied_(" + urlRejectReason + ")"); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("denied_(" + urlRejectReason + ")"); return "denied_(" + urlRejectReason + ")"; } // check blacklist if (Switchboard.urlBlacklist.isListed(BlacklistType.CRAWLER, url)) { - this.log.fine("URL '" + urlstring + "' is in blacklist."); + CrawlStacker.log.fine("URL '" + urlstring + "' is in blacklist."); return "url in blacklist"; } // filter with must-match for URLs if ((depth > 0) && !profile.urlMustMatchPattern().matcher(urlstring).matches()) { - if (this.log.isFine()) this.log.fine("URL '" + urlstring + "' does not match must-match crawling filter '" + profile.urlMustMatchPattern().toString() + "'."); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("URL '" + urlstring + "' does not match must-match crawling filter '" + profile.urlMustMatchPattern().toString() + "'."); return ERROR_NO_MATCH_MUST_MATCH_FILTER + profile.urlMustMatchPattern().toString(); } // filter with must-not-match for URLs if ((depth > 0) && profile.urlMustNotMatchPattern().matcher(urlstring).matches()) { - if (this.log.isFine()) this.log.fine("URL '" + urlstring + "' matches must-not-match crawling filter '" + profile.urlMustNotMatchPattern().toString() + "'."); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("URL '" + urlstring + "' matches must-not-match crawling filter '" + profile.urlMustNotMatchPattern().toString() + "'."); return ERROR_MATCH_WITH_MUST_NOT_MATCH_FILTER + profile.urlMustNotMatchPattern().toString(); } // deny cgi if (url.isIndividual() && !profile.crawlingQ()) { // TODO: make special property for crawlingIndividual - if (this.log.isFine()) this.log.fine("URL '" + urlstring + "' is CGI URL."); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("URL '" + urlstring + "' is CGI URL."); return "individual url (sessionid etc) not wanted"; } // deny post properties if (url.isPOST() && !profile.crawlingQ()) { - if (this.log.isFine()) this.log.fine("URL '" + urlstring + "' is post URL."); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("URL '" + urlstring + "' is post URL."); return "post url not allowed"; } @@ -501,13 +500,13 @@ public final class CrawlStacker { // filter with must-match for IPs if ((depth > 0) && profile.ipMustMatchPattern() != CrawlProfile.MATCH_ALL_PATTERN && url.getHost() != null && !profile.ipMustMatchPattern().matcher(url.getInetAddress().getHostAddress()).matches()) { - if (this.log.isFine()) this.log.fine("IP " + url.getInetAddress().getHostAddress() + " of URL '" + urlstring + "' does not match must-match crawling filter '" + profile.ipMustMatchPattern().toString() + "'."); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("IP " + url.getInetAddress().getHostAddress() + " of URL '" + urlstring + "' does not match must-match crawling filter '" + profile.ipMustMatchPattern().toString() + "'."); return "ip " + url.getInetAddress().getHostAddress() + " of url does not match must-match filter"; } // filter with must-not-match for IPs if ((depth > 0) && profile.ipMustNotMatchPattern() != CrawlProfile.MATCH_NEVER_PATTERN && url.getHost() != null && profile.ipMustNotMatchPattern().matcher(url.getInetAddress().getHostAddress()).matches()) { - if (this.log.isFine()) this.log.fine("IP " + url.getInetAddress().getHostAddress() + " of URL '" + urlstring + "' matches must-not-match crawling filter '" + profile.ipMustNotMatchPattern().toString() + "'."); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("IP " + url.getInetAddress().getHostAddress() + " of URL '" + urlstring + "' matches must-not-match crawling filter '" + profile.ipMustNotMatchPattern().toString() + "'."); return "ip " + url.getInetAddress().getHostAddress() + " of url matches must-not-match filter"; } @@ -525,7 +524,7 @@ public final class CrawlStacker { } } if (!granted) { - if (this.log.isFine()) this.log.fine("IP " + url.getInetAddress().getHostAddress() + " of URL '" + urlstring + "' does not match must-match crawling filter '" + profile.ipMustMatchPattern().toString() + "'."); + if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("IP " + url.getInetAddress().getHostAddress() + " of URL '" + urlstring + "' does not match must-match crawling filter '" + profile.ipMustMatchPattern().toString() + "'."); return "country " + c0 + " of url does not match must-match filter for countries"; } } diff --git a/source/net/yacy/crawler/data/CrawlQueues.java b/source/net/yacy/crawler/data/CrawlQueues.java index 7da49f812..1f2e59c16 100644 --- a/source/net/yacy/crawler/data/CrawlQueues.java +++ b/source/net/yacy/crawler/data/CrawlQueues.java @@ -62,9 +62,10 @@ import net.yacy.search.index.ErrorCache; import net.yacy.search.schema.CollectionConfiguration; public class CrawlQueues { + + private final static ConcurrentLog log = new ConcurrentLog("CRAWLER"); private Switchboard sb; - private ConcurrentLog log; private Map workers; // mapping from url hash to Worker thread object private final ArrayList remoteCrawlProviderHashes; @@ -74,12 +75,11 @@ public class CrawlQueues { public CrawlQueues(final Switchboard sb, final File queuePath) { this.sb = sb; - this.log = new ConcurrentLog("CRAWLER"); this.workers = new ConcurrentHashMap(); this.remoteCrawlProviderHashes = new ArrayList(); // start crawling management - this.log.config("Starting Crawling Management"); + log.config("Starting Crawling Management"); this.noticeURL = new NoticedURL(queuePath, sb.useTailCache, sb.exceed134217727); this.errorURL = new ErrorCache(sb.index.fulltext()); this.delegatedURL = new ConcurrentHashMap(); @@ -214,7 +214,7 @@ public class CrawlQueues { for (int i = 0; i < toshift; i++) { this.noticeURL.shift(NoticedURL.StackType.GLOBAL, NoticedURL.StackType.LOCAL, this.sb.crawler, this.sb.robots); } - this.log.info("shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()=" + coreCrawlJobSize() + + CrawlQueues.log.info("shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()=" + coreCrawlJobSize() + ", limitCrawlJobSize()=" + limitCrawlJobSize() + ", cluster.mode=" + this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "") + ", robinsonMode=" + ((this.sb.isRobinsonMode()) ? "on" : "off")); } @@ -222,15 +222,15 @@ public class CrawlQueues { final String queueCheckCore = loadIsPossible(NoticedURL.StackType.LOCAL); final String queueCheckNoload = loadIsPossible(NoticedURL.StackType.NOLOAD); if (queueCheckCore != null && queueCheckNoload != null) { - if (this.log.isFine()) { - this.log.fine("omitting de-queue/local: " + queueCheckCore + ":" + queueCheckNoload); + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine("omitting de-queue/local: " + queueCheckCore + ":" + queueCheckNoload); } return false; } if (isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) { - if (this.log.isFine()) { - this.log.fine("omitting de-queue/local: paused"); + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine("omitting de-queue/local: paused"); } return false; } @@ -252,12 +252,12 @@ public class CrawlQueues { } final String profileHandle = urlEntry.profileHandle(); if (profileHandle == null) { - this.log.severe(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); + CrawlQueues.log.severe(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); return true; } final CrawlProfile profile = this.sb.crawler.get(ASCII.getBytes(profileHandle)); if (profile == null) { - this.log.severe(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); + CrawlQueues.log.severe(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); return true; } this.sb.indexingDocumentProcessor.enQueue(new IndexingQueueEntry(new Response(urlEntry, profile), null, null)); @@ -272,13 +272,13 @@ public class CrawlQueues { // System.out.println("DEBUG plasmaSwitchboard.processCrawling: // profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url()); if (urlEntry.profileHandle() == null) { - this.log.severe(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); + CrawlQueues.log.severe(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); return true; } load(urlEntry, stats); return true; } catch (final IOException e) { - this.log.severe(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); + CrawlQueues.log.severe(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); if (e.getMessage().indexOf("hash is null",0) > 0) { this.noticeURL.clear(NoticedURL.StackType.LOCAL); } @@ -303,8 +303,8 @@ public class CrawlQueues { final DigestURL url = urlEntry.url(); final String urlProtocol = url.getProtocol(); if (this.sb.loader.isSupportedProtocol(urlProtocol)) { - if (this.log.isFine()) { - this.log.fine(stats + ": URL=" + urlEntry.url() + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine(stats + ": URL=" + urlEntry.url() + ", initiator=" + ((urlEntry.initiator() == null) ? "" : ASCII.String(urlEntry.initiator())) + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() @@ -316,7 +316,7 @@ public class CrawlQueues { // work off one Crawl stack entry if (urlEntry == null || urlEntry.url() == null) { - this.log.info(stats + ": urlEntry = null"); + CrawlQueues.log.info(stats + ": urlEntry = null"); } else { if (!this.workers.containsKey(Integer.valueOf(urlEntry.hashCode()))) { Loader loader = new Loader(urlEntry); @@ -331,10 +331,10 @@ public class CrawlQueues { } } else { - this.log.severe("Unsupported protocol in URL '" + url.toString()); + CrawlQueues.log.severe("Unsupported protocol in URL '" + url.toString()); } } else { - if (this.log.isFine()) this.log.fine(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); + if (CrawlQueues.log.isFine()) CrawlQueues.log.fine(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); } } @@ -407,30 +407,30 @@ public class CrawlQueues { } // check again if (this.workers.size() >= this.sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 20)) { - if (this.log.isFine()) { - this.log.fine("remoteCrawlLoaderJob: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.workers.size() + "), httpClients = " + ConnectionInfo.getCount()); + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine("remoteCrawlLoaderJob: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.workers.size() + "), httpClients = " + ConnectionInfo.getCount()); } return false; } final String cautionCause = this.sb.onlineCaution(); if (cautionCause != null) { - if (this.log.isFine()) { - this.log.fine("remoteCrawlLoaderJob: online caution for " + cautionCause + ", omitting processing"); + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine("remoteCrawlLoaderJob: online caution for " + cautionCause + ", omitting processing"); } return false; } if (remoteTriggeredCrawlJobSize() > 200) { - if (this.log.isFine()) { - this.log.fine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing"); + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing"); } return false; } if (coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) { - if (this.log.isFine()) { - this.log.fine("remoteCrawlLoaderJob: a local crawl is running, omitting processing"); + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine("remoteCrawlLoaderJob: a local crawl is running, omitting processing"); } return false; } @@ -521,7 +521,7 @@ public class CrawlQueues { item.getSize() )); } else { - this.log.warn("crawlOrder: Rejected URL '" + urlToString(url) + "': " + urlRejectReason); + CrawlQueues.log.warn("crawlOrder: Rejected URL '" + urlToString(url) + "': " + urlRejectReason); } } return true; @@ -554,15 +554,15 @@ public class CrawlQueues { // or there is no global crawl on the stack final String queueCheck = loadIsPossible(NoticedURL.StackType.REMOTE); if (queueCheck != null) { - if (this.log.isFinest()) { - this.log.finest("omitting de-queue/remote: " + queueCheck); + if (CrawlQueues.log.isFinest()) { + CrawlQueues.log.finest("omitting de-queue/remote: " + queueCheck); } return false; } if (isPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) { - if (this.log.isFinest()) { - this.log.finest("omitting de-queue/remote: paused"); + if (CrawlQueues.log.isFinest()) { + CrawlQueues.log.finest("omitting de-queue/remote: paused"); } return false; } @@ -576,7 +576,7 @@ public class CrawlQueues { load(urlEntry, stats); return true; } catch (final IOException e) { - this.log.severe(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); + CrawlQueues.log.severe(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); if (e.getMessage().indexOf("hash is null",0) > 0) { this.noticeURL.clear(NoticedURL.StackType.REMOTE); } @@ -632,8 +632,8 @@ public class CrawlQueues { final Response response = CrawlQueues.this.sb.loader.load(this.request, profile == null ? CacheStrategy.IFEXIST : profile.cacheStrategy(), BlacklistType.CRAWLER, this.profile.getAgent()); if (response == null) { this.request.setStatus("error", WorkflowJob.STATUS_FINISHED); - if (CrawlQueues.this.log.isFine()) { - CrawlQueues.this.log.fine("problem loading " + this.request.url().toString() + ": no content (possibly caused by cache policy)"); + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine("problem loading " + this.request.url().toString() + ": no content (possibly caused by cache policy)"); } result = "no content (possibly caused by cache policy)"; } else { @@ -644,8 +644,8 @@ public class CrawlQueues { } } catch (final IOException e) { this.request.setStatus("error", WorkflowJob.STATUS_FINISHED); - if (CrawlQueues.this.log.isFine()) { - CrawlQueues.this.log.fine("problem loading " + this.request.url().toString() + ": " + e.getMessage()); + if (CrawlQueues.log.isFine()) { + CrawlQueues.log.fine("problem loading " + this.request.url().toString() + ": " + e.getMessage()); } result = "load error - " + e.getMessage(); } diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.java index fb8b9f7f0..c90039e92 100644 --- a/source/net/yacy/crawler/robots/RobotsTxt.java +++ b/source/net/yacy/crawler/robots/RobotsTxt.java @@ -54,7 +54,7 @@ import net.yacy.repository.Blacklist.BlacklistType; public class RobotsTxt { - private static ConcurrentLog log = new ConcurrentLog(RobotsTxt.class.getName()); + private final static ConcurrentLog log = new ConcurrentLog(RobotsTxt.class.getName()); protected static final String ROBOTS_DB_PATH_SEPARATOR = ";"; protected static final Pattern ROBOTS_DB_PATH_SEPARATOR_MATCHER = Pattern.compile(ROBOTS_DB_PATH_SEPARATOR); diff --git a/source/net/yacy/document/language/LanguageStatistics.java b/source/net/yacy/document/language/LanguageStatistics.java index fe97bcbf5..373f01ca0 100644 --- a/source/net/yacy/document/language/LanguageStatistics.java +++ b/source/net/yacy/document/language/LanguageStatistics.java @@ -41,7 +41,7 @@ import net.yacy.cora.util.ConcurrentLog; */ public class LanguageStatistics { - private static ConcurrentLog logger = new ConcurrentLog("LANGUAGESTATISTICS"); + private final static ConcurrentLog logger = new ConcurrentLog("LANGUAGESTATISTICS"); /** This variable holds the name of the language. */ private String langName = null; diff --git a/source/net/yacy/gui/framework/Switchboard.java b/source/net/yacy/gui/framework/Switchboard.java index d655f041b..e89af2545 100644 --- a/source/net/yacy/gui/framework/Switchboard.java +++ b/source/net/yacy/gui/framework/Switchboard.java @@ -56,7 +56,7 @@ public class Switchboard { */ private static Properties properties = new Properties(); - public static ConcurrentLog log = new ConcurrentLog(Switchboard.class.getName()); + public final static ConcurrentLog log = new ConcurrentLog(Switchboard.class.getName()); public static void startInfoUpdater() { diff --git a/source/net/yacy/kelondro/util/OS.java b/source/net/yacy/kelondro/util/OS.java index 502d2e655..1a5b0e3e1 100644 --- a/source/net/yacy/kelondro/util/OS.java +++ b/source/net/yacy/kelondro/util/OS.java @@ -102,6 +102,8 @@ public final class OS { return maxmem; } + private final static ConcurrentLog memchecklog = new ConcurrentLog("MEMCHECK"); + /** * checks heap (may cause high system load) * @param mem heap to check in -Xmx[heap]m @@ -115,7 +117,7 @@ public final class OS { processArgs.add("-Xms4m"); processArgs.add("-Xmx" + Integer.toString(mem) + "m"); try { - line = ConsoleInterface.getLastLineConsoleOutput(processArgs, new ConcurrentLog("MEMCHECK")); + line = ConsoleInterface.getLastLineConsoleOutput(processArgs, memchecklog); } catch (final IOException e) { return false; } diff --git a/source/net/yacy/repository/LoaderDispatcher.java b/source/net/yacy/repository/LoaderDispatcher.java index d421d5258..668ebb2a5 100644 --- a/source/net/yacy/repository/LoaderDispatcher.java +++ b/source/net/yacy/repository/LoaderDispatcher.java @@ -67,6 +67,7 @@ import net.yacy.search.Switchboard; public final class LoaderDispatcher { private final static int accessTimeMaxsize = 1000; + private final static ConcurrentLog log = new ConcurrentLog("LOADER"); private static final ConcurrentHashMap accessTime = new ConcurrentHashMap(); // to protect targets from DDoS private final Switchboard sb; @@ -76,18 +77,16 @@ public final class LoaderDispatcher { private final SMBLoader smbLoader; private final FileLoader fileLoader; private final ConcurrentHashMap loaderSteering; // a map that delivers a 'finish' semaphore for urls - private final ConcurrentLog log; public LoaderDispatcher(final Switchboard sb) { this.sb = sb; this.supportedProtocols = new HashSet(Arrays.asList(new String[]{"http","https","ftp","smb","file"})); // initiate loader objects - this.log = new ConcurrentLog("LOADER"); - this.httpLoader = new HTTPLoader(sb, this.log); - this.ftpLoader = new FTPLoader(sb, this.log); - this.smbLoader = new SMBLoader(sb, this.log); - this.fileLoader = new FileLoader(sb, this.log); + this.httpLoader = new HTTPLoader(sb, LoaderDispatcher.log); + this.ftpLoader = new FTPLoader(sb, LoaderDispatcher.log); + this.smbLoader = new SMBLoader(sb, LoaderDispatcher.log); + this.fileLoader = new FileLoader(sb, LoaderDispatcher.log); this.loaderSteering = new ConcurrentHashMap(); } @@ -224,7 +223,7 @@ public final class LoaderDispatcher { // well, just take the cache and don't care about freshness of the content final byte[] content = Cache.getContent(url.hash()); if (content != null) { - this.log.info("cache hit/useall for: " + url.toNormalform(true)); + LoaderDispatcher.log.info("cache hit/useall for: " + url.toNormalform(true)); response.setContent(content); return response; } @@ -235,14 +234,14 @@ public final class LoaderDispatcher { if (response.isFreshForProxy()) { final byte[] content = Cache.getContent(url.hash()); if (content != null) { - this.log.info("cache hit/fresh for: " + url.toNormalform(true)); + LoaderDispatcher.log.info("cache hit/fresh for: " + url.toNormalform(true)); response.setContent(content); return response; } } - this.log.info("cache hit/stale for: " + url.toNormalform(true)); + LoaderDispatcher.log.info("cache hit/stale for: " + url.toNormalform(true)); } else if (cachedResponse != null) { - this.log.warn("HTCACHE contained response header, but not content for url " + url.toNormalform(true)); + LoaderDispatcher.log.warn("HTCACHE contained response header, but not content for url " + url.toNormalform(true)); } } @@ -266,7 +265,7 @@ public final class LoaderDispatcher { cleanupAccessTimeTable(untilTime); if (System.currentTimeMillis() < untilTime) { long frcdslp = untilTime - System.currentTimeMillis(); - this.log.info("Forcing sleep of " + frcdslp + " ms for host " + host); + LoaderDispatcher.log.info("Forcing sleep of " + frcdslp + " ms for host " + host); try {Thread.sleep(frcdslp);} catch (final InterruptedException ee) {} } } @@ -310,10 +309,10 @@ public final class LoaderDispatcher { try { Cache.store(url, response.getResponseHeader(), response.getContent()); } catch (final IOException e) { - this.log.warn("cannot write " + response.url() + " to Cache (3): " + e.getMessage(), e); + LoaderDispatcher.log.warn("cannot write " + response.url() + " to Cache (3): " + e.getMessage(), e); } } else { - this.log.warn("cannot write " + response.url() + " to Cache (4): " + storeError); + LoaderDispatcher.log.warn("cannot write " + response.url() + " to Cache (4): " + storeError); } return response; } diff --git a/source/net/yacy/search/index/ErrorCache.java b/source/net/yacy/search/index/ErrorCache.java index d460ab680..3edd65c4e 100644 --- a/source/net/yacy/search/index/ErrorCache.java +++ b/source/net/yacy/search/index/ErrorCache.java @@ -46,7 +46,7 @@ import net.yacy.search.schema.CollectionSchema; public class ErrorCache { - private static ConcurrentLog log = new ConcurrentLog("REJECTED"); + private static final ConcurrentLog log = new ConcurrentLog("REJECTED"); private static final int maxStackSize = 1000; // the class object diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 73c794b55..8b5f1777e 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -113,7 +113,7 @@ public final class SearchEvent { } */ - public static ConcurrentLog log = new ConcurrentLog("SEARCH"); + public final static ConcurrentLog log = new ConcurrentLog("SEARCH"); public static final int SNIPPET_MAX_LENGTH = 220; private static final int MAX_TOPWORDS = 12; // default count of words for topicnavigagtor diff --git a/source/net/yacy/utils/gzip.java b/source/net/yacy/utils/gzip.java index 11b01fef6..2a387cf3b 100644 --- a/source/net/yacy/utils/gzip.java +++ b/source/net/yacy/utils/gzip.java @@ -44,7 +44,7 @@ import net.yacy.cora.util.ConcurrentLog; public class gzip { - private static ConcurrentLog logger = new ConcurrentLog("GZIP"); + private final static ConcurrentLog logger = new ConcurrentLog("GZIP"); public static void gzipFile(final String inFile, final String outFile) { try {