diff --git a/htroot/PerformanceGraph.java b/htroot/PerformanceGraph.java index 45ad9dc81..1eeef24b9 100644 --- a/htroot/PerformanceGraph.java +++ b/htroot/PerformanceGraph.java @@ -26,6 +26,7 @@ import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaProfiling; +import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.ymage.ymageMatrix; @@ -33,14 +34,14 @@ import de.anomic.ymage.ymageMatrix; public class PerformanceGraph { public static ymageMatrix respond(httpHeader header, serverObjects post, serverSwitch env) { - //plasmaSwitchboard sb = (plasmaSwitchboard) env; + plasmaSwitchboard sb = (plasmaSwitchboard) env; if (post == null) post = new serverObjects(); int width = post.getInt("width", 660); int height = post.getInt("height", 240); - return plasmaProfiling.performanceGraph(width, height); + return plasmaProfiling.performanceGraph(width, height, sb.wordIndex.countURL() + " URLS / " + sb.wordIndex.collectionsSize() + " WORDS IN COLLECTIONS / " + sb.wordIndex.cacheSize() + " WORDS IN CACHE"); } } \ No newline at end of file diff --git a/source/dbtest.java b/source/dbtest.java index 758c99888..d4e90e188 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -542,7 +542,7 @@ final class memprofiler extends Thread { public memprofiler(int width, int height, int expectedTimeSeconds, File outputFile) { this.outputFile = outputFile; int expectedKilobytes = 20 * 1024;//(Runtime.getRuntime().totalMemory() / 1024); - memChart = new ymageChart(width, height, "FFFFFF", "000000", 50, 20, 20, 20, "MEMORY CHART FROM EXECUTION AT " + new Date()); + memChart = new ymageChart(width, height, "FFFFFF", "000000", "000000", 50, 20, 20, 20, "MEMORY CHART FROM EXECUTION AT " + new Date(), null); int timescale = 10; // steps with each 10 seconds int memscale = 1024; memChart.declareDimension(ymageChart.DIMENSION_BOTTOM, timescale, (width - 40) * timescale / expectedTimeSeconds, 0, "FFFFFF", "555555", "SECONDS"); diff --git a/source/de/anomic/kelondro/kelondroBase64Order.java b/source/de/anomic/kelondro/kelondroBase64Order.java index 69a27069b..e23231515 100644 --- a/source/de/anomic/kelondro/kelondroBase64Order.java +++ b/source/de/anomic/kelondro/kelondroBase64Order.java @@ -267,7 +267,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implement } catch (ArrayIndexOutOfBoundsException e) { // maybe the input was not base64 // throw new RuntimeException("input probably not base64"); - this.log.logFine("wrong string receive: " + in + ", call: " + info); + if (this.log.isFine()) this.log.logFine("wrong string receive: " + in + ", call: " + info); return new byte[0]; } } diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index faf2932ec..f46f9611d 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -167,7 +167,7 @@ public class kelondroEcoTable implements kelondroIndex { System.out.flush(); this.file = new kelondroBufferedEcoFS(new kelondroEcoFS(tablefile, rowdef.objectsize), this.buffersize); ArrayList doubles = index.removeDoubles(); - assert index.size() + doubles.size() + fail == i; + //assert index.size() + doubles.size() + fail == i; System.out.println(" -removed " + doubles.size() + " doubles- done."); if (doubles.size() > 0) { System.out.println("DEBUG " + tablefile + ": WARNING - EcoTable " + tablefile + " has " + doubles.size() + " doubles"); @@ -196,7 +196,7 @@ public class kelondroEcoTable implements kelondroIndex { } try { - assert file.size() == index.size() + doubles.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size() + ", doubles.size() = " + doubles.size() + ", fail = " + fail + ", i = " + i; + assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size() + ", doubles.size() = " + doubles.size() + ", fail = " + fail + ", i = " + i; } catch (IOException e) { e.printStackTrace(); } diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java index 1e4f02c56..7583da12a 100644 --- a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java +++ b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java @@ -168,15 +168,15 @@ public class plasmaCrawlQueues { return false; } if (sb.sbQueue.size() >= (int) sb.getConfigLong(plasmaSwitchboard.INDEXER_SLOTS, 30)) { - log.logFine("CoreCrawl: too many processes in indexing queue, dismissed (" + "sbQueueSize=" + sb.sbQueue.size() + ")"); + if (this.log.isFine()) log.logFine("CoreCrawl: too many processes in indexing queue, dismissed (" + "sbQueueSize=" + sb.sbQueue.size() + ")"); return false; } if (this.size() >= sb.getConfigLong(plasmaSwitchboard.CRAWLER_THREADS_ACTIVE_MAX, 10)) { - log.logFine("CoreCrawl: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.size() + ")"); + if (this.log.isFine()) log.logFine("CoreCrawl: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.size() + ")"); return false; } if (sb.onlineCaution()) { - log.logFine("CoreCrawl: online caution, omitting processing"); + if (this.log.isFine()) log.logFine("CoreCrawl: online caution, omitting processing"); return false; } @@ -218,7 +218,7 @@ public class plasmaCrawlQueues { return true; } - log.logFine("LOCALCRAWL: URL=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + profile.generalFilter() + if (this.log.isFine()) log.logFine("LOCALCRAWL: URL=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + profile.generalFilter() + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); processLocalCrawling(urlEntry, stats); @@ -245,17 +245,17 @@ public class plasmaCrawlQueues { } if (sb.sbQueue.size() >= (int) sb.getConfigLong(plasmaSwitchboard.INDEXER_SLOTS, 30)) { - log.logFine("remoteCrawlLoaderJob: too many processes in indexing queue, dismissed (" + "sbQueueSize=" + sb.sbQueue.size() + ")"); + if (this.log.isFine()) log.logFine("remoteCrawlLoaderJob: too many processes in indexing queue, dismissed (" + "sbQueueSize=" + sb.sbQueue.size() + ")"); return false; } if (this.size() >= sb.getConfigLong(plasmaSwitchboard.CRAWLER_THREADS_ACTIVE_MAX, 10)) { - log.logFine("remoteCrawlLoaderJob: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.size() + ")"); + if (this.log.isFine()) log.logFine("remoteCrawlLoaderJob: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.size() + ")"); return false; } if (sb.onlineCaution()) { - log.logFine("remoteCrawlLoaderJob: online caution, omitting processing"); + if (this.log.isFine()) log.logFine("remoteCrawlLoaderJob: online caution, omitting processing"); return false; } @@ -360,15 +360,15 @@ public class plasmaCrawlQueues { return false; } if (sb.sbQueue.size() >= (int) sb.getConfigLong(plasmaSwitchboard.INDEXER_SLOTS, 30)) { - log.logFine("GlobalCrawl: too many processes in indexing queue, dismissed (" + "sbQueueSize=" + sb.sbQueue.size() + ")"); + if (this.log.isFine()) log.logFine("GlobalCrawl: too many processes in indexing queue, dismissed (" + "sbQueueSize=" + sb.sbQueue.size() + ")"); return false; } if (this.size() >= sb.getConfigLong(plasmaSwitchboard.CRAWLER_THREADS_ACTIVE_MAX, 10)) { - log.logFine("GlobalCrawl: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.size() + ")"); + if (this.log.isFine()) log.logFine("GlobalCrawl: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.size() + ")"); return false; } if (sb.onlineCaution()) { - log.logFine("GlobalCrawl: online caution, omitting processing"); + if (this.log.isFine()) log.logFine("GlobalCrawl: online caution, omitting processing"); return false; } @@ -407,7 +407,7 @@ public class plasmaCrawlQueues { return true; } - log.logFine("plasmaSwitchboard.remoteTriggeredCrawlJob: url=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + if (this.log.isFine()) log.logFine("plasmaSwitchboard.remoteTriggeredCrawlJob: url=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + profile.generalFilter() + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); processLocalCrawling(urlEntry, stats); @@ -476,7 +476,7 @@ public class plasmaCrawlQueues { // checking robots.txt for http(s) resources this.entry.setStatus("worker-checkingrobots"); if ((entry.url().getProtocol().equals("http") || entry.url().getProtocol().equals("https")) && robotsParser.isDisallowed(entry.url())) { - log.logFine("Crawling of URL '" + entry.url().toString() + "' disallowed by robots.txt."); + if (log.isFine()) log.logFine("Crawling of URL '" + entry.url().toString() + "' disallowed by robots.txt."); plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), "denied by robots.txt"); eentry.store(); errorURL.push(eentry); diff --git a/source/de/anomic/plasma/dbImport/dbImportManager.java b/source/de/anomic/plasma/dbImport/dbImportManager.java index 12bbe231f..cbbfbfbab 100644 --- a/source/de/anomic/plasma/dbImport/dbImportManager.java +++ b/source/de/anomic/plasma/dbImport/dbImportManager.java @@ -106,11 +106,11 @@ public class dbImportManager { runningJobs.interrupt(); // we need to use a timeout here because of missing interruptable session threads ... - log.logFine("Waiting for " + runningJobs.activeCount() + " remaining dbImporter threads to finish shutdown ..."); + if (log.isFine()) log.logFine("Waiting for " + runningJobs.activeCount() + " remaining dbImporter threads to finish shutdown ..."); for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) { Thread currentThread = threadList[currentThreadIdx]; if (currentThread.isAlive()) { - log.logFine("Waiting for dbImporter thread '" + currentThread.getName() + "' [" + currentThreadIdx + "] to finish shutdown."); + if (log.isFine()) log.logFine("Waiting for dbImporter thread '" + currentThread.getName() + "' [" + currentThreadIdx + "] to finish shutdown."); try { currentThread.join(500); } catch (InterruptedException ex) {} } } diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index 3d5fe89c7..3e7764843 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -188,7 +188,7 @@ public final class plasmaCrawlStacker extends Thread { } terminateDNSPrefetcher(); - this.log.logFine("Shutdown. Closing stackCrawl queue."); + this.log.logInfo("Shutdown. Closing stackCrawl queue."); // closing the db this.urlEntryCache.close(); @@ -387,7 +387,7 @@ public final class plasmaCrawlStacker extends Thread { // check if ip is local ip address if (!sb.acceptURL(entry.url())) { reason = plasmaCrawlEURL.DENIED_IP_ADDRESS_NOT_IN_DECLARED_DOMAIN + "[" + sb.getConfig("network.unit.domain", "unknown") + "]"; - this.log.logFine("Host in URL '" + entry.url().toString() + "' has IP address outside of declared range (" + sb.getConfig("network.unit.domain", "unknown") + "). " + + if (this.log.isFine()) this.log.logFine("Host in URL '" + entry.url().toString() + "' has IP address outside of declared range (" + sb.getConfig("network.unit.domain", "unknown") + "). " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } @@ -395,7 +395,7 @@ public final class plasmaCrawlStacker extends Thread { // check blacklist if (plasmaSwitchboard.urlBlacklist.isListed(indexReferenceBlacklist.BLACKLIST_CRAWLER, entry.url())) { reason = plasmaCrawlEURL.DENIED_URL_IN_BLACKLIST; - this.log.logFine("URL '" + entry.url().toString() + "' is in blacklist. " + + if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' is in blacklist. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } @@ -411,7 +411,7 @@ public final class plasmaCrawlStacker extends Thread { if ((entry.depth() > 0) && (profile != null) && (!(entry.url().toString().matches(profile.generalFilter())))) { reason = plasmaCrawlEURL.DENIED_URL_DOES_NOT_MATCH_FILTER; - this.log.logFine("URL '" + entry.url().toString() + "' does not match crawling filter '" + profile.generalFilter() + "'. " + + if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' does not match crawling filter '" + profile.generalFilter() + "'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } @@ -420,7 +420,7 @@ public final class plasmaCrawlStacker extends Thread { if (entry.url().isCGI()) { reason = plasmaCrawlEURL.DENIED_CGI_URL; - this.log.logFine("URL '" + entry.url().toString() + "' is CGI URL. " + + if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' is CGI URL. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } @@ -429,7 +429,7 @@ public final class plasmaCrawlStacker extends Thread { if ((entry.url().isPOST()) && (profile != null) && (!(profile.crawlingQ()))) { reason = plasmaCrawlEURL.DENIED_POST_URL; - this.log.logFine("URL '" + entry.url().toString() + "' is post URL. " + + if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' is post URL. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } @@ -444,7 +444,7 @@ public final class plasmaCrawlStacker extends Thread { // deny urls that do not match with the profile domain list if (!(profile.grantedDomAppearance(entry.url().getHost()))) { reason = plasmaCrawlEURL.DENIED_NO_MATCH_WITH_DOMAIN_FILTER; - this.log.logFine("URL '" + entry.url().toString() + "' is not listed in granted domains. " + + if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' is not listed in granted domains. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } @@ -452,7 +452,7 @@ public final class plasmaCrawlStacker extends Thread { // deny urls that exceed allowed number of occurrences if (!(profile.grantedDomCount(entry.url().getHost()))) { reason = plasmaCrawlEURL.DENIED_DOMAIN_COUNT_EXCEEDED; - this.log.logFine("URL '" + entry.url().toString() + "' appeared too often, a maximum of " + profile.domMaxPages() + " is allowed. "+ + if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' appeared too often, a maximum of " + profile.domMaxPages() + " is allowed. "+ "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } @@ -464,18 +464,18 @@ public final class plasmaCrawlStacker extends Thread { // do double-check if ((dbocc != null) && (!recrawl)) { reason = plasmaCrawlEURL.DOUBLE_REGISTERED + dbocc + ")"; - this.log.logFine("URL '" + entry.url().toString() + "' is double registered in '" + dbocc + "'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); + if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' is double registered in '" + dbocc + "'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } if ((oldEntry != null) && (!recrawl)) { reason = plasmaCrawlEURL.DOUBLE_REGISTERED + "LURL)"; - this.log.logFine("URL '" + entry.url().toString() + "' is double registered in 'LURL'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); + if (this.log.isFine()) this.log.logFine("URL '" + entry.url().toString() + "' is double registered in 'LURL'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms"); return reason; } // show potential re-crawl if (recrawl) { - this.log.logFine("RE-CRAWL of URL '" + entry.url().toString() + "': this url was crawled " + + if (this.log.isFine()) this.log.logFine("RE-CRAWL of URL '" + entry.url().toString() + "': this url was crawled " + ((System.currentTimeMillis() - oldEntry.loaddate().getTime()) / 60000 / 60 / 24) + " days ago."); } diff --git a/source/de/anomic/plasma/plasmaDHTChunk.java b/source/de/anomic/plasma/plasmaDHTChunk.java index 6486f124b..92d0a9f8c 100644 --- a/source/de/anomic/plasma/plasmaDHTChunk.java +++ b/source/de/anomic/plasma/plasmaDHTChunk.java @@ -132,13 +132,13 @@ public class plasmaDHTChunk { this.log = log; this.wordIndex = wordIndex; this.startPointHash = selectTransferStart(); - log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, this.startPointHash)); + if (this.log.isFine()) log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, this.startPointHash)); selectTransferContainers(this.startPointHash, minCount, maxCount, maxtime); // count the indexes, can be smaller as expected this.idxCount = indexCounter(); if (this.idxCount < minCount) { - log.logFine("Too few (" + this.idxCount + ") indexes selected for transfer."); + if (this.log.isFine()) log.logFine("Too few (" + this.idxCount + ") indexes selected for transfer."); this.status = chunkStatus_FAILED; } } catch (InterruptedException e) { @@ -150,13 +150,13 @@ public class plasmaDHTChunk { try { this.log = log; this.wordIndex = wordIndex; - log.logFine("Demanded hash " + startHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, this.startPointHash)); + if (this.log.isFine()) log.logFine("Demanded hash " + startHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, this.startPointHash)); selectTransferContainers(startHash, minCount, maxCount, maxtime); // count the indexes, can be smaller as expected this.idxCount = indexCounter(); if (this.idxCount < minCount) { - log.logFine("Too few (" + this.idxCount + ") indexes selected for transfer."); + if (this.log.isFine()) log.logFine("Too few (" + this.idxCount + ") indexes selected for transfer."); this.status = chunkStatus_FAILED; } } catch (InterruptedException e) { @@ -194,11 +194,11 @@ public class plasmaDHTChunk { this.selectionStartTime = System.currentTimeMillis(); int refcountRAM = selectTransferContainersResource(hash, true, maxcount, maxtime); if (refcountRAM >= mincount) { - log.logFine("DHT selection from RAM: " + refcountRAM + " entries"); + if (this.log.isFine()) log.logFine("DHT selection from RAM: " + refcountRAM + " entries"); return; } int refcountFile = selectTransferContainersResource(hash, false, maxcount, maxtime); - log.logFine("DHT selection from FILE: " + refcountFile + " entries, RAM provided only " + refcountRAM + " entries"); + if (this.log.isFine()) log.logFine("DHT selection from FILE: " + refcountFile + " entries, RAM provided only " + refcountRAM + " entries"); return; } finally { this.selectionEndTime = System.currentTimeMillis(); @@ -268,7 +268,7 @@ public class plasmaDHTChunk { } // use whats left - log.logFine("Selected partial index (" + container.size() + " from " + wholesize + " URLs, " + notBoundCounter + " not bound) for word " + container.getWordHash()); + if (this.log.isFine()) log.logFine("Selected partial index (" + container.size() + " from " + wholesize + " URLs, " + notBoundCounter + " not bound) for word " + container.getWordHash()); tmpContainers.add(container); } catch (kelondroException e) { log.logSevere("plasmaWordIndexDistribution/2: deleted DB for word " + container.getWordHash(), e); @@ -279,7 +279,7 @@ public class plasmaDHTChunk { indexContainers = (indexContainer[]) tmpContainers.toArray(new indexContainer[tmpContainers.size()]); //[C[16GwGuFzwffp] has 1 entries, C[16hGKMAl0w97] has 9 entries, C[17A8cDPF6SfG] has 9 entries, C[17Kdj__WWnUy] has 1 entries, C[1 if ((indexContainers == null) || (indexContainers.length == 0)) { - log.logFine("No index available for index transfer, hash start-point " + startPointHash); + if (this.log.isFine()) log.logFine("No index available for index transfer, hash start-point " + startPointHash); this.status = chunkStatus_FAILED; return 0; } @@ -304,7 +304,7 @@ public class plasmaDHTChunk { for (int i = 0; i < this.indexContainers.length; i++) { // delete entries separately if (this.indexContainers[i] == null) { - log.logFine("Deletion of partial index #" + i + " not possible, entry is null"); + if (this.log.isFine()) log.logFine("Deletion of partial index #" + i + " not possible, entry is null"); continue; } int c = this.indexContainers[i].size(); @@ -317,7 +317,7 @@ public class plasmaDHTChunk { String wordHash = indexContainers[i].getWordHash(); count = wordIndex.removeEntriesExpl(this.indexContainers[i].getWordHash(), urlHashes); if (log.isFine()) - log.logFine("Deleted partial index (" + c + " URLs) for word " + wordHash + "; " + this.wordIndex.indexSize(wordHash) + " entries left"); + if (this.log.isFine()) log.logFine("Deleted partial index (" + c + " URLs) for word " + wordHash + "; " + this.wordIndex.indexSize(wordHash) + " entries left"); this.indexContainers[i] = null; } return count; diff --git a/source/de/anomic/plasma/plasmaDHTFlush.java b/source/de/anomic/plasma/plasmaDHTFlush.java index dca80fab2..3b23f8f57 100644 --- a/source/de/anomic/plasma/plasmaDHTFlush.java +++ b/source/de/anomic/plasma/plasmaDHTFlush.java @@ -154,7 +154,7 @@ public class plasmaDHTFlush extends Thread { plasmaDHTChunk newDHTChunk = null, oldDHTChunk = null; try { // initial startingpoint of intex transfer is "AAAAAAAAAAAA" - this.log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution of whole index"); + if (this.log.isFine()) this.log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution of whole index"); /* Loop until we have * - finished transfer of whole index @@ -181,7 +181,7 @@ public class plasmaDHTFlush extends Thread { this.startPointHash = "AAAAAAAAAAAA"; } else { // otherwise we could end transfer now - this.log.logFine("No index available for index transfer, hash start-point " + this.startPointHash); + if (this.log.isFine()) this.log.logFine("No index available for index transfer, hash start-point " + this.startPointHash); this.status = "Finished. " + iteration + " chunks transfered."; this.finished = true; } @@ -223,7 +223,7 @@ public class plasmaDHTFlush extends Thread { if (this.delete) { this.status = "Running: Deleting chunk " + iteration; String urlReferences = oldDHTChunk.deleteTransferIndexes(); - this.log.logFine("Deleted from " + oldDHTChunk.containerSize() + " transferred RWIs locally " + urlReferences + " URL references"); + if (this.log.isFine()) this.log.logFine("Deleted from " + oldDHTChunk.containerSize() + " transferred RWIs locally " + urlReferences + " URL references"); } oldDHTChunk = null; } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 75114444d..47c01f0b9 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -407,7 +407,7 @@ public final class plasmaHTCache { if (deleteFileandDirs(getCachePath(url), "FROM")) { try { // As the file is gone, the entry in responseHeader.db is not needed anymore - log.logFinest("Trying to remove responseHeader from URL: " + url.toNormalform(false, true)); + if (log.isFinest()) log.logFinest("Trying to remove responseHeader from URL: " + url.toNormalform(false, true)); responseHeaderDB.remove(url.hash()); } catch (IOException e) { resetResponseHeaderDB(); @@ -436,7 +436,7 @@ public final class plasmaHTCache { // If the has been emptied, remove it // Loop as long as we produce empty driectoriers, but stop at HTCACHE while ((!(obj.equals(cachePath))) && (obj.isDirectory()) && (obj.list().length == 0)) { - if (obj.delete()) log.logFine("DELETED EMPTY DIRECTORY : " + obj.toString()); + if (obj.delete()) if (log.isFine()) log.logFine("DELETED EMPTY DIRECTORY : " + obj.toString()); obj = obj.getParentFile(); } return true; @@ -458,19 +458,19 @@ public final class plasmaHTCache { if (System.currentTimeMillis() - t < 300000) break; // files must have been at least 5 minutes in the cache before they are deleted if (file != null) { if (filesInUse.contains(file)) continue; - log.logFinest("Trying to delete [" + key + "] = old file: " + file.toString()); + if (log.isFinest()) log.logFinest("Trying to delete [" + key + "] = old file: " + file.toString()); // This needs to be called *before* the file is deleted String urlHash = getHash(file); if (deleteFileandDirs(file, "OLD")) { try { // As the file is gone, the entry in responseHeader.db is not needed anymore if (urlHash != null) { - log.logFinest("Trying to remove responseHeader for URLhash: " + urlHash); + if (log.isFinest()) log.logFinest("Trying to remove responseHeader for URLhash: " + urlHash); responseHeaderDB.remove(urlHash); } else { yacyURL url = getURL(file); if (url != null) { - log.logFinest("Trying to remove responseHeader for URL: " + url.toNormalform(false, true)); + if (log.isFinest()) log.logFinest("Trying to remove responseHeader for URL: " + url.toNormalform(false, true)); responseHeaderDB.remove(url.hash()); } } @@ -676,13 +676,13 @@ public final class plasmaHTCache { cacheAge.put(ageString(d, newpath), newpath); File obj = oldpath.getParentFile(); while ((!(obj.equals(cachePath))) && (obj.isDirectory()) && (obj.list().length == 0)) { - if (obj.delete()) log.logFine("DELETED EMPTY DIRECTORY : " + obj.toString()); + if (obj.delete()) if (log.isFine()) log.logFine("DELETED EMPTY DIRECTORY : " + obj.toString()); obj = obj.getParentFile(); } } } } catch (Exception e) { - log.logFine("moveCachedObject('" + oldpath.toString() + "','" + + if (log.isFine()) log.logFine("moveCachedObject('" + oldpath.toString() + "','" + newpath.toString() + "')", e); } } diff --git a/source/de/anomic/plasma/plasmaProfiling.java b/source/de/anomic/plasma/plasmaProfiling.java index 4f53991ad..1588e019a 100644 --- a/source/de/anomic/plasma/plasmaProfiling.java +++ b/source/de/anomic/plasma/plasmaProfiling.java @@ -63,7 +63,7 @@ public class plasmaProfiling { return max; } - public static ymageMatrix performanceGraph(int width, int height) { + public static ymageMatrix performanceGraph(int width, int height, String subline) { // find maximum values for automatic graph dimension adoption int maxppm = (int) maxPayload("ppm", 25); long maxbytes = maxPayload("memory", 110 * 1024 * 1024); @@ -73,13 +73,13 @@ public class plasmaProfiling { int rightborder = 30; int topborder = 20; int bottomborder = 20; - int leftscale = 20; + int leftscale = 50; int rightscale = 100; int bottomscale = 60; int vspace = height - topborder - bottomborder; int hspace = width - leftborder - rightborder; int maxtime = 600; - ymageChart chart = new ymageChart(width, height, "FFFFFF", "000000", leftborder, rightborder, topborder, bottomborder, "PEER PERFORMANCE GRAPH: PAGES/MINUTE and USED MEMORY"); + ymageChart chart = new ymageChart(width, height, "FFFFFF", "000000", "AAAAAA", leftborder, rightborder, topborder, bottomborder, "PEER PERFORMANCE GRAPH: PAGES/MINUTE and USED MEMORY", subline); chart.declareDimension(ymageChart.DIMENSION_BOTTOM, bottomscale, hspace / (maxtime / bottomscale), -maxtime, "000000", "CCCCCC", "TIME/SECONDS"); chart.declareDimension(ymageChart.DIMENSION_LEFT, leftscale, vspace * leftscale / maxppm, 0, "008800", null , "PPM [PAGES/MINUTE]"); chart.declareDimension(ymageChart.DIMENSION_RIGHT, rightscale, vspace * rightscale / (int)(maxbytes / 1024 / 1024), 0, "0000FF", "CCCCCC", "MEMORY/MEGABYTE"); diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 896fc1a3e..dad0699ca 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -1631,7 +1631,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch= getConfigLong(CRAWLSTACK_SLOTS, 2000)) { - log.logFine("deQueue: too many processes in stack crawl thread queue (" + "stackCrawlQueue=" + crawlStacker.size() + ")"); + if (this.log.isFine()) log.logFine("deQueue: too many processes in stack crawl thread queue (" + "stackCrawlQueue=" + crawlStacker.size() + ")"); return doneSomething; } // if we were interrupted we should return now if (Thread.currentThread().isInterrupted()) { - log.logFine("deQueue: thread was interrupted"); + if (this.log.isFine()) log.logFine("deQueue: thread was interrupted"); return false; } @@ -1932,7 +1932,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch 1000)) { - log.logFine("Cleaning Delegated-URLs report stack, " + crawlQueues.delegatedURL.stackSize() + " entries on stack"); + if (this.log.isFine()) log.logFine("Cleaning Delegated-URLs report stack, " + crawlQueues.delegatedURL.stackSize() + " entries on stack"); crawlQueues.delegatedURL.clearStack(); hasDoneSomething = true; } @@ -1940,7 +1940,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch 1000)) { - log.logFine("Cleaning Error-URLs report stack, " + crawlQueues.errorURL.stackSize() + " entries on stack"); + if (this.log.isFine()) log.logFine("Cleaning Error-URLs report stack, " + crawlQueues.errorURL.stackSize() + " entries on stack"); crawlQueues.errorURL.clearStack(); hasDoneSomething = true; } @@ -1949,7 +1949,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch 1000) { - log.logFine("Cleaning Loaded-URLs report stack, " + crawlResults.getStackSize(i) + " entries on stack " + i); + if (this.log.isFine()) log.logFine("Cleaning Loaded-URLs report stack, " + crawlResults.getStackSize(i) + " entries on stack " + i); crawlResults.clearStack(i); hasDoneSomething = true; } @@ -1962,7 +1962,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch 0) hasDoneSomething = true; } catch (IOException e) {} if (getConfigBool("cleanup.deletionProcessedNews", true)) { @@ -2134,7 +2134,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch (10000 * peerCount)) { dhtTransferIndexCount--; @@ -2595,11 +2595,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch= maxChunkFails) { //System.out.println("DEBUG: " + dhtTransferChunk.getTransferFailedCounter() + " of " + maxChunkFails + " sendings failed for this chunk, aborting!"); dhtTransferChunk.setStatus(plasmaDHTChunk.chunkStatus_FAILED); - log.logFine("DHT distribution: transfer FAILED"); + if (this.log.isFine()) log.logFine("DHT distribution: transfer FAILED"); } else { //System.out.println("DEBUG: " + dhtTransferChunk.getTransferFailedCounter() + " of " + maxChunkFails + " sendings failed for this chunk, retrying!"); - log.logFine("DHT distribution: transfer FAILED, sending this chunk again"); + if (this.log.isFine()) log.logFine("DHT distribution: transfer FAILED, sending this chunk again"); } return false; } diff --git a/source/de/anomic/plasma/plasmaWebStructure.java b/source/de/anomic/plasma/plasmaWebStructure.java index ab8a86be9..9916696ac 100644 --- a/source/de/anomic/plasma/plasmaWebStructure.java +++ b/source/de/anomic/plasma/plasmaWebStructure.java @@ -165,7 +165,7 @@ public class plasmaWebStructure { crg.insert(0, header.toString()); try { serverFileUtils.writeAndGZip(crg.toString().getBytes(), file); - log.logFine("wrote citation reference dump " + file.toString()); + if (this.log.isFine()) log.logFine("wrote citation reference dump " + file.toString()); } catch (IOException e) { e.printStackTrace(); } diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 7ddf187c5..035500414 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -484,6 +484,14 @@ public final class plasmaWordIndex implements indexRI { return java.lang.Math.max(collections.size(), java.lang.Math.max(dhtInCache.size(), dhtOutCache.size())); } + public int collectionsSize() { + return collections.size(); + } + + public int cacheSize() { + return dhtInCache.size() + dhtOutCache.size(); + } + public int indexSize(String wordHash) { int size = 0; size += dhtInCache.indexSize(wordHash); diff --git a/source/de/anomic/server/portForwarding/sch/serverPortForwardingSch.java b/source/de/anomic/server/portForwarding/sch/serverPortForwardingSch.java index f03e4c015..9f71e00ec 100644 --- a/source/de/anomic/server/portForwarding/sch/serverPortForwardingSch.java +++ b/source/de/anomic/server/portForwarding/sch/serverPortForwardingSch.java @@ -110,7 +110,7 @@ public class serverPortForwardingSch implements serverPortForwarding{ int localPort ) throws Exception { try { - this.log.logFine("Initializing port forwarding via sch ..."); + if (this.log.isFine()) this.log.logFine("Initializing port forwarding via sch ..."); this.switchboard = switchboard; @@ -205,7 +205,7 @@ public class serverPortForwardingSch implements serverPortForwarding{ public synchronized boolean reconnect() throws IOException { if ((!this.isConnected()) && (!Thread.currentThread().isInterrupted())) { - this.log.logFine("Trying to reconnect to port forwarding host."); + if (this.log.isFine()) this.log.logFine("Trying to reconnect to port forwarding host."); this.disconnect(); this.connect(); return this.isConnected(); diff --git a/source/de/anomic/server/serverAbstractBusyThread.java b/source/de/anomic/server/serverAbstractBusyThread.java index 6fd4e31ac..0158c0c7c 100644 --- a/source/de/anomic/server/serverAbstractBusyThread.java +++ b/source/de/anomic/server/serverAbstractBusyThread.java @@ -207,7 +207,7 @@ public abstract class serverAbstractBusyThread extends serverAbstractThread impl if (this.syncObject != null) { synchronized (this.syncObject) { if (this.log != null) - this.log.logFine("thread '" + this.getName() + if (this.log.isFine()) this.log.logFine("thread '" + this.getName() + "' has received a notification from thread '" + Thread.currentThread().getName() + "'."); this.syncObject.notifyAll(); diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index e0d1e3687..902c83649 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -300,7 +300,7 @@ public final class serverCore extends serverAbstractBusyThread implements server if (bindIP.startsWith("#")) { String interfaceName = bindIP.substring(1); String hostName = null; - this.log.logFine("Trying to determine IP address of interface '" + interfaceName + "'."); + if (this.log.isFine()) this.log.logFine("Trying to determine IP address of interface '" + interfaceName + "'."); Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); if (interfaces != null) { @@ -402,7 +402,7 @@ public final class serverCore extends serverAbstractBusyThread implements server // prepare for new connection // idleThreadCheck(); this.switchboard.handleBusyState(this.busySessions.size()); - this.log.logFinest("* waiting for connections, " + this.busySessions.size() + " sessions running"); + if (log.isFinest()) this.log.logFinest("* waiting for connections, " + this.busySessions.size() + " sessions running"); announceThreadBlockApply(); @@ -638,7 +638,7 @@ public final class serverCore extends serverAbstractBusyThread implements server */ public void log(boolean outgoing, String request) { - serverCore.this.log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " + + if (log.isFine()) log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " + "[" + ((busySessions == null)? -1 : busySessions.size()) + ", " + this.commandCounter + ((outgoing) ? "] > " : "] < ") + request); diff --git a/source/de/anomic/server/serverMemory.java b/source/de/anomic/server/serverMemory.java index c7a535572..d9e84b717 100644 --- a/source/de/anomic/server/serverMemory.java +++ b/source/de/anomic/server/serverMemory.java @@ -54,7 +54,7 @@ public class serverMemory { lastGC = System.currentTimeMillis(); if (log.isFine()) log.logInfo("[gc] before: " + bytesToString(free) + ", after: " + bytesToString(free()) + ", call: " + info); } else if (log.isFine()) { - log.logFinest("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info); + if (log.isFinest()) log.logFinest("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info); } } diff --git a/source/de/anomic/ymage/ymageChart.java b/source/de/anomic/ymage/ymageChart.java index a4d1ca8a2..cecab46d5 100644 --- a/source/de/anomic/ymage/ymageChart.java +++ b/source/de/anomic/ymage/ymageChart.java @@ -67,9 +67,9 @@ public class ymageChart extends ymageMatrix { String name; String backgroundColor, foregroundColor; - public ymageChart(int width, int height, String backgroundColor, String foregroundColor, + public ymageChart(int width, int height, String backgroundColor, String foregroundColor, String lightColor, int leftborder, int rightborder, int topborder, int bottomborder, - String name) { + String name, String subline) { super(width, height, ymageMatrix.MODE_REPLACE, backgroundColor); this.leftborder = leftborder; this.rightborder = rightborder; @@ -82,6 +82,10 @@ public class ymageChart extends ymageMatrix { this.setColor(foregroundColor); ymageToolPrint.print(this, width / 2 - name.length() * 3, 6, 0, name, -1); } + if (subline != null) { + this.setColor(lightColor); + ymageToolPrint.print(this, width / 2 - subline.length() * 3, 14, 0, subline, -1); + } } public void declareDimension(int dimensionType, int scale, int pixelperscale, int offset, String colorNaming, String colorScale, String name) { @@ -166,7 +170,7 @@ public class ymageChart extends ymageMatrix { String scale = (invers) ? "333333" : "CCCCCC"; String green = (invers) ? "008800" : "008800"; String blue = (invers) ? "0000FF" : "0000FF"; - ymageChart ip = new ymageChart(660, 240, bg, fg, 30, 30, 20, 20, "PEER PERFORMANCE GRAPH: PAGES/MINUTE and USED MEMORY"); + ymageChart ip = new ymageChart(660, 240, bg, fg, fg, 30, 30, 20, 20, "PEER PERFORMANCE GRAPH: PAGES/MINUTE and USED MEMORY", ""); ip.declareDimension(DIMENSION_BOTTOM, 60, 60, -600, fg, scale, "TIME/SECONDS"); //ip.declareDimension(DIMENSION_TOP, 10, 40, "000000", null, "count"); ip.declareDimension(DIMENSION_LEFT, 50, 40, 0, green, scale , "PPM [PAGES/MINUTE]");