From 8281e29963e1aebbad419de5b240cd4cd5fb023d Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 8 Dec 2009 14:25:51 +0000 Subject: [PATCH] - more configuration for profiling graph (number of events) - more logging for a shutdown: print reason and accessing IP into log git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6520 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/ConfigUpdate_p.java | 2 +- htroot/PerformanceSearch_p.java | 18 ++- htroot/Steering.java | 2 +- htroot/yacy/search.java | 8 +- htroot/yacysearch.java | 4 +- htroot/yacysearchitem.java | 4 +- htroot/yacysearchtrailer.java | 4 +- source/de/anomic/search/DocumentIndex.java | 8 +- source/de/anomic/search/RankingProcess.java | 8 +- source/de/anomic/search/ResultFetcher.java | 7 +- source/de/anomic/search/SearchEvent.java | 8 +- source/de/anomic/search/Switchboard.java | 24 ++-- source/de/anomic/server/serverCore.java | 16 +-- source/de/anomic/server/serverSwitch.java | 1 - source/de/anomic/yacy/Tray.java | 2 +- .../anomic/yacy/graphics/ProfilingGraph.java | 99 ++++++++------ .../yacy/graphics/WebStructureGraph.java | 3 +- source/de/anomic/yacy/yacyRelease.java | 6 +- .../net/yacy/document/parser/xlsParser.java | 1 - source/net/yacy/kelondro/rwi/IndexCell.java | 8 +- .../net/yacy/kelondro/util/EventTracker.java | 123 ++++++++++++++++++ .../workflow/AbstractBlockingThread.java | 3 +- .../kelondro/workflow/AbstractBusyThread.java | 11 +- .../kelondro/workflow/AbstractThread.java | 8 +- .../kelondro/workflow/InstantBusyThread.java | 1 - .../kelondro/workflow/WorkflowThread.java | 5 - source/net/yacy/yacy.java | 8 +- 27 files changed, 260 insertions(+), 132 deletions(-) create mode 100644 source/net/yacy/kelondro/util/EventTracker.java diff --git a/htroot/ConfigUpdate_p.java b/htroot/ConfigUpdate_p.java index c63a43d6c..cf42e8bcc 100644 --- a/htroot/ConfigUpdate_p.java +++ b/htroot/ConfigUpdate_p.java @@ -132,7 +132,7 @@ public class ConfigUpdate_p { prop.put("candeploy_autoUpdate", "4"); } else { yacyRelease.deployRelease(downloaded); - sb.terminate(5000); + sb.terminate(5000, "manual release update to " + downloaded.getName()); sb.getLog().logInfo("AUTO-UPDATE: deploy and restart initiated"); prop.put("candeploy_autoUpdate", "1"); } diff --git a/htroot/PerformanceSearch_p.java b/htroot/PerformanceSearch_p.java index 543563b55..726144c86 100644 --- a/htroot/PerformanceSearch_p.java +++ b/htroot/PerformanceSearch_p.java @@ -24,11 +24,10 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.util.ArrayList; import java.util.Date; import java.util.Iterator; -import net.yacy.kelondro.util.MemoryTracker; +import net.yacy.kelondro.util.EventTracker; import de.anomic.http.server.RequestHeader; import de.anomic.server.serverObjects; @@ -41,15 +40,14 @@ public class PerformanceSearch_p { // return variable that accumulates replacements final serverObjects prop = new serverObjects(); - final ArrayList events = MemoryTracker.history("SEARCH"); + final Iterator events = EventTracker.getHistory("SEARCH"); int c = 0; - MemoryTracker.Event event; - ProfilingGraph.searchEvent search; - long lastt = 0; - if (events != null) synchronized (events) { - Iterator i = events.iterator(); - while (i.hasNext()) { - event = i.next(); + if (events != null) { + EventTracker.Event event; + ProfilingGraph.searchEvent search; + long lastt = 0; + while (events.hasNext()) { + event = events.next(); search = (ProfilingGraph.searchEvent) event.payload; prop.put("table_" + c + "_query", search.queryID); prop.put("table_" + c + "_event", search.processName); diff --git a/htroot/Steering.java b/htroot/Steering.java index a2f011696..9decbf0ea 100644 --- a/htroot/Steering.java +++ b/htroot/Steering.java @@ -59,7 +59,7 @@ public class Steering { if (post.containsKey("shutdown")) { Log.logInfo("STEERING", "shutdown request from " + requestIP); - sb.terminate(100); + sb.terminate(100, "shutdown request from Steering; ip = " + requestIP); prop.put("info", "3"); return prop; diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 4e2ad6eb4..2ae4637e3 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -41,7 +41,7 @@ import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Bitfield; import net.yacy.kelondro.rwi.ReferenceContainer; -import net.yacy.kelondro.util.MemoryTracker; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.SortStack; import net.yacy.kelondro.util.ISO639; @@ -225,7 +225,7 @@ public final class search { //final Map>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls)); final HashMap> incc = indexSegment.termIndex().searchConjunction(theQuery.queryHashes, QueryParams.hashes2StringSet(urls)); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.COLLECTION, incc.size(), System.currentTimeMillis() - timer), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.COLLECTION, incc.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime); if (incc != null) { final Iterator>> ci = incc.entrySet().iterator(); Map.Entry> entry; @@ -342,7 +342,7 @@ public final class search { refstr.append(",").append(e.name); } prop.put("references", (refstr.length() > 0) ? refstr.substring(1) : refstr.toString()); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime); } prop.put("indexabstract", indexabstract.toString()); @@ -369,7 +369,7 @@ public final class search { } prop.put("links", links.toString()); prop.put("linkcount", accu.size()); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime); } // add information about forward peers diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index e98ab2345..6322f3efb 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -44,9 +44,9 @@ import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Bitfield; import net.yacy.kelondro.util.Domains; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.MemoryControl; -import net.yacy.kelondro.util.MemoryTracker; import net.yacy.kelondro.util.SetTools; import net.yacy.kelondro.util.ISO639; import net.yacy.repository.LoaderDispatcher; @@ -461,7 +461,7 @@ public class yacysearch { authenticated, indexSegment, ranking); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.INITIALIZATION, 0, 0), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.INITIALIZATION, 0, 0), false, 30000, ProfilingGraph.maxTime); // tell all threads to do nothing for a specific time sb.intermissionAllThreads(10000); diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 0afb06ac7..b693cb2d0 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -31,8 +31,8 @@ import java.util.ArrayList; import java.util.TreeSet; import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.Formatter; -import net.yacy.kelondro.util.MemoryTracker; import de.anomic.http.server.HeaderFramework; import de.anomic.http.server.RequestHeader; @@ -153,7 +153,7 @@ public class yacysearchitem { prop.put("content_description", desc); prop.putXML("content_description-xml", desc); prop.putJSON("content_description-json", desc); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + item, 0, 0), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + item, 0, 0), false, 30000, ProfilingGraph.maxTime); return prop; } diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index b1bc066e6..b6b532adf 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.Iterator; -import net.yacy.kelondro.util.MemoryTracker; +import net.yacy.kelondro.util.EventTracker; import de.anomic.http.server.RequestHeader; import de.anomic.search.QueryParams; @@ -146,7 +146,7 @@ public class yacysearchtrailer { prop.put("nav-about_body", aboutBody); } - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + "bottomline", 0, 0), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + "bottomline", 0, 0), false, 30000, ProfilingGraph.maxTime); return prop; } diff --git a/source/de/anomic/search/DocumentIndex.java b/source/de/anomic/search/DocumentIndex.java index 1eb92d141..480bbd87e 100644 --- a/source/de/anomic/search/DocumentIndex.java +++ b/source/de/anomic/search/DocumentIndex.java @@ -53,12 +53,12 @@ public class DocumentIndex extends Segment { private static final RankingProfile textRankingDefault = new RankingProfile(ContentDomain.TEXT); //private Bitfield zeroConstraint = new Bitfield(4); - private final static File poison = new File("."); - private BlockingQueue queue; + final static File poison = new File("."); + BlockingQueue queue; private Worker[] worker; - private CallbackListener callback; + CallbackListener callback; - private static final ThreadGroup workerThreadGroup = new ThreadGroup("workerThreadGroup"); + static final ThreadGroup workerThreadGroup = new ThreadGroup("workerThreadGroup"); public DocumentIndex(Log log, final File segmentPath, CallbackListener callback, int cachesize) throws IOException { super(log, segmentPath, cachesize, targetFileSize * 4 - 1, false, false); diff --git a/source/de/anomic/search/RankingProcess.java b/source/de/anomic/search/RankingProcess.java index 93116d688..116f87dbc 100644 --- a/source/de/anomic/search/RankingProcess.java +++ b/source/de/anomic/search/RankingProcess.java @@ -52,8 +52,8 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.TermSearch; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.FileUtils; -import net.yacy.kelondro.util.MemoryTracker; import net.yacy.kelondro.util.SortStack; import de.anomic.yacy.graphics.ProfilingGraph; @@ -135,7 +135,7 @@ public final class RankingProcess extends Thread { query.maxDistance); this.localSearchInclusion = search.inclusion(); final ReferenceContainer index = search.joined(); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.JOIN, index.size(), System.currentTimeMillis() - timer), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.JOIN, index.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime); if (index.isEmpty()) { return; } @@ -164,7 +164,7 @@ public final class RankingProcess extends Thread { // normalize entries final BlockingQueue decodedEntries = this.order.normalizeWith(index); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.NORMALIZING, index.size(), System.currentTimeMillis() - timer), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.NORMALIZING, index.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime); // iterate over normalized entries and select some that are better than currently stored timer = System.currentTimeMillis(); @@ -259,7 +259,7 @@ public final class RankingProcess extends Thread { } //if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime); } /** diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index 15f2af0c4..74608aa7d 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -39,7 +39,7 @@ import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.MemoryTracker; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.SetTools; import net.yacy.kelondro.util.SortStack; import net.yacy.kelondro.util.SortStore; @@ -95,8 +95,7 @@ public class ResultFetcher { // start worker threads to fetch urls and snippets this.workerThreads = null; deployWorker(Math.min(10, query.itemsPerPage), query.neededResults()); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), this.workerThreads.length + " online snippet fetch threads started", 0, 0), false); - + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), this.workerThreads.length + " online snippet fetch threads started", 0, 0), false, 30000, ProfilingGraph.maxTime); } public void deployWorker(int deployCount, int neededResults) { @@ -281,7 +280,7 @@ public class ResultFetcher { public ResultEntry oneResult(final int item) { // check if we already retrieved this item // (happens if a search pages is accessed a second time) - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "obtain one result entry - start", 0, 0), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "obtain one result entry - start", 0, 0), false, 30000, ProfilingGraph.maxTime); if (this.result.sizeStore() > item) { // we have the wanted result already in the result array .. return that return this.result.element(item).element; diff --git a/source/de/anomic/search/SearchEvent.java b/source/de/anomic/search/SearchEvent.java index 97504dc10..62fc261da 100644 --- a/source/de/anomic/search/SearchEvent.java +++ b/source/de/anomic/search/SearchEvent.java @@ -38,8 +38,8 @@ import net.yacy.kelondro.data.word.WordReferenceVars; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.rwi.ReferenceContainer; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.MemoryControl; -import net.yacy.kelondro.util.MemoryTracker; import net.yacy.kelondro.util.SetTools; import de.anomic.crawler.ResultURLs; @@ -139,7 +139,7 @@ public final class SearchEvent { (query.domType == QueryParams.SEARCHDOM_GLOBALDHT) ? null : preselectedPeerHashes); if (this.primarySearchThreads != null) { if (this.primarySearchThreads.length > fetchpeers) this.rankedCache.moreFeeders(this.primarySearchThreads.length - fetchpeers); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "remote search thread start", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "remote search thread start", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime); // finished searching Log.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); } else { @@ -179,7 +179,7 @@ public final class SearchEvent { IACount.put(wordhash, Integer.valueOf(container.size())); IAResults.put(wordhash, ReferenceContainer.compressIndex(container, null, 1000).toString()); } - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "abstract generation", this.rankedCache.searchContainerMap().size(), System.currentTimeMillis() - timer), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "abstract generation", this.rankedCache.searchContainerMap().size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime); } // start worker threads to fetch urls and snippets @@ -188,7 +188,7 @@ public final class SearchEvent { // clean up events SearchEventCache.cleanupEvents(false); - MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "event-cleanup", 0, 0), false); + EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "event-cleanup", 0, 0), false, 30000, ProfilingGraph.maxTime); // store this search to a cache so it can be re-used if (MemoryControl.available() < 1024 * 1024 * 10) SearchEventCache.cleanupEvents(true); diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index e2a5d5dd5..213d2f3c0 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -128,6 +128,7 @@ import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.Domains; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryTracker; @@ -193,6 +194,7 @@ import de.anomic.yacy.yacyUpdateLocation; import de.anomic.yacy.yacyRelease; import de.anomic.yacy.dht.Dispatcher; import de.anomic.yacy.dht.PeerSelection; +import de.anomic.yacy.graphics.ProfilingGraph; import de.anomic.yacy.graphics.WebStructureGraph; public final class Switchboard extends serverSwitch { @@ -1486,7 +1488,7 @@ public final class Switchboard extends serverSwitch { log.logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found, is too small or signature is bad)"); } else { yacyRelease.deployRelease(downloaded); - terminate(5000); + terminate(5000, "auto-update to install " + downloaded.getName()); log.logInfo("AUTO-UPDATE: deploy and restart initiated"); } } @@ -1765,10 +1767,10 @@ public final class Switchboard extends serverSwitch { if (System.currentTimeMillis() - lastPPMUpdate > 20000) { // we don't want to do this too often updateMySeed(); - MemoryTracker.update("ppm", Long.valueOf(currentPPM()), true); + EventTracker.update("ppm", Long.valueOf(currentPPM()), true, 30000, ProfilingGraph.maxTime); lastPPMUpdate = System.currentTimeMillis(); } - MemoryTracker.update("indexed", queueEntry.url().toNormalform(true, false), false); + EventTracker.update("indexed", queueEntry.url().toNormalform(true, false), false, 30000, ProfilingGraph.maxTime); // if this was performed for a remote crawl request, notify requester if ((processCase == EventOrigin.GLOBAL_CRAWLING) && (queueEntry.initiator() != null)) { @@ -2005,7 +2007,7 @@ public final class Switchboard extends serverSwitch { } public int currentPPM() { - return MemoryTracker.countEvents("indexed", 20000) * 3; + return EventTracker.countEvents("indexed", 20000) * 3; } public String makeDefaultPeerName() { @@ -2127,13 +2129,15 @@ public final class Switchboard extends serverSwitch { else if (this.terminate || curThread.isInterrupted()) throw new InterruptedException("Shutdown in progress ..."); } - public void terminate(final long delay) { + public void terminate(final long delay, String reason) { if (delay <= 0) throw new IllegalArgumentException("The shutdown delay must be greater than 0."); - (new delayedShutdown(this,delay)).start(); + log.logInfo("caught delayed terminate request: " + reason); + (new delayedShutdown(this, delay, reason)).start(); } - public void terminate() { + public void terminate(String reason) { this.terminate = true; + log.logInfo("caught terminate request: " + reason); this.shutdownSync.release(); } @@ -2171,9 +2175,11 @@ public final class Switchboard extends serverSwitch { class delayedShutdown extends Thread { private final Switchboard sb; private final long delay; - public delayedShutdown(final Switchboard sb, final long delay) { + private final String reason; + public delayedShutdown(final Switchboard sb, final long delay, final String reason) { this.sb = sb; this.delay = delay; + this.reason = reason; } public void run() { @@ -2184,6 +2190,6 @@ class delayedShutdown extends Thread { } catch (final Exception e) { Log.logException(e); } - this.sb.terminate(); + this.sb.terminate(reason); } } diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index c8484b53c..f2bed9440 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -93,13 +93,13 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { public static final String LF_STRING = new String(new byte[]{LF}); public static final Class[] stringType = {"".getClass()}; // set up some reflection public static final long startupTime = System.currentTimeMillis(); - private static final ThreadGroup sessionThreadGroup = new ThreadGroup("sessionThreadGroup"); - private static final HashMap commandObjMethodCache = new HashMap(5); + static final ThreadGroup sessionThreadGroup = new ThreadGroup("sessionThreadGroup"); + static final HashMap commandObjMethodCache = new HashMap(5); /** * will be increased with each session and is used to return a hash code */ - private static int sessionCounter = 0; + static int sessionCounter = 0; // static variables private static final long keepAliveTimeout = 60000; // time that a connection is kept alive if requested with a keepAlive statement @@ -123,18 +123,18 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { /** * specifies if the server should try to do a restart */ - private boolean forceRestart = false; + boolean forceRestart = false; public static boolean useStaticIP = false; - + protected Log log; private SSLSocketFactory sslSocketFactory = null; private ServerSocket socket; // listener private final int timeout; // connection time-out of the socket - private serverHandler handlerPrototype; // the command class (a serverHandler) + serverHandler handlerPrototype; // the command class (a serverHandler) private final serverSwitch switchboard; // the command class switchboard - private HashMap denyHost; - private int commandMaxLength; + HashMap denyHost; + int commandMaxLength; private int maxBusySessions; private long lastAutoTermination; diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index 1d8a7ef3f..d81eeb109 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -380,7 +380,6 @@ public class serverSwitch { newThread.setMemPreReqisite(initialMemoryPreRequisite); setConfig(threadName + "_memprereq", initialMemoryPreRequisite); } - newThread.setLog(log); newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL); workerThreads.put(threadName, newThread); // start the thread diff --git a/source/de/anomic/yacy/Tray.java b/source/de/anomic/yacy/Tray.java index 15c5b06f7..adc75f50b 100644 --- a/source/de/anomic/yacy/Tray.java +++ b/source/de/anomic/yacy/Tray.java @@ -136,7 +136,7 @@ public final class Tray { menuItem = new MenuItem(label); menuItem.addActionListener(new ActionListener() { public void actionPerformed(final ActionEvent e) { - sb.terminate(); + sb.terminate("shutdown from tray"); } }); menu.add(menuItem); diff --git a/source/de/anomic/yacy/graphics/ProfilingGraph.java b/source/de/anomic/yacy/graphics/ProfilingGraph.java index 9bb566316..9e9ddfdea 100644 --- a/source/de/anomic/yacy/graphics/ProfilingGraph.java +++ b/source/de/anomic/yacy/graphics/ProfilingGraph.java @@ -26,26 +26,27 @@ package de.anomic.yacy.graphics; -import java.util.ArrayList; import java.util.ConcurrentModificationException; +import java.util.Iterator; - -import net.yacy.kelondro.util.MemoryTracker; -import net.yacy.kelondro.util.MemoryTracker.Event; +import net.yacy.kelondro.util.EventTracker; +import net.yacy.kelondro.util.EventTracker.Event; import net.yacy.visualization.RasterPlotter; import net.yacy.visualization.ChartPlotter; - public class ProfilingGraph { private static ChartPlotter bufferChart = null; + public static long maxTime = 600000L; public static long maxPayload(final String eventname, final long min) { - final ArrayList list = MemoryTracker.history(eventname); + final Iterator list = EventTracker.getHistory(eventname); if (list == null) return min; long max = min, l; synchronized (list) { - for (MemoryTracker.Event event: list) { + EventTracker.Event event; + while (list.hasNext()) { + event = list.next(); l = ((Long) event.payload).longValue(); if (l > max) max = l; } @@ -102,49 +103,61 @@ public class ProfilingGraph { } */ // draw memory - ArrayList events = MemoryTracker.history("memory"); + Iterator events = EventTracker.getHistory("memory"); x0 = 1; y0 = 0; - if (events != null) synchronized (events) {for (MemoryTracker.Event event: events) { - time = event.time - now; - bytes = ((Long) event.payload).longValue(); - x1 = (int) (time/1000); - y1 = (int) (bytes / 1024 / 1024); - chart.setColor("AAAAFF"); - chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x1, y1, 2, null, 0); - chart.setColor("0000FF"); - if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x0, y0, x1, y1); - x0 = x1; y0 = y1; - }} + if (events != null) { + EventTracker.Event event; + while (events.hasNext()) { + event = events.next(); + time = event.time - now; + bytes = ((Long) event.payload).longValue(); + x1 = (int) (time/1000); + y1 = (int) (bytes / 1024 / 1024); + chart.setColor("AAAAFF"); + chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x1, y1, 2, null, 0); + chart.setColor("0000FF"); + if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x0, y0, x1, y1); + x0 = x1; y0 = y1; + } + } // draw wordcache - events = MemoryTracker.history("wordcache"); + events = EventTracker.getHistory("wordcache"); x0 = 1; y0 = 0; - if (events != null) synchronized (events) {for (MemoryTracker.Event event: events) { - time = event.time - now; - words = (int) ((Long) event.payload).longValue(); - x1 = (int) (time/1000); - y1 = words; - chart.setColor("228822"); - chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, x1, y1, 2, null, 315); - chart.setColor("008800"); - if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, x0, y0, x1, y1); - x0 = x1; y0 = y1; - }} + if (events != null) { + EventTracker.Event event; + while (events.hasNext()) { + event = events.next(); + time = event.time - now; + words = (int) ((Long) event.payload).longValue(); + x1 = (int) (time/1000); + y1 = words; + chart.setColor("228822"); + chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, x1, y1, 2, null, 315); + chart.setColor("008800"); + if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, x0, y0, x1, y1); + x0 = x1; y0 = y1; + } + } // draw ppm - events = MemoryTracker.history("ppm"); + events = EventTracker.getHistory("ppm"); x0 = 1; y0 = 0; - if (events != null) synchronized (events) {for (MemoryTracker.Event event: events) { - time = event.time - now; - ppm = (int) ((Long) event.payload).longValue(); - x1 = (int) (time/1000); - y1 = ppm; - chart.setColor("AA8888"); - if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT0, x0, y0, x1, y1); - chart.setColor("AA2222"); - chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT0, x1, y1, 2, ppm + " PPM", 0); - x0 = x1; y0 = y1; - }} + if (events != null) { + EventTracker.Event event; + while (events.hasNext()) { + event = events.next(); + time = event.time - now; + ppm = (int) ((Long) event.payload).longValue(); + x1 = (int) (time/1000); + y1 = ppm; + chart.setColor("AA8888"); + if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT0, x0, y0, x1, y1); + chart.setColor("AA2222"); + chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT0, x1, y1, 2, ppm + " PPM", 0); + x0 = x1; y0 = y1; + } + } bufferChart = chart; } catch (final ConcurrentModificationException cme) { diff --git a/source/de/anomic/yacy/graphics/WebStructureGraph.java b/source/de/anomic/yacy/graphics/WebStructureGraph.java index aa3d9d2e4..bebbf6fad 100644 --- a/source/de/anomic/yacy/graphics/WebStructureGraph.java +++ b/source/de/anomic/yacy/graphics/WebStructureGraph.java @@ -58,7 +58,8 @@ public class WebStructureGraph { private final Log log; private final File rankingPath, structureFile; private final String crlFile, crgFile; - private TreeMap structure_old, structure_new; // ',' to {}* + TreeMap structure_old; // ',' to {}* + TreeMap structure_new; public WebStructureGraph(final Log log, final File rankingPath, final String crlFile, final String crgFile, final File structureFile) { this.log = log; diff --git a/source/de/anomic/yacy/yacyRelease.java b/source/de/anomic/yacy/yacyRelease.java index 77c051a72..fb5c4f5df 100644 --- a/source/de/anomic/yacy/yacyRelease.java +++ b/source/de/anomic/yacy/yacyRelease.java @@ -416,7 +416,7 @@ public final class yacyRelease extends yacyVersion { Log.logInfo("RESTART", "wrote restart-script to " + scriptFile.getAbsolutePath()); OS.execAsynchronous(scriptFile); Log.logInfo("RESTART", "script is running"); - sb.terminate(5000); + sb.terminate(5000, "windows restart"); } catch (final IOException e) { Log.logSevere("RESTART", "restart failed", e); } @@ -466,7 +466,7 @@ public final class yacyRelease extends yacyVersion { Log.logInfo("RESTART", "wrote restart-script to " + scriptFile.getAbsolutePath()); OS.execAsynchronous(scriptFile); Log.logInfo("RESTART", "script is running"); - sb.terminate(5000); + sb.terminate(5000, "unix restart"); } catch (final IOException e) { Log.logSevere("RESTART", "restart failed", e); } @@ -567,7 +567,7 @@ public final class yacyRelease extends yacyVersion { OS.execAsynchronous(scriptFile); Log.logInfo("UPDATE", "script is running"); sb.setConfig("update.time.deploy", System.currentTimeMillis()); - sb.terminate(5000); + sb.terminate(5000, "auto-deploy for " + releaseFile.getName()); } catch (final IOException e) { Log.logSevere("UPDATE", "update failed", e); } diff --git a/source/net/yacy/document/parser/xlsParser.java b/source/net/yacy/document/parser/xlsParser.java index b82db1ea1..007d39522 100644 --- a/source/net/yacy/document/parser/xlsParser.java +++ b/source/net/yacy/document/parser/xlsParser.java @@ -160,7 +160,6 @@ public class xlsParser extends AbstractParser implements Idiom { */ Log.logException(e); final String errorMsg = "Unable to parse the xls document '" + location + "':" + e.getMessage(); - theLogger.logSevere(errorMsg); throw new ParserException(errorMsg, location); } } diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index cc11b24c1..21e24332e 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -31,6 +31,8 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import de.anomic.yacy.graphics.ProfilingGraph; + import net.yacy.kelondro.index.ARC; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.SimpleARC; @@ -39,8 +41,8 @@ import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.MergeIterator; import net.yacy.kelondro.order.Order; import net.yacy.kelondro.util.ByteArray; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.MemoryControl; -import net.yacy.kelondro.util.MemoryTracker; /* @@ -108,7 +110,7 @@ public final class IndexCell extends AbstractBu public void add(ReferenceContainer newEntries) throws IOException { this.ram.add(newEntries); if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) { - MemoryTracker.update("wordcache", Long.valueOf(this.ram.size()), true); + EventTracker.update("wordcache", Long.valueOf(this.ram.size()), true, 30000, ProfilingGraph.maxTime); cleanCache(); } } @@ -116,7 +118,7 @@ public final class IndexCell extends AbstractBu public void add(byte[] termHash, ReferenceType entry) throws IOException { this.ram.add(termHash, entry); if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) { - MemoryTracker.update("wordcache", Long.valueOf(this.ram.size()), true); + EventTracker.update("wordcache", Long.valueOf(this.ram.size()), true, 30000, ProfilingGraph.maxTime); cleanCache(); } } diff --git a/source/net/yacy/kelondro/util/EventTracker.java b/source/net/yacy/kelondro/util/EventTracker.java new file mode 100644 index 000000000..b075db9bb --- /dev/null +++ b/source/net/yacy/kelondro/util/EventTracker.java @@ -0,0 +1,123 @@ +// EventTracker.java +// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 17.11.2007 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2009-11-05 21:28:37 +0100 (Do, 05 Nov 2009) $ +// $LastChangedRevision: 6458 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.kelondro.util; + +import java.util.Iterator; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class EventTracker { + + private final static Map> historyMaps = new ConcurrentHashMap>(); + private final static Map eventAccess = new ConcurrentHashMap(); // value: last time when this was accessed + + public final static void update( + final String eventName, + final Object eventPayload, + boolean useProtection, + int maxQueueSize, + long maxQueueAge) { + // check protection against too heavy access + if (useProtection) { + Long lastAcc = eventAccess.get(eventName); + if (lastAcc == null) { + eventAccess.put(eventName, Long.valueOf(System.currentTimeMillis())); + } else { + long time = System.currentTimeMillis(); + if (time - lastAcc.longValue() < 1000) { + return; // protect against too heavy load + } + eventAccess.put(eventName, Long.valueOf(time)); + } + } + + // get event history container + Queue history = historyMaps.get(eventName); + + // create history + if (history == null) { + history = new ConcurrentLinkedQueue(); + + // update entry + history.offer(new Event(eventPayload)); + + // store map + historyMaps.put(eventName, history); + return; + } + + // update history + synchronized (history) { + + // update entry + history.offer(new Event(eventPayload)); + + // clean up too old entries + int tp = history.size() - maxQueueSize; + while (tp-- > 0) history.poll(); + if (history.size() % 10 == 0) { // reduce number of System.currentTimeMillis() calls + Event e; + final long now = System.currentTimeMillis(); + while (history.size() > 0) { + e = history.peek(); + if (now - e.time < maxQueueAge) break; + history.poll(); + } + } + } + } + + public final static Iterator getHistory(final String eventName) { + Queue list = historyMaps.get(eventName); + if (list == null) return null; + return list.iterator(); + } + + public final static int countEvents(final String eventName, long time) { + Iterator event = getHistory(eventName); + if (event == null) return 0; + long now = System.currentTimeMillis(); + int count = 0; + while (event.hasNext()) { + if (now - event.next().time < time) count++; + } + return count; + } + + public final static class Event { + public Object payload; + public long time; + + public Event(final Object payload) { + this.payload = payload; + this.time = System.currentTimeMillis(); + } + } + +} diff --git a/source/net/yacy/kelondro/workflow/AbstractBlockingThread.java b/source/net/yacy/kelondro/workflow/AbstractBlockingThread.java index 167e4ac14..718f55d1c 100644 --- a/source/net/yacy/kelondro/workflow/AbstractBlockingThread.java +++ b/source/net/yacy/kelondro/workflow/AbstractBlockingThread.java @@ -30,7 +30,8 @@ import net.yacy.kelondro.util.MemoryControl; public abstract class AbstractBlockingThread extends AbstractThread implements BlockingThread { private WorkflowProcessor manager = null; - + private final static Log log = new Log("BlockingThread"); + public void setManager(final WorkflowProcessor manager) { this.manager = manager; } diff --git a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java index d16a2d6cb..3512e22be 100644 --- a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java +++ b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java @@ -33,6 +33,7 @@ import net.yacy.kelondro.util.MemoryControl; public abstract class AbstractBusyThread extends AbstractThread implements BusyThread { + private final static Log log = new Log("BusyThread"); private long startup = 0, intermission = 0, idlePause = 0, busyPause = 0; private long idletime = 0, memprereq = 0; private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0; @@ -167,7 +168,7 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT idletime += System.currentTimeMillis() - timestamp; } catch (final SocketException e) { // in case that a socket is interrupted, this method must die silently (shutdown) - this.log.logFine("socket-job interrupted: " + e.getMessage()); + log.logFine("socket-job interrupted: " + e.getMessage()); } catch (final Exception e) { // handle exceptions: thread must not die on any unexpected exceptions // if the exception is too bad it should call terminate() @@ -201,16 +202,16 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT Thread.sleep(millis); //} } catch (final InterruptedException e) { - if (this.log != null) - this.log.logConfig("thread '" + this.getName() + "' interrupted because of shutdown."); + if (log != null) + log.logConfig("thread '" + this.getName() + "' interrupted because of shutdown."); } } public void notifyThread() { if (this.syncObject != null) { synchronized (this.syncObject) { - if (this.log != null) - if (this.log.isFine()) this.log.logFine("thread '" + this.getName() + if (log != null) + if (log.isFine()) log.logFine("thread '" + this.getName() + "' has received a notification from thread '" + Thread.currentThread().getName() + "'."); this.syncObject.notifyAll(); diff --git a/source/net/yacy/kelondro/workflow/AbstractThread.java b/source/net/yacy/kelondro/workflow/AbstractThread.java index 78a6556d5..86cf9a77d 100644 --- a/source/net/yacy/kelondro/workflow/AbstractThread.java +++ b/source/net/yacy/kelondro/workflow/AbstractThread.java @@ -39,8 +39,8 @@ import net.yacy.kelondro.logging.Log; public abstract class AbstractThread extends Thread implements WorkflowThread { + private static Log log = new Log("WorkflowThread"); protected boolean running = true; - protected Log log = null; protected long busytime = 0, memuse = 0; private long blockPause = 0; private String shortDescr = "", longDescr = ""; @@ -100,12 +100,6 @@ public abstract class AbstractThread extends Thread implements WorkflowThread { // returns the sum of all memory usage differences before and after one busy job return memuse; } - - public final void setLog(final Log log) { - // defines a log where process states can be written to - this.log = log; - } - public boolean shutdownInProgress() { return !this.running || Thread.currentThread().isInterrupted(); diff --git a/source/net/yacy/kelondro/workflow/InstantBusyThread.java b/source/net/yacy/kelondro/workflow/InstantBusyThread.java index ba9919b98..f6532b1a9 100644 --- a/source/net/yacy/kelondro/workflow/InstantBusyThread.java +++ b/source/net/yacy/kelondro/workflow/InstantBusyThread.java @@ -151,7 +151,6 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT thread.setIdleSleep(-1); thread.setBusySleep(-1); thread.setMemPreReqisite(0); - thread.setLog(log); thread.start(); return thread; } diff --git a/source/net/yacy/kelondro/workflow/WorkflowThread.java b/source/net/yacy/kelondro/workflow/WorkflowThread.java index c6cc32e8b..20a1990e3 100644 --- a/source/net/yacy/kelondro/workflow/WorkflowThread.java +++ b/source/net/yacy/kelondro/workflow/WorkflowThread.java @@ -24,8 +24,6 @@ package net.yacy.kelondro.workflow; -import net.yacy.kelondro.logging.Log; - public interface WorkflowThread { // ------------------------------------------------------- @@ -58,9 +56,6 @@ public interface WorkflowThread { public long getMemoryUse(); // returns the sum of all memory usage differences before and after one busy job - public void setLog(Log log); - // defines a log where process states can be written to - public void jobExceptionHandler(Exception e); // handles any action necessary during job execution diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index c4bfacf2e..1588801bb 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -316,8 +316,6 @@ public final class yacy { // first start the server sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0); //server.start(); - // repair log settings, gets overwritten with "PLASMA" in deployThread - server.setLog(new Log("SERVER")); // open the browser window final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true"); @@ -515,10 +513,10 @@ public final class yacy { return config; } - public static void shutdown() { + public static void shutdown(String reason) { if (sb != null) { // YaCy is running in the same runtime. we can shutdown via interrupt - sb.terminate(); + sb.terminate(reason); } else { final File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/')); shutdown(applicationRoot); @@ -1056,7 +1054,7 @@ class shutdownHookThread extends Thread { // sending the yacy main thread a shutdown signal Log.logFine("SHUTDOWN","Signaling shutdown to the switchboard."); - this.sb.terminate(); + this.sb.terminate("shutdown hook"); // waiting for the yacy thread to finish execution Log.logFine("SHUTDOWN","Waiting for main thread to finish.");