From 11dc653de38abb35bd933781580224083bfbf58f Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 14 Jul 2011 07:07:06 +0000 Subject: [PATCH] added a visualization of peer pings to the performance graphic git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7837 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.java | 38 +++---- htroot/PerformanceSearch_p.java | 4 +- htroot/yacy/hello.java | 47 ++++---- htroot/yacy/search.java | 6 +- htroot/yacysearch.java | 2 +- htroot/yacysearchitem.java | 2 +- htroot/yacysearchtrailer.java | 2 +- source/de/anomic/search/RankingProcess.java | 6 +- source/de/anomic/search/ResultFetcher.java | 10 +- source/de/anomic/search/SearchEvent.java | 6 +- .../anomic/yacy/graphics/ProfilingGraph.java | 54 ++++++++- source/de/anomic/yacy/yacyClient.java | 10 +- source/de/anomic/yacy/yacyCore.java | 106 +++++++++--------- source/de/anomic/yacy/yacyPeerActions.java | 62 +++++----- .../net/yacy/kelondro/util/EventTracker.java | 40 +++---- .../net/yacy/visualization/ChartPlotter.java | 16 +-- source/net/yacy/visualization/PrintTool.java | 14 +-- 17 files changed, 247 insertions(+), 178 deletions(-) diff --git a/htroot/Network.java b/htroot/Network.java index 25ac5dd53..b2902ae61 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -32,6 +32,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -40,8 +41,6 @@ import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.util.MapTools; - -//import de.anomic.http.client.Client; import de.anomic.search.Switchboard; import de.anomic.search.SwitchboardConstants; import de.anomic.server.serverObjects; @@ -52,7 +51,6 @@ import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyPeerActions; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyVersion; -import java.util.concurrent.ConcurrentMap; public class Network { @@ -61,12 +59,12 @@ public class Network { public static serverObjects respond(final RequestHeader requestHeader, final serverObjects post, final serverSwitch switchboard) { final Switchboard sb = (Switchboard) switchboard; final long start = System.currentTimeMillis(); - + // return variable that accumulates replacements final serverObjects prop = new serverObjects(); prop.put("menu", post == null ? 2 : (post.get("menu", "").equals("embed")) ? 0 : (post.get("menu","").equals("simple")) ? 1 : 2); if (sb.peers.mySeed() != null) prop.put("menu_newpeer_peerhash", sb.peers.mySeed().hash); - + prop.setLocalized(!(requestHeader.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml")); prop.putHTML("page_networkTitle", sb.getConfig("network.unit.description", "unspecified")); prop.putHTML("page_networkName", sb.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified")); @@ -98,7 +96,7 @@ public class Network { if (sb.peers.mySeed() != null){ //our Peer // update seed info sb.updateMySeed(); - + final long LCount = seed.getLinkCount(); final long ICount = seed.getWordCount(); final long RCount = seed.getLong(yacySeed.RCOUNT, 0L); @@ -144,17 +142,17 @@ public class Network { prop.putNum("table_my-seeds", seed.getLong(yacySeed.SCOUNT, 0L)); prop.putNum("table_my-connects", seed.getFloat(yacySeed.CCOUNT, 0F)); prop.put("table_my-url", seed.get(yacySeed.SEEDLISTURL, "")); - + // generating the location string prop.putHTML("table_my-location", ClientIdentification.generateLocation()); } // overall results: Network statistics if (iAmActive) conCount++; else if (mySeedType.equals(yacySeed.PEERTYPE_JUNIOR)) potCount++; - int activeLastMonth = sb.peers.sizeActiveSince(30 * 1440); - int activeLastWeek = sb.peers.sizeActiveSince(7 * 1440); - int activeLastDay = sb.peers.sizeActiveSince(1440); - int activeSwitch = + final int activeLastMonth = sb.peers.sizeActiveSince(30 * 1440); + final int activeLastWeek = sb.peers.sizeActiveSince(7 * 1440); + final int activeLastDay = sb.peers.sizeActiveSince(1440); + final int activeSwitch = (activeLastDay <= conCount) ? 0 : (activeLastWeek <= activeLastDay) ? 1 : (activeLastMonth <= activeLastWeek) ? 2 : 3; @@ -182,7 +180,7 @@ public class Network { } else if (post != null && post.getInt("page", 1) == 4) { prop.put("table", 4); // triggers overview prop.put("page", 4); - + if (sb.peers.mySeed() != null) { prop.put("table_my-hash", sb.peers.mySeed().hash ); prop.put("table_my-ip", sb.peers.mySeed().getIP() ); @@ -203,7 +201,7 @@ public class Network { yacySeed peer = new yacySeed(post.get("peerHash"), map); sb.updateMySeed(); - final int added = yacyClient.hello(sb.peers.mySeed(), sb.peers.peerActions, peer.getPublicAddress(), peer.hash); + final int added = yacyClient.hello(sb.peers.mySeed(), sb.peers.peerActions, peer.getPublicAddress(), peer.hash, peer.getName()); if (added <= 0) { prop.put("table_comment",1); @@ -222,7 +220,7 @@ public class Network { prop.putHTML("table_peerHash",post.get("peerHash")); prop.putHTML("table_peerIP",post.get("peerIP")); - prop.putHTML("table_peerPort",post.get("peerPort")); + prop.putHTML("table_peerPort",post.get("peerPort")); } else { prop.put("table_peerHash",""); prop.put("table_peerIP",""); @@ -234,7 +232,7 @@ public class Network { // generate table final int page = (post == null ? 1 : post.getInt("page", 1)); final int maxCount = (post == null ? 300 : post.getInt("maxCount", 300)); - int conCount = 0; + int conCount = 0; if (sb.peers == null) { prop.put("table", 0);//no remote senior/principal proxies known" } else { @@ -360,7 +358,7 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_hash", seed.hash); String shortname = seed.get(yacySeed.NAME, "deadlink"); if (shortname.length() > 20) { - shortname = shortname.substring(0, 20) + "..."; + shortname = shortname.substring(0, 20) + "..."; } prop.putHTML(STR_TABLE_LIST + conCount + "_shortname", shortname); prop.putHTML(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink")); @@ -408,9 +406,9 @@ public class Network { // junior: red/green=direct or red/yellow=passive prop.put(STR_TABLE_LIST + conCount + "_type_direct", seed.getFlagDirectConnect() ? 1 : 0); } - + if (page == 1) { - prop.put(STR_TABLE_LIST + conCount + "_acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0); // green=on or red=off + prop.put(STR_TABLE_LIST + conCount + "_acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0); // green=on or red=off prop.put(STR_TABLE_LIST + conCount + "_dhtreceive", seed.getFlagAcceptRemoteIndex() ? 1 : 0); // green=on or red=off } else { // Passive, Potential Peers if (seed.getFlagAcceptRemoteCrawl()) { @@ -452,7 +450,7 @@ public class Network { prop.put("table", 1); prop.putNum("table_num", conCount); prop.putNum("table_total", ((page == 1) && (iAmActive)) ? (size + 1) : size ); - prop.put("table_complete", ((complete)? 1 : 0) ); + prop.put("table_complete", ((complete)? 1 : 0) ); } } prop.put("page", page); @@ -465,7 +463,7 @@ public class Network { default: break; } } - + prop.putNum("table_rt", System.currentTimeMillis() - start); // return rewrite properties diff --git a/htroot/PerformanceSearch_p.java b/htroot/PerformanceSearch_p.java index c83c9db5d..594b95860 100644 --- a/htroot/PerformanceSearch_p.java +++ b/htroot/PerformanceSearch_p.java @@ -44,11 +44,11 @@ public class PerformanceSearch_p { int c = 0; if (events != null) { EventTracker.Event event; - ProfilingGraph.searchEvent search; + ProfilingGraph.EventSearch search; long lastt = 0; while (events.hasNext()) { event = events.next(); - search = (ProfilingGraph.searchEvent) event.payload; + search = (ProfilingGraph.EventSearch) event.payload; prop.put("table_" + c + "_query", search.queryID); prop.put("table_" + c + "_event", search.processName.name()); prop.put("table_" + c + "_comment", search.comment); diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 61ff899b5..a9b062eff 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -1,4 +1,4 @@ -// hello.java +// hello.java // ----------------------- // part of the AnomicHTTPD caching proxy // (C) by Michael Peter Christen; mc@yacy.net @@ -37,7 +37,7 @@ import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; - +import net.yacy.kelondro.util.EventTracker; import de.anomic.search.Switchboard; import de.anomic.server.serverCore; import de.anomic.server.serverObjects; @@ -48,6 +48,7 @@ import de.anomic.yacy.yacyNetwork; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyVersion; import de.anomic.yacy.dht.PeerSelection; +import de.anomic.yacy.graphics.ProfilingGraph; public final class hello { @@ -57,7 +58,7 @@ public final class hello { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws InterruptedException { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); - long start = System.currentTimeMillis(); + final long start = System.currentTimeMillis(); prop.put("message", "none"); if ((post == null) || (env == null)) { prop.put("message", "no post or no enviroment"); @@ -67,18 +68,18 @@ public final class hello { prop.put("message", "not in my network"); return prop; } - + // final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer // final String mytime = (String) post.get(MYTIME, ""); // final String key = post.get("key", ""); // transmission key for response final String seed = post.get("seed", ""); int count = post.getInt("count", 0); - long magic = post.getLong("magic", 0); + final long magic = post.getLong("magic", 0); // final Date remoteTime = yacyCore.parseUniversalDate(post.get(MYTIME)); // read remote time final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""); // read an artificial header addendum long time = System.currentTimeMillis(); final InetAddress ias = Domains.dnsResolve(clientip); - long time_dnsResolve = System.currentTimeMillis() - time; + final long time_dnsResolve = System.currentTimeMillis() - time; if (ias == null) { yacyCore.log.logInfo("hello/server: failed contacting seed; clientip not resolvable (clientip=" + clientip + ", time_dnsResolve=" + time_dnsResolve + ")"); prop.put("message", "cannot resolve your IP from your reported location " + clientip); @@ -92,18 +93,18 @@ public final class hello { yacySeed remoteSeed; try { remoteSeed = yacySeed.genRemoteSeed(seed, key, true, ias.getHostAddress()); - } catch (IOException e) { + } catch (final IOException e) { yacyCore.log.logInfo("hello/server: bad seed: " + e.getMessage() + ", time_dnsResolve=" + time_dnsResolve); prop.put("message", "bad seed: " + e.getMessage()); return prop; } - + if (remoteSeed == null || remoteSeed.hash == null) { yacyCore.log.logInfo("hello/server: bad seed: null, time_dnsResolve=" + time_dnsResolve); prop.put("message", "cannot parse your seed"); return prop; } - + // final String properTest = remoteSeed.isProper(); // The remote peer might not know its IP yet, so don't abort if the IP check fails // if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; } @@ -119,12 +120,12 @@ public final class hello { prop.put("message", "I am robinson, I do not answer"); return prop; } - + long[] callback = new long[]{-1, -1}; if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress(sb.clusterhashes.get(remoteSeed.hash.getBytes())); - + // if the remote client has reported its own IP address and the client supports - // the port forwarding feature (if client version >= 0.383) then we try to + // the port forwarding feature (if client version >= 0.383) then we try to // connect to the reported IP address first time = System.currentTimeMillis(); long time_backping = 0; @@ -132,9 +133,9 @@ public final class hello { if (reportedip.length() > 0 && !clientip.equals(reportedip) && clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING && - magic != 0) { + magic != 0) { serverCore.checkInterruption(); - + // try first the reportedip, since this may be a connect from a port-forwarding host prop.put("yourip", reportedip); remoteSeed.setIP(reportedip); @@ -158,7 +159,7 @@ public final class hello { if (isNotLocal) { serverCore.checkInterruption(); - + prop.put("yourip", clientip); remoteSeed.setIP(clientip); callback = yacyClient.queryUrlCount(remoteSeed); @@ -170,8 +171,9 @@ public final class hello { // System.out.println("YACYHELLO: YOUR IP=" + clientip); // set lastseen value (we have seen that peer, it contacted us!) remoteSeed.setLastSeenUTC(); - + // assign status + final int connectedBefore = sb.peers.sizeConnected(); if (callback[0] >= 0) { if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR); @@ -194,6 +196,11 @@ public final class hello { sb.peers.peerActions.peerPing(remoteSeed); } } + final int connectedAfter = sb.peers.sizeConnected(); + + // update event tracker + EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false); + sb.peers.peerActions.setUserAgent(clientip, userAgent); if (!(prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) { yacyCore.log.logInfo("hello/server: changing remote peer '" + remoteSeed.getName() + @@ -208,14 +215,14 @@ public final class hello { if (sb.peers.sizeConnected() > 0) { if (count > sb.peers.sizeConnected()) { count = sb.peers.sizeConnected(); } if (count > 100) { count = 100; } - + // latest seeds final Map ySeeds = PeerSelection.seedsByAge(sb.peers, true, count); // peerhash/yacySeed relation - + // attach also my own seed seeds.append("seed0=").append(sb.peers.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING); - count = 1; - + count = 1; + // attach other seeds if (ySeeds != null) { seeds.ensureCapacity((ySeeds.size() + 1) * 768); diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index c4ca01da8..b87e2c32e 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -250,7 +250,7 @@ public final class search { //final Map>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls)); final TreeMap> incc = indexSegment.termIndex().searchConjunction(theQuery.queryHashes, QueryParams.hashes2Handles(urls)); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.COLLECTION, "", incc.size(), System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.COLLECTION, "", incc.size(), System.currentTimeMillis() - timer), false); if (incc != null) { final Iterator>> ci = incc.entrySet().iterator(); Map.Entry> entry; @@ -373,7 +373,7 @@ public final class search { i++; } prop.put("references", (refstr.length() > 0) ? refstr.substring(1) : refstr.toString()); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.REFERENCECOLLECTION, "", i, System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.REFERENCECOLLECTION, "", i, System.currentTimeMillis() - timer), false); } prop.put("indexabstract", indexabstract.toString()); @@ -401,7 +401,7 @@ public final class search { theQuery.transmitcount = accu.size() + 1; prop.put("links", links.toString()); prop.put("linkcount", accu.size()); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.RESULTLIST, "", accu.size(), System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.RESULTLIST, "", accu.size(), System.currentTimeMillis() - timer), false); } // add information about forward peers diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index b90c89880..5d6e8d239 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -604,7 +604,7 @@ public class yacysearch { header.get(RequestHeader.USER_AGENT, ""), sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex()); EventTracker.delete(EventTracker.EClass.SEARCH); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false); // tell all threads to do nothing for a specific time sb.intermissionAllThreads(3000); diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index d44f469cc..e8be770ed 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -186,7 +186,7 @@ public class yacysearchitem { } prop.put("content_heuristic_name", heuristic.heuristicName); } - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.FINALIZATION, "" + item, 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.FINALIZATION, "" + item, 0, 0), false); final String ext = resultURL.getFileExtension().toLowerCase(); if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) { final String license = sb.licensedURLs.aquireLicense(resultURL); diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 28902ce54..d6a4424fd 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -240,7 +240,7 @@ public class yacysearchtrailer { prop.put("cat-location_queryenc", theQuery.queryString(true).replace(' ', '+')); } - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.FINALIZATION, "bottomline", 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.FINALIZATION, "bottomline", 0, 0), false); return prop; } diff --git a/source/de/anomic/search/RankingProcess.java b/source/de/anomic/search/RankingProcess.java index dc82e4f2e..1820e3695 100644 --- a/source/de/anomic/search/RankingProcess.java +++ b/source/de/anomic/search/RankingProcess.java @@ -149,7 +149,7 @@ public final class RankingProcess extends Thread { this.query.maxDistance); this.localSearchInclusion = search.inclusion(); final ReferenceContainer index = search.joined(); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.JOIN, this.query.queryString, index.size(), System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.JOIN, this.query.queryString, index.size(), System.currentTimeMillis() - timer), false); if (index.isEmpty()) { return; } @@ -188,7 +188,7 @@ public final class RankingProcess extends Thread { // normalize entries final BlockingQueue decodedEntries = this.order.normalizeWith(index); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.NORMALIZING, resourceName, index.size(), System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.NORMALIZING, resourceName, index.size(), System.currentTimeMillis() - timer), false); // iterate over normalized entries and select some that are better than currently stored timer = System.currentTimeMillis(); @@ -279,7 +279,7 @@ public final class RankingProcess extends Thread { } //if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.PRESORT, resourceName, index.size(), System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.PRESORT, resourceName, index.size(), System.currentTimeMillis() - timer), false); } /** diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index ba55a0bae..a3f6ac1de 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -111,7 +111,7 @@ public class ResultFetcher { // start worker threads to fetch urls and snippets this.workerThreads = null; deployWorker(Math.min(10, query.itemsPerPage), query.neededResults()); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), SearchEvent.Type.SNIPPETFETCH_START, ((this.workerThreads == null) ? "no" : this.workerThreads.length) + " online snippet fetch threads started", 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), SearchEvent.Type.SNIPPETFETCH_START, ((this.workerThreads == null) ? "no" : this.workerThreads.length) + " online snippet fetch threads started", 0, 0), false); } public void setCleanupState() { @@ -132,11 +132,11 @@ public class ResultFetcher { if (!this.query.isLocal() && item == 0) try { Thread.sleep(100); } catch (final InterruptedException e1) {} // wait a little time to get first results in the search final long finishTime = System.currentTimeMillis() + timeout; - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.ONERESULT, "started, item = " + item + ", available = " + this.result.sizeAvailable(), 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.ONERESULT, "started, item = " + item + ", available = " + this.result.sizeAvailable(), 0, 0), false); if (this.result.sizeAvailable() > item) { // we have the wanted result already in the result array .. return that final ResultEntry re = this.result.element(item).getElement(); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.ONERESULT, "prefetched, item = " + item + ", available = " + this.result.sizeAvailable() + ": " + re.urlstring(), 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.ONERESULT, "prefetched, item = " + item + ", available = " + this.result.sizeAvailable() + ": " + re.urlstring(), 0, 0), false); return re; } @@ -155,11 +155,11 @@ public class ResultFetcher { // finally, if there is something, return the result if (entry == null) { - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.ONERESULT, "not found, item = " + item + ", available = " + this.result.sizeAvailable(), 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.ONERESULT, "not found, item = " + item + ", available = " + this.result.sizeAvailable(), 0, 0), false); return null; } final ResultEntry re = entry.getElement(); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.ONERESULT, "retrieved, item = " + item + ", available = " + this.result.sizeAvailable() + ": " + re.urlstring(), 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.ONERESULT, "retrieved, item = " + item + ", available = " + this.result.sizeAvailable() + ": " + re.urlstring(), 0, 0), false); return re; } diff --git a/source/de/anomic/search/SearchEvent.java b/source/de/anomic/search/SearchEvent.java index 75858ba49..c22d4df81 100644 --- a/source/de/anomic/search/SearchEvent.java +++ b/source/de/anomic/search/SearchEvent.java @@ -150,7 +150,7 @@ public final class SearchEvent { if (this.primarySearchThreads != null) { Log.logFine("SEARCH_EVENT", "STARTING " + this.primarySearchThreads.length + " THREADS TO CATCH EACH " + remote_maxcount + " URLs"); this.rankingProcess.moreFeeders(this.primarySearchThreads.length); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false); // finished searching Log.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + this.primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); } else { @@ -189,7 +189,7 @@ public final class SearchEvent { this.IACount.put(wordhash, LargeNumberCache.valueOf(container.size())); this.IAResults.put(wordhash, WordReferenceFactory.compressIndex(container, null, 1000).toString()); } - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false); } else { this.rankingProcess.start(); // start concurrently // but give process time to accumulate a certain amount of data @@ -208,7 +208,7 @@ public final class SearchEvent { // clean up events SearchEventCache.cleanupEvents(false); - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), Type.CLEANUP, "", 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.CLEANUP, "", 0, 0), false); // store this search to a cache so it can be re-used if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true); diff --git a/source/de/anomic/yacy/graphics/ProfilingGraph.java b/source/de/anomic/yacy/graphics/ProfilingGraph.java index 273389650..a9084269d 100644 --- a/source/de/anomic/yacy/graphics/ProfilingGraph.java +++ b/source/de/anomic/yacy/graphics/ProfilingGraph.java @@ -84,12 +84,13 @@ public class ProfilingGraph { } chart.declareDimension(ChartPlotter.DIMENSION_ANOT0, anotscale, vspace * anotscale / maxppm, 0, "008800", null , "PPM [PAGES/MINUTE]"); chart.declareDimension(ChartPlotter.DIMENSION_ANOT1, vspace / 6, vspace / 6, 0, "888800", null , "URL"); + chart.declareDimension(ChartPlotter.DIMENSION_ANOT2, 1, 1, 0, "888800", null , "PING"); // draw chart long time; final long now = System.currentTimeMillis(); long bytes; - int x0, x1, y0, y1, ppm, words; + int x0, x1, y0, y1; try { // draw urls /* @@ -134,6 +135,7 @@ public class ProfilingGraph { x0 = 1; y0 = 0; if (events != null) { EventTracker.Event event; + int words; while (events.hasNext()) { event = events.next(); time = event.time - now; @@ -153,6 +155,7 @@ public class ProfilingGraph { x0 = 1; y0 = 0; if (events != null) { EventTracker.Event event; + int ppm; while (events.hasNext()) { event = events.next(); time = event.time - now; @@ -167,6 +170,24 @@ public class ProfilingGraph { } } + // draw peer ping + events = EventTracker.getHistory(EventTracker.EClass.PEERPING); + x0 = 1; y0 = 0; + if (events != null) { + EventTracker.Event event; + EventPing ping; + while (events.hasNext()) { + event = events.next(); + time = event.time - now; + ping = (EventPing) event.payload; + x1 = (int) (time/1000); + y1 = Math.abs((ping.outgoing ? ping.toPeer : ping.fromPeer).hashCode()) % vspace; + chart.setColor("444444"); + chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT2, x1, y1, 2, "PING " + ping.fromPeer.toUpperCase() + " -> " + ping.toPeer.toUpperCase() + (ping.newPeers > 0 ? "(+" + ping.newPeers + ")" : ""), 0); + x0 = x1; y0 = y1; + } + } + bufferChart = chart; } catch (final ConcurrentModificationException cme) { chart = bufferChart; @@ -175,14 +196,14 @@ public class ProfilingGraph { return chart; } - public static class searchEvent { + public static class EventSearch { public SearchEvent.Type processName; public String comment; public String queryID; public long duration; public int resultCount; - public searchEvent(final String queryID, final SearchEvent.Type processName, final String comment, final int resultCount, final long duration) { + public EventSearch(final String queryID, final SearchEvent.Type processName, final String comment, final int resultCount, final long duration) { this.queryID = queryID; this.processName = processName; this.comment = comment; @@ -191,4 +212,31 @@ public class ProfilingGraph { } } + public static class EventDHT { + public String fromPeer, toPeer; + public boolean outgoing; + public int totalReferences, newReferences; + + public EventDHT(final String fromPeer, final String toPeer, final boolean outgoing, final int totalReferences, final int newReferences) { + this.fromPeer = fromPeer; + this.toPeer = toPeer; + this.outgoing = outgoing; + this.totalReferences = totalReferences; + this.newReferences = newReferences; + } + } + + public static class EventPing { + + public String fromPeer, toPeer; + public boolean outgoing; + public int newPeers; + + public EventPing(final String fromPeer, final String toPeer, final boolean outgoing, final int newPeers) { + this.fromPeer = fromPeer; + this.toPeer = toPeer; + this.outgoing = outgoing; + this.newPeers = newPeers; + } + } } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index b42c3a9dd..74a95d67c 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -87,6 +87,7 @@ import net.yacy.kelondro.rwi.Reference; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainerCache; import net.yacy.kelondro.util.ByteBuffer; +import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; @@ -104,6 +105,7 @@ import de.anomic.search.Switchboard; import de.anomic.search.TextSnippet; import de.anomic.server.serverCore; import de.anomic.tools.crypt; +import de.anomic.yacy.graphics.ProfilingGraph; import de.anomic.yacy.graphics.WebStructureGraph; import de.anomic.yacy.graphics.WebStructureGraph.HostReference; @@ -143,7 +145,7 @@ public final class yacyClient { * * @return the number of new seeds */ - public static int hello(final yacySeed mySeed, final yacyPeerActions peerActions, final String address, final String otherHash) { + public static int hello(final yacySeed mySeed, final yacyPeerActions peerActions, final String address, final String otherHash, final String otherName) { Map result = null; final String salt = crypt.randomSalt(); @@ -256,6 +258,7 @@ public final class yacyClient { int count = 0; String seedStr; yacySeed s; + final int connectedBefore = peerActions.sizeConnected(); while ((seedStr = result.get("seed" + i++)) != null) { // integrate new seed into own database // the first seed, "seed0" is the seed of the responding peer @@ -277,6 +280,11 @@ public final class yacyClient { } } } + final int connectedAfter = peerActions.sizeConnected(); + + // update event tracker + EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(mySeed.getName(), otherName, true, connectedAfter - connectedBefore), false); + return count; } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index 440ab71e5..f2056a2f8 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -80,7 +80,7 @@ public class yacyCore { private static final int PING_MIN_DBSIZE = 5; private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds - + // public static yacyShare shareManager = null; // public static boolean terminate = false; @@ -97,11 +97,11 @@ public class yacyCore { this.sb = sb; sb.setConfig("yacyStatus", ""); - + // create a peer news channel final RSSFeed peernews = yacyChannel.channels(yacyChannel.PEERNEWS); peernews.addMessage(new RSSMessage("YaCy started", "", "")); - + // ensure that correct IP is used final String staticIP = sb.getConfig("staticIP", ""); if (staticIP.length() != 0 && yacySeed.isProperIP(staticIP) == null) { @@ -124,7 +124,7 @@ public class yacyCore { synchronized static public void triggerOnlineAction() { lastOnlineTime = System.currentTimeMillis(); } - + public final void publishSeedList() { if (log.isFine()) log.logFine("yacyCore.publishSeedList: Triggered Seed Publish"); @@ -137,11 +137,11 @@ public class yacyCore { yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself"); */ - if ((sb.peers.lastSeedUpload_myIP.equals(sb.peers.mySeed().getIP())) && - (sb.peers.lastSeedUpload_seedDBSize == sb.peers.sizeConnected()) && + if ((this.sb.peers.lastSeedUpload_myIP.equals(this.sb.peers.mySeed().getIP())) && + (this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected()) && (canReachMyself()) && - (System.currentTimeMillis() - sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) && - (sb.peers.mySeed().isPrincipal()) + (System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) && + (this.sb.peers.mySeed().isPrincipal()) ) { if (log.isFine()) log.logFine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP."); return; @@ -164,7 +164,7 @@ public class yacyCore { } } // we want to be a principal... - saveSeedList(sb); + saveSeedList(this.sb); } else { if (seedUploadMethod.equals("")) { this.sb.setConfig("seedUploadMethod", "none"); @@ -175,7 +175,7 @@ public class yacyCore { } public final void peerPing() { - if ((sb.isRobinsonMode()) && (sb.getConfig("cluster.mode", "").equals("privatepeer"))) { + if ((this.sb.isRobinsonMode()) && (this.sb.getConfig("cluster.mode", "").equals("privatepeer"))) { // in case this peer is a privat peer we omit the peer ping // all other robinson peer types do a peer ping: // the privatecluster does the ping to the other cluster members @@ -185,34 +185,34 @@ public class yacyCore { } // before publishing, update some seed data - sb.updateMySeed(); + this.sb.updateMySeed(); // publish own seed to other peer, this can every peer, but makes only sense for senior peers - if (sb.peers.sizeConnected() == 0) { + if (this.sb.peers.sizeConnected() == 0) { // reload the seed lists - sb.loadSeedLists(); - log.logInfo("re-initialized seed list. received " + sb.peers.sizeConnected() + " new peer(s)"); + this.sb.loadSeedLists(); + log.logInfo("re-initialized seed list. received " + this.sb.peers.sizeConnected() + " new peer(s)"); } final int newSeeds = publishMySeed(false); if (newSeeds > 0) { - log.logInfo("received " + newSeeds + " new peer(s), know a total of " + sb.peers.sizeConnected() + " different peers"); + log.logInfo("received " + newSeeds + " new peer(s), know a total of " + this.sb.peers.sizeConnected() + " different peers"); } } private boolean canReachMyself() { // TODO: check if this method is necessary - depending on the used router it will not work // returns true if we can reach ourself under our known peer address // if we cannot reach ourself, we call a forced publishMySeed and return false - final long[] callback = yacyClient.queryUrlCount(sb.peers.mySeed()); + final long[] callback = yacyClient.queryUrlCount(this.sb.peers.mySeed()); if (callback[0] >= 0 && callback[1] == magic) { - sb.peers.mySeed().setLastSeenUTC(); + this.sb.peers.mySeed().setLastSeenUTC(); return true; } log.logInfo("re-connect own seed"); - final String oldAddress = sb.peers.mySeed().getPublicAddress(); + final String oldAddress = this.sb.peers.mySeed().getPublicAddress(); /*final int newSeeds =*/ publishMySeed(true); - return (oldAddress != null && oldAddress.equals(sb.peers.mySeed().getPublicAddress())); + return (oldAddress != null && oldAddress.equals(this.sb.peers.mySeed().getPublicAddress())); } - + // use our own formatter to prevent concurrency locks with other processes private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); @@ -236,24 +236,24 @@ public class yacyCore { public final void run() { try { - this.added = yacyClient.hello(sb.peers.mySeed(), sb.peers.peerActions, seed.getClusterAddress(), seed.hash); + this.added = yacyClient.hello(yacyCore.this.sb.peers.mySeed(), yacyCore.this.sb.peers.peerActions, this.seed.getClusterAddress(), this.seed.hash, this.seed.getName()); if (this.added < 0) { // no or wrong response, delete that address final String cause = "peer ping to peer resulted in error response (added < 0)"; log.logInfo("publish: disconnected " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getPublicAddress() + ": " + cause); - sb.peers.peerActions.peerDeparture(this.seed, cause); + yacyCore.this.sb.peers.peerActions.peerDeparture(this.seed, cause); } else { // success! we have published our peer to a senior peer // update latest news from the other peer log.logInfo("publish: handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress()); // check if seed's lastSeen has been updated - final yacySeed newSeed = sb.peers.getConnected(this.seed.hash); + final yacySeed newSeed = yacyCore.this.sb.peers.getConnected(this.seed.hash); if (newSeed != null) { if (!newSeed.isOnline()) { if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " is not online." + " Removing Peer from connected"); - sb.peers.peerActions.peerDeparture(newSeed, "peer not online"); + yacyCore.this.sb.peers.peerActions.peerDeparture(newSeed, "peer not online"); } else if (newSeed.getLastSeenUTC() < (System.currentTimeMillis() - 10000)) { // update last seed date @@ -262,14 +262,14 @@ public class yacyCore { " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "'"); newSeed.setLastSeenUTC(); - sb.peers.peerActions.peerArrival(newSeed, true); + yacyCore.this.sb.peers.peerActions.peerArrival(newSeed, true); } else { if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" + my_SHORT_SECOND_FORMATTER.format(new Date(this.seed.getLastSeenUTC())) + "'"); this.seed.setLastSeenUTC(); - sb.peers.peerActions.peerArrival(this.seed, true); + yacyCore.this.sb.peers.peerActions.peerArrival(this.seed, true); } } } else { @@ -277,7 +277,7 @@ public class yacyCore { } } } catch (final Exception e) { - log.logSevere("publishThread: error with target seed " + seed.toString() + ": " + e.getMessage(), e); + log.logSevere("publishThread: error with target seed " + this.seed.toString() + ": " + e.getMessage(), e); } finally { this.syncList.add(this); this.sync.release(); @@ -306,13 +306,13 @@ public class yacyCore { // init yacyHello-process Map seeds; // hash/yacySeed relation - int attempts = sb.peers.sizeConnected(); + int attempts = this.sb.peers.sizeConnected(); // getting a list of peers to contact - if (sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) { + if (this.sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) { if (attempts > PING_INITIAL) { attempts = PING_INITIAL; } final Map ch = Switchboard.getSwitchboard().clusterhashes; - seeds = PeerSelection.seedsByAge(sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection + seeds = PeerSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection // add also all peers from cluster if this is a public robinson cluster if (ch != null) { final Iterator> i = ch.entrySet().iterator(); @@ -324,7 +324,7 @@ public class yacyCore { hash = ASCII.String(entry.getKey()); seed = seeds.get(hash); if (seed == null) { - seed = sb.peers.get(hash); + seed = this.sb.peers.get(hash); if (seed == null) continue; } seed.setAlternativeAddress(entry.getValue()); @@ -339,7 +339,7 @@ public class yacyCore { } else { if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; } } - seeds = PeerSelection.seedsByAge(sb.peers, false, attempts); // best for seed list maintenance/cleaning + seeds = PeerSelection.seedsByAge(this.sb.peers, false, attempts); // best for seed list maintenance/cleaning } if (seeds == null || seeds.isEmpty()) { return 0; } @@ -351,16 +351,16 @@ public class yacyCore { // include a YaCyNews record to my seed try { - final yacyNewsDB.Record record = sb.peers.newsPool.myPublication(); + final yacyNewsDB.Record record = this.sb.peers.newsPool.myPublication(); if (record == null) { - sb.peers.mySeed().put("news", ""); + this.sb.peers.mySeed().put("news", ""); } else { - sb.peers.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString())); + this.sb.peers.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString())); } } catch (final Exception e) { log.logSevere("publishMySeed: problem with news encoding", e); } - sb.peers.mySeed().setUnusedFlags(); + this.sb.peers.mySeed().setUnusedFlags(); int newSeeds = -1; //if (seeds.length > 1) { // holding a reference to all started threads @@ -383,7 +383,7 @@ public class yacyCore { final String seederror = seed.isProper(false); if ((address == null) || (seederror != null)) { // we don't like that address, delete it - sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror); + this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror); sync.acquire(); } else { // starting a new publisher thread @@ -448,7 +448,7 @@ public class yacyCore { if ((accessible >= PING_MIN_PEERSEEN) || (accessible >= notaccessible)) { // We can be reached from a majority of other Peers - if (sb.peers.mySeed().isPrincipal()) { + if (this.sb.peers.mySeed().isPrincipal()) { newPeerType = yacySeed.PEERTYPE_PRINCIPAL; } else { newPeerType = yacySeed.PEERTYPE_SENIOR; @@ -457,38 +457,38 @@ public class yacyCore { // We cannot be reached from the outside newPeerType = yacySeed.PEERTYPE_JUNIOR; } - if (sb.peers.mySeed().orVirgin().equals(newPeerType)) { - log.logInfo("PeerPing: myType is " + sb.peers.mySeed().orVirgin()); + if (this.sb.peers.mySeed().orVirgin().equals(newPeerType)) { + log.logInfo("PeerPing: myType is " + this.sb.peers.mySeed().orVirgin()); } else { - log.logInfo("PeerPing: changing myType from '" + sb.peers.mySeed().orVirgin() + "' to '" + newPeerType + "'"); - sb.peers.mySeed().put(yacySeed.PEERTYPE, newPeerType); + log.logInfo("PeerPing: changing myType from '" + this.sb.peers.mySeed().orVirgin() + "' to '" + newPeerType + "'"); + this.sb.peers.mySeed().put(yacySeed.PEERTYPE, newPeerType); } } else { - log.logInfo("PeerPing: No data, staying at myType: " + sb.peers.mySeed().orVirgin()); + log.logInfo("PeerPing: No data, staying at myType: " + this.sb.peers.mySeed().orVirgin()); } // success! we have published our peer to a senior peer // update latest news from the other peer // log.logInfo("publish: handshaked " + t.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + t.seed.getName() + "' at " + t.seed.getAddress()); - sb.peers.saveMySeed(); + this.sb.peers.saveMySeed(); // if we have an address, we do nothing - if (sb.peers.mySeed().isProper(true) == null && !force) { return 0; } + if (this.sb.peers.mySeed().isProper(true) == null && !force) { return 0; } if (newSeeds > 0) return newSeeds; - + // still no success: ask own NAT or internet responder //final boolean DI604use = switchboard.getConfig("DI604use", "false").equals("true"); //final String DI604pw = switchboard.getConfig("DI604pw", ""); - final String ip = sb.getConfig("staticIP", ""); + final String ip = this.sb.getConfig("staticIP", ""); //if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw); - + // yacyCore.log.logDebug("DEBUG: new IP=" + ip); - if (yacySeed.isProperIP(ip) == null) sb.peers.mySeed().setIP(ip); - if (sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ??????????????? - sb.peers.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer + if (yacySeed.isProperIP(ip) == null) this.sb.peers.mySeed().setIP(ip); + if (this.sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ??????????????? + this.sb.peers.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer log.logInfo("publish: no recipient found, our address is " + - ((sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : sb.peers.mySeed().getPublicAddress())); - sb.peers.saveMySeed(); + ((this.sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : this.sb.peers.mySeed().getPublicAddress())); + this.sb.peers.saveMySeed(); return 0; } catch (final InterruptedException e) { try { diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index f214a6da3..eb6e6dfa1 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -1,4 +1,4 @@ -// yacyPeerActions.java +// yacyPeerActions.java // ------------------------------------- // (C) by Michael Peter Christen; mc@yacy.net // first published on http://yacy.net @@ -34,12 +34,12 @@ import net.yacy.kelondro.util.MapTools; public class yacyPeerActions { - + private final yacySeedDB seedDB; private Map userAgents; public long disconnects; private final yacyNewsPool newsPool; - + public yacyPeerActions(final yacySeedDB seedDB, final yacyNewsPool newsPool) { this.seedDB = seedDB; this.newsPool = newsPool; @@ -49,10 +49,10 @@ public class yacyPeerActions { public void close() { // the seedDB and newsPool should be cleared elsewhere - if (userAgents != null) userAgents.clear(); - userAgents = null; + if (this.userAgents != null) this.userAgents.clear(); + this.userAgents = null; } - + public boolean connectPeer(final yacySeed seed, final boolean direct) { // store a remote peer's seed // returns true if the peer is new and previously unknown @@ -113,7 +113,7 @@ public class yacyPeerActions { // disconnection time long dtimeUTC0; - final yacySeed disconnectedSeed = seedDB.getDisconnected(seed.hash); + final yacySeed disconnectedSeed = this.seedDB.getDisconnected(seed.hash); if (disconnectedSeed == null) { dtimeUTC0 = 0; // never disconnected: virtually disconnected maximum time ago } else { @@ -173,12 +173,12 @@ public class yacyPeerActions { return false; } if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); - seedDB.addConnected(seed); + this.seedDB.addConnected(seed); return true; } - + // the seed is new - if ((seedDB.mySeedIsDefined()) && (seed.getIP().equals(this.seedDB.mySeed().getIP()))) { + if ((this.seedDB.mySeedIsDefined()) && (seed.getIP().equals(this.seedDB.mySeed().getIP()))) { // seed from the same IP as the calling client: can be // the case if there runs another one over a NAT if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: saved NEW seed (myself IP) " + seed.getPublicAddress()); @@ -195,33 +195,33 @@ public class yacyPeerActions { final boolean res = connectPeer(peer, direct); if (res) { // perform all actions if peer is effective new - this.processPeerArrival(peer); + processPeerArrival(peer); yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " joined the network", "", "")); } return res; } - + public void peerDeparture(final yacySeed peer, final String cause) { if (peer == null) return; // we do this if we did not get contact with the other peer if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: no contact to a " + peer.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress() + ". Cause: " + cause); - synchronized (seedDB) { - if (!seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { disconnects++; } + synchronized (this.seedDB) { + if (!this.seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { this.disconnects++; } peer.put("dct", Long.toString(System.currentTimeMillis())); - seedDB.addDisconnected(peer); // update info + this.seedDB.addDisconnected(peer); // update info } yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " left the network", "", "")); } - + public void peerPing(final yacySeed peer) { if (peer == null) return; // this is called only if the peer has junior status - seedDB.addPotential(peer); + this.seedDB.addPotential(peer); // perform all actions processPeerArrival(peer); yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " sent me a ping", "", "")); } - + private void processPeerArrival(final yacySeed peer) { final String recordString = peer.get("news", null); //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached")); @@ -243,14 +243,18 @@ public class yacyPeerActions { } } } - + + public int sizeConnected() { + return this.seedDB.sizeConnected(); + } + public void setUserAgent(final String IP, final String userAgent) { - if (userAgents == null) return; // case can happen during shutdown - userAgents.put(IP, userAgent); + if (this.userAgents == null) return; // case can happen during shutdown + this.userAgents.put(IP, userAgent); } - + public String getUserAgent(final String IP) { - final String userAgent = userAgents.get(IP); + final String userAgent = this.userAgents.get(IP); return (userAgent == null) ? "" : userAgent; } @@ -261,22 +265,22 @@ public class yacyPeerActions { public static String formatInterval(final long millis) { try { final long mins = millis / 60000; - + final StringBuilder uptime = new StringBuilder(40); - + final int uptimeDays = (int) (Math.floor(mins/1440.0)); final int uptimeHours = (int) (Math.floor(mins/60.0)%24); final int uptimeMins = (int) mins%60; - + uptime.append(uptimeDays) .append(((uptimeDays == 1)?" day ":" days ")) .append((uptimeHours < 10)?"0":"") .append(uptimeHours) .append(':') .append((uptimeMins < 10)?"0":"") - .append(uptimeMins); - - return uptime.toString(); + .append(uptimeMins); + + return uptime.toString(); } catch (final Exception e) { return "unknown"; } diff --git a/source/net/yacy/kelondro/util/EventTracker.java b/source/net/yacy/kelondro/util/EventTracker.java index da7174645..22015147c 100644 --- a/source/net/yacy/kelondro/util/EventTracker.java +++ b/source/net/yacy/kelondro/util/EventTracker.java @@ -34,59 +34,61 @@ import java.util.concurrent.ConcurrentLinkedQueue; import de.anomic.yacy.graphics.ProfilingGraph; public class EventTracker { - + private final static int maxQueueSize = 30000; private final static long maxQueueAge = ProfilingGraph.maxTime; - + public enum EClass { WORDCACHE, MEMORY, PPM, + PEERPING, + DHT, INDEX, SEARCH; } - + private final static Map> historyMaps = new ConcurrentHashMap>(); private final static Map eventAccess = new ConcurrentHashMap(); // value: last time when this was accessed - + public final static void delete(final EClass eventName) { historyMaps.remove(eventName); eventAccess.remove(eventName); } - - public final static void update(final EClass eventName, final Object eventPayload, boolean useProtection) { + + public final static void update(final EClass eventName, final Object eventPayload, final boolean useProtection) { // check protection against too heavy access if (useProtection) { - Long lastAcc = eventAccess.get(eventName); + final Long lastAcc = eventAccess.get(eventName); if (lastAcc == null) { eventAccess.put(eventName, Long.valueOf(System.currentTimeMillis())); } else { - long time = System.currentTimeMillis(); + final long time = System.currentTimeMillis(); if (time - lastAcc.longValue() < 1000) { return; // protect against too heavy load } eventAccess.put(eventName, Long.valueOf(time)); } } - + // get event history container ConcurrentLinkedQueue history = historyMaps.get(eventName); - + // create history if (history == null) { history = new ConcurrentLinkedQueue(); // update entry history.offer(new Event(eventPayload)); - + // store map historyMaps.put(eventName, history); return; } - + // update history history.offer(new Event(eventPayload)); - + // clean up too old entries int tp = history.size() - maxQueueSize; while (tp-- > 0) history.poll(); @@ -104,24 +106,24 @@ public class EventTracker { } } } - + public final static Iterator getHistory(final EClass eventName) { - ConcurrentLinkedQueue list = historyMaps.get(eventName); + final ConcurrentLinkedQueue list = historyMaps.get(eventName); if (list == null) return null; return list.iterator(); } - public final static int countEvents(final EClass eventName, long time) { - Iterator event = getHistory(eventName); + public final static int countEvents(final EClass eventName, final long time) { + final Iterator event = getHistory(eventName); if (event == null) return 0; - long now = System.currentTimeMillis(); + final long now = System.currentTimeMillis(); int count = 0; while (event.hasNext()) { if (now - event.next().time < time) count++; } return count; } - + public final static class Event { public Object payload; public long time; diff --git a/source/net/yacy/visualization/ChartPlotter.java b/source/net/yacy/visualization/ChartPlotter.java index 00a17e6b7..c1754554d 100644 --- a/source/net/yacy/visualization/ChartPlotter.java +++ b/source/net/yacy/visualization/ChartPlotter.java @@ -40,17 +40,19 @@ public class ChartPlotter extends RasterPlotter { public static final int DIMENSION_BOTTOM = 3; public static final int DIMENSION_ANOT0 = 4; public static final int DIMENSION_ANOT1 = 5; + public static final int DIMENSION_ANOT2 = 6; + public static final int DIMENSION_ANOT3 = 7; private final int leftborder; private final int rightborder; private final int topborder; private final int bottomborder; - private final int[] scales = new int[]{0,0,0,0,0,0}; - private final int[] pixels = new int[]{0,0,0,0,0,0}; - private final int[] offsets = new int[]{0,0,0,0,0,0}; - private final String[] colnames = new String[]{"FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF"}; - private final String[] colscale = new String[]{null,null,null,null,null,null}; - private final String[] tablenames = new String[]{"","","","","",""}; + private final int[] scales = new int[]{0,0,0,0,0,0,0,0}; + private final int[] pixels = new int[]{0,0,0,0,0,0,0,0}; + private final int[] offsets = new int[]{0,0,0,0,0,0,0,0}; + private final String[] colnames = new String[]{"FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF"}; + private final String[] colscale = new String[]{null,null,null,null,null,null,null,null}; + private final String[] tablenames = new String[]{"","","","","","","",""}; public ChartPlotter(final int width, final int height, final String backgroundColor, final String foregroundColor, final String lightColor, final int leftborder, final int rightborder, final int topborder, final int bottomborder, @@ -93,7 +95,7 @@ public class ChartPlotter extends RasterPlotter { final int y = (coord_y - this.offsets[dimension_y]) * this.pixels[dimension_y] / this.scales[dimension_y]; if (dotsize == 1) plot(this.leftborder + x, this.height - this.bottomborder - y, 100); else dot(this.leftborder + x, this.height - this.bottomborder - y, dotsize, true, 100); - if (anot != null) PrintTool.print(this, this.leftborder + x + dotsize + 2 + ((anotAngle == 315) ? -9 : 0), this.height - this.bottomborder - y + ((anotAngle == 315) ? -3 : 0), anotAngle, anot, (anotAngle == 0) ? -1 : ((anotAngle == 315) ? 1 : 0)); + if (anot != null) PrintTool.print(this, this.leftborder + x + dotsize + 2 + ((anotAngle == 315) ? -9 : 0), this.height - this.bottomborder - y + ((anotAngle == 315) ? -3 : 0), anotAngle, anot, (anotAngle == 0) ? (anot.length() * 6 + x > this.width ? 1 : -1) : ((anotAngle == 315) ? 1 : 0)); } public void chartLine(final int dimension_x, final int dimension_y, final int coord_x1, final int coord_y1, final int coord_x2, final int coord_y2) { diff --git a/source/net/yacy/visualization/PrintTool.java b/source/net/yacy/visualization/PrintTool.java index bae2f460c..a5dea67d3 100644 --- a/source/net/yacy/visualization/PrintTool.java +++ b/source/net/yacy/visualization/PrintTool.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -29,7 +29,7 @@ package net.yacy.visualization; public class PrintTool { - + private static long[] font = new long[]{ 0x00000000000000L,0x00300C03000030L,0x00CC3300000000L,0x00CCFFCCCFFCCCL,0x02FFCC2FE0CFFEL,0x00C3CF0FC3CF0CL,0x02FCE02ECCE2ECL,0x00300C00000000L, 0x00030380C03803L,0x0300B00C0B0000L,0x0000332BA03000L,0x00000C0FC0C000L,0x0000000302C0C0L,0x0000000FC00000L,0x00000000000030L,0x00030383838380L, @@ -84,7 +84,7 @@ public class PrintTool { } } } - + public static void print(final RasterPlotter matrix, final int x, final int y, final int angle, final String message, final int align) { // align = -1 : left // align = 1 : right @@ -107,8 +107,8 @@ public class PrintTool { else if (angle == 315) {xx += 6; yy += 6;} } } - - + + private static final int arcDist = 8; public static void arcPrint(final RasterPlotter matrix, final int cx, final int cy, final int radius, final int angle, final String message) { final int x = cx + (int) ((radius + 1) * Math.cos(RasterPlotter.PI180 * angle)); @@ -123,6 +123,6 @@ public class PrintTool { if ((angle < (90 - arcDist)) || (angle > (270 + arcDist))) xp = x; print(matrix, xp, yp, 0, message, -1); } - - + + }