boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added a visualization of peer pings to the performance graphic

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7837 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 3a191cdf14
commit 11dc653de3
17 changed files with 247 additions and 178 deletions
Show Stats Download Patch File Download Diff File

38
htroot/Network.java
Unescape View File

@ -32,6 +32,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@ -40,8 +41,6 @@ import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.MapTools;
//import de.anomic.http.client.Client;
import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
import de.anomic.server.serverObjects;
@ -52,7 +51,6 @@ import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyPeerActions;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyVersion;
import java.util.concurrent.ConcurrentMap;
public class Network {
@ -61,12 +59,12 @@ public class Network {
public static serverObjects respond(final RequestHeader requestHeader, final serverObjects post, final serverSwitch switchboard) {
final Switchboard sb = (Switchboard) switchboard;
final long start = System.currentTimeMillis();
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
prop.put("menu", post == null ? 2 : (post.get("menu", "").equals("embed")) ? 0 : (post.get("menu","").equals("simple")) ? 1 : 2);
if (sb.peers.mySeed() != null) prop.put("menu_newpeer_peerhash", sb.peers.mySeed().hash);
prop.setLocalized(!(requestHeader.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml"));
prop.putHTML("page_networkTitle", sb.getConfig("network.unit.description", "unspecified"));
prop.putHTML("page_networkName", sb.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"));
@ -98,7 +96,7 @@ public class Network {
if (sb.peers.mySeed() != null){ //our Peer
// update seed info
sb.updateMySeed();
final long LCount = seed.getLinkCount();
final long ICount = seed.getWordCount();
final long RCount = seed.getLong(yacySeed.RCOUNT, 0L);
@ -144,17 +142,17 @@ public class Network {
prop.putNum("table_my-seeds", seed.getLong(yacySeed.SCOUNT, 0L));
prop.putNum("table_my-connects", seed.getFloat(yacySeed.CCOUNT, 0F));
prop.put("table_my-url", seed.get(yacySeed.SEEDLISTURL, ""));
// generating the location string
prop.putHTML("table_my-location", ClientIdentification.generateLocation());
}
// overall results: Network statistics
if (iAmActive) conCount++; else if (mySeedType.equals(yacySeed.PEERTYPE_JUNIOR)) potCount++;
int activeLastMonth = sb.peers.sizeActiveSince(30 * 1440);
int activeLastWeek = sb.peers.sizeActiveSince(7 * 1440);
int activeLastDay = sb.peers.sizeActiveSince(1440);
int activeSwitch =
final int activeLastMonth = sb.peers.sizeActiveSince(30 * 1440);
final int activeLastWeek = sb.peers.sizeActiveSince(7 * 1440);
final int activeLastDay = sb.peers.sizeActiveSince(1440);
final int activeSwitch =
(activeLastDay <= conCount) ? 0 :
(activeLastWeek <= activeLastDay) ? 1 :
(activeLastMonth <= activeLastWeek) ? 2 : 3;
@ -182,7 +180,7 @@ public class Network {
} else if (post != null && post.getInt("page", 1) == 4) {
prop.put("table", 4); // triggers overview
prop.put("page", 4);
if (sb.peers.mySeed() != null) {
prop.put("table_my-hash", sb.peers.mySeed().hash );
prop.put("table_my-ip", sb.peers.mySeed().getIP() );
@ -203,7 +201,7 @@ public class Network {
yacySeed peer = new yacySeed(post.get("peerHash"), map);
sb.updateMySeed();
final int added = yacyClient.hello(sb.peers.mySeed(), sb.peers.peerActions, peer.getPublicAddress(), peer.hash);
final int added = yacyClient.hello(sb.peers.mySeed(), sb.peers.peerActions, peer.getPublicAddress(), peer.hash, peer.getName());
if (added <= 0) {
prop.put("table_comment",1);
@ -222,7 +220,7 @@ public class Network {
prop.putHTML("table_peerHash",post.get("peerHash"));
prop.putHTML("table_peerIP",post.get("peerIP"));
prop.putHTML("table_peerPort",post.get("peerPort"));
prop.putHTML("table_peerPort",post.get("peerPort"));
} else {
prop.put("table_peerHash","");
prop.put("table_peerIP","");
@ -234,7 +232,7 @@ public class Network {
// generate table
final int page = (post == null ? 1 : post.getInt("page", 1));
final int maxCount = (post == null ? 300 : post.getInt("maxCount", 300));
int conCount = 0;
int conCount = 0;
if (sb.peers == null) {
prop.put("table", 0);//no remote senior/principal proxies known"
} else {
@ -360,7 +358,7 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_hash", seed.hash);
String shortname = seed.get(yacySeed.NAME, "deadlink");
if (shortname.length() > 20) {
shortname = shortname.substring(0, 20) + "...";
shortname = shortname.substring(0, 20) + "...";
}
prop.putHTML(STR_TABLE_LIST + conCount + "_shortname", shortname);
prop.putHTML(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink"));
@ -408,9 +406,9 @@ public class Network {
// junior: red/green=direct or red/yellow=passive
prop.put(STR_TABLE_LIST + conCount + "_type_direct", seed.getFlagDirectConnect() ? 1 : 0);
}
if (page == 1) {
prop.put(STR_TABLE_LIST + conCount + "_acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0); // green=on or red=off
prop.put(STR_TABLE_LIST + conCount + "_acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0); // green=on or red=off
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive", seed.getFlagAcceptRemoteIndex() ? 1 : 0); // green=on or red=off
} else { // Passive, Potential Peers
if (seed.getFlagAcceptRemoteCrawl()) {
@ -452,7 +450,7 @@ public class Network {
prop.put("table", 1);
prop.putNum("table_num", conCount);
prop.putNum("table_total", ((page == 1) && (iAmActive)) ? (size + 1) : size );
prop.put("table_complete", ((complete)? 1 : 0) );
prop.put("table_complete", ((complete)? 1 : 0) );
}
}
prop.put("page", page);
@ -465,7 +463,7 @@ public class Network {
default: break;
}
}
prop.putNum("table_rt", System.currentTimeMillis() - start);
// return rewrite properties

4
htroot/PerformanceSearch_p.java
Unescape View File

@ -44,11 +44,11 @@ public class PerformanceSearch_p {
int c = 0;
if (events != null) {
EventTracker.Event event;
ProfilingGraph.searchEvent search;
ProfilingGraph.EventSearch search;
long lastt = 0;
while (events.hasNext()) {
event = events.next();
search = (ProfilingGraph.searchEvent) event.payload;
search = (ProfilingGraph.EventSearch) event.payload;
prop.put("table_" + c + "_query", search.queryID);
prop.put("table_" + c + "_event", search.processName.name());
prop.put("table_" + c + "_comment", search.comment);

47
htroot/yacy/hello.java
Unescape View File

@ -1,4 +1,4 @@
// hello.java
// hello.java
// -----------------------
// part of the AnomicHTTPD caching proxy
// (C) by Michael Peter Christen; mc@yacy.net
@ -37,7 +37,7 @@ import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.EventTracker;
import de.anomic.search.Switchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
@ -48,6 +48,7 @@ import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyVersion;
import de.anomic.yacy.dht.PeerSelection;
import de.anomic.yacy.graphics.ProfilingGraph;
public final class hello {
@ -57,7 +58,7 @@ public final class hello {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws InterruptedException {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
long start = System.currentTimeMillis();
final long start = System.currentTimeMillis();
prop.put("message", "none");
if ((post == null) || (env == null)) {
prop.put("message", "no post or no enviroment");
@ -67,18 +68,18 @@ public final class hello {
prop.put("message", "not in my network");
return prop;
}
// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer
// final String mytime = (String) post.get(MYTIME, ""); //
final String key = post.get("key", ""); // transmission key for response
final String seed = post.get("seed", "");
int count = post.getInt("count", 0);
long magic = post.getLong("magic", 0);
final long magic = post.getLong("magic", 0);
// final Date remoteTime = yacyCore.parseUniversalDate(post.get(MYTIME)); // read remote time
final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
long time = System.currentTimeMillis();
final InetAddress ias = Domains.dnsResolve(clientip);
long time_dnsResolve = System.currentTimeMillis() - time;
final long time_dnsResolve = System.currentTimeMillis() - time;
if (ias == null) {
yacyCore.log.logInfo("hello/server: failed contacting seed; clientip not resolvable (clientip=" + clientip + ", time_dnsResolve=" + time_dnsResolve + ")");
prop.put("message", "cannot resolve your IP from your reported location " + clientip);
@ -92,18 +93,18 @@ public final class hello {
yacySeed remoteSeed;
try {
remoteSeed = yacySeed.genRemoteSeed(seed, key, true, ias.getHostAddress());
} catch (IOException e) {
} catch (final IOException e) {
yacyCore.log.logInfo("hello/server: bad seed: " + e.getMessage() + ", time_dnsResolve=" + time_dnsResolve);
prop.put("message", "bad seed: " + e.getMessage());
return prop;
}
if (remoteSeed == null || remoteSeed.hash == null) {
yacyCore.log.logInfo("hello/server: bad seed: null, time_dnsResolve=" + time_dnsResolve);
prop.put("message", "cannot parse your seed");
return prop;
}
// final String properTest = remoteSeed.isProper();
// The remote peer might not know its IP yet, so don't abort if the IP check fails
// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
@ -119,12 +120,12 @@ public final class hello {
prop.put("message", "I am robinson, I do not answer");
return prop;
}
long[] callback = new long[]{-1, -1};
if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress(sb.clusterhashes.get(remoteSeed.hash.getBytes()));
// if the remote client has reported its own IP address and the client supports
// the port forwarding feature (if client version >= 0.383) then we try to
// the port forwarding feature (if client version >= 0.383) then we try to
// connect to the reported IP address first
time = System.currentTimeMillis();
long time_backping = 0;
@ -132,9 +133,9 @@ public final class hello {
if (reportedip.length() > 0 &&
!clientip.equals(reportedip) &&
clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING &&
magic != 0) {
magic != 0) {
serverCore.checkInterruption();
// try first the reportedip, since this may be a connect from a port-forwarding host
prop.put("yourip", reportedip);
remoteSeed.setIP(reportedip);
@ -158,7 +159,7 @@ public final class hello {
if (isNotLocal) {
serverCore.checkInterruption();
prop.put("yourip", clientip);
remoteSeed.setIP(clientip);
callback = yacyClient.queryUrlCount(remoteSeed);
@ -170,8 +171,9 @@ public final class hello {
// System.out.println("YACYHELLO: YOUR IP=" + clientip);
// set lastseen value (we have seen that peer, it contacted us!)
remoteSeed.setLastSeenUTC();
// assign status
final int connectedBefore = sb.peers.sizeConnected();
if (callback[0] >= 0) {
if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
@ -194,6 +196,11 @@ public final class hello {
sb.peers.peerActions.peerPing(remoteSeed);
}
}
final int connectedAfter = sb.peers.sizeConnected();
// update event tracker
EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false);
sb.peers.peerActions.setUserAgent(clientip, userAgent);
if (!(prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) {
yacyCore.log.logInfo("hello/server: changing remote peer '" + remoteSeed.getName() +
@ -208,14 +215,14 @@ public final class hello {
if (sb.peers.sizeConnected() > 0) {
if (count > sb.peers.sizeConnected()) { count = sb.peers.sizeConnected(); }
if (count > 100) { count = 100; }
// latest seeds
final Map<String, yacySeed> ySeeds = PeerSelection.seedsByAge(sb.peers, true, count); // peerhash/yacySeed relation
// attach also my own seed
seeds.append("seed0=").append(sb.peers.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
count = 1;
count = 1;
// attach other seeds
if (ySeeds != null) {
seeds.ensureCapacity((ySeeds.size() + 1) * 768);

6
htroot/yacy/search.java
Unescape View File

@ -250,7 +250,7 @@ public final class search {
//final Map<byte[], ReferenceContainer<WordReference>>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls));
final TreeMap<byte[], ReferenceContainer<WordReference>> incc = indexSegment.termIndex().searchConjunction(theQuery.queryHashes, QueryParams.hashes2Handles(urls));
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.COLLECTION, "", incc.size(), System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.COLLECTION, "", incc.size(), System.currentTimeMillis() - timer), false);
if (incc != null) {
final Iterator<Map.Entry<byte[], ReferenceContainer<WordReference>>> ci = incc.entrySet().iterator();
Map.Entry<byte[], ReferenceContainer<WordReference>> entry;
@ -373,7 +373,7 @@ public final class search {
i++;
}
prop.put("references", (refstr.length() > 0) ? refstr.substring(1) : refstr.toString());
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.REFERENCECOLLECTION, "", i, System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.REFERENCECOLLECTION, "", i, System.currentTimeMillis() - timer), false);
}
prop.put("indexabstract", indexabstract.toString());
@ -401,7 +401,7 @@ public final class search {
theQuery.transmitcount = accu.size() + 1;
prop.put("links", links.toString());
prop.put("linkcount", accu.size());
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.RESULTLIST, "", accu.size(), System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.RESULTLIST, "", accu.size(), System.currentTimeMillis() - timer), false);
}
// add information about forward peers

2
htroot/yacysearch.java
Unescape View File

@ -604,7 +604,7 @@ public class yacysearch {
header.get(RequestHeader.USER_AGENT, ""),
sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false);
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(3000);

2
htroot/yacysearchitem.java
Unescape View File

@ -186,7 +186,7 @@ public class yacysearchitem {
}
prop.put("content_heuristic_name", heuristic.heuristicName);
}
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.FINALIZATION, "" + item, 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.FINALIZATION, "" + item, 0, 0), false);
final String ext = resultURL.getFileExtension().toLowerCase();
if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) {
final String license = sb.licensedURLs.aquireLicense(resultURL);

2
htroot/yacysearchtrailer.java
Unescape View File

@ -240,7 +240,7 @@ public class yacysearchtrailer {
prop.put("cat-location_queryenc", theQuery.queryString(true).replace(' ', '+'));
}
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.FINALIZATION, "bottomline", 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.FINALIZATION, "bottomline", 0, 0), false);
return prop;
}

6
source/de/anomic/search/RankingProcess.java
Unescape View File

@ -149,7 +149,7 @@ public final class RankingProcess extends Thread {
this.query.maxDistance);
this.localSearchInclusion = search.inclusion();
final ReferenceContainer<WordReference> index = search.joined();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.JOIN, this.query.queryString, index.size(), System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.JOIN, this.query.queryString, index.size(), System.currentTimeMillis() - timer), false);
if (index.isEmpty()) {
return;
}
@ -188,7 +188,7 @@ public final class RankingProcess extends Thread {
// normalize entries
final BlockingQueue<WordReferenceVars> decodedEntries = this.order.normalizeWith(index);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.NORMALIZING, resourceName, index.size(), System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.NORMALIZING, resourceName, index.size(), System.currentTimeMillis() - timer), false);
// iterate over normalized entries and select some that are better than currently stored
timer = System.currentTimeMillis();
@ -279,7 +279,7 @@ public final class RankingProcess extends Thread {
}
//if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.PRESORT, resourceName, index.size(), System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.PRESORT, resourceName, index.size(), System.currentTimeMillis() - timer), false);
}
/**

10
source/de/anomic/search/ResultFetcher.java
Unescape View File

@ -111,7 +111,7 @@ public class ResultFetcher {
// start worker threads to fetch urls and snippets
this.workerThreads = null;
deployWorker(Math.min(10, query.itemsPerPage), query.neededResults());
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), SearchEvent.Type.SNIPPETFETCH_START, ((this.workerThreads == null) ? "no" : this.workerThreads.length) + " online snippet fetch threads started", 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), SearchEvent.Type.SNIPPETFETCH_START, ((this.workerThreads == null) ? "no" : this.workerThreads.length) + " online snippet fetch threads started", 0, 0), false);
}
public void setCleanupState() {
@ -132,11 +132,11 @@ public class ResultFetcher {
if (!this.query.isLocal() && item == 0) try { Thread.sleep(100); } catch (final InterruptedException e1) {} // wait a little time to get first results in the search
final long finishTime = System.currentTimeMillis() + timeout;
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.ONERESULT, "started, item = " + item + ", available = " + this.result.sizeAvailable(), 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.ONERESULT, "started, item = " + item + ", available = " + this.result.sizeAvailable(), 0, 0), false);
if (this.result.sizeAvailable() > item) {
// we have the wanted result already in the result array .. return that
final ResultEntry re = this.result.element(item).getElement();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.ONERESULT, "prefetched, item = " + item + ", available = " + this.result.sizeAvailable() + ": " + re.urlstring(), 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.ONERESULT, "prefetched, item = " + item + ", available = " + this.result.sizeAvailable() + ": " + re.urlstring(), 0, 0), false);
return re;
}
@ -155,11 +155,11 @@ public class ResultFetcher {
// finally, if there is something, return the result
if (entry == null) {
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.ONERESULT, "not found, item = " + item + ", available = " + this.result.sizeAvailable(), 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.ONERESULT, "not found, item = " + item + ", available = " + this.result.sizeAvailable(), 0, 0), false);
return null;
}
final ResultEntry re = entry.getElement();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(this.query.id(true), SearchEvent.Type.ONERESULT, "retrieved, item = " + item + ", available = " + this.result.sizeAvailable() + ": " + re.urlstring(), 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.ONERESULT, "retrieved, item = " + item + ", available = " + this.result.sizeAvailable() + ": " + re.urlstring(), 0, 0), false);
return re;
}

6
source/de/anomic/search/SearchEvent.java
Unescape View File

@ -150,7 +150,7 @@ public final class SearchEvent {
if (this.primarySearchThreads != null) {
Log.logFine("SEARCH_EVENT", "STARTING " + this.primarySearchThreads.length + " THREADS TO CATCH EACH " + remote_maxcount + " URLs");
this.rankingProcess.moreFeeders(this.primarySearchThreads.length);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false);
// finished searching
Log.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + this.primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
} else {
@ -189,7 +189,7 @@ public final class SearchEvent {
this.IACount.put(wordhash, LargeNumberCache.valueOf(container.size()));
this.IAResults.put(wordhash, WordReferenceFactory.compressIndex(container, null, 1000).toString());
}
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false);
} else {
this.rankingProcess.start(); // start concurrently
// but give process time to accumulate a certain amount of data
@ -208,7 +208,7 @@ public final class SearchEvent {
// clean up events
SearchEventCache.cleanupEvents(false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), Type.CLEANUP, "", 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.CLEANUP, "", 0, 0), false);
// store this search to a cache so it can be re-used
if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true);

54
source/de/anomic/yacy/graphics/ProfilingGraph.java
Unescape View File

@ -84,12 +84,13 @@ public class ProfilingGraph {
}
chart.declareDimension(ChartPlotter.DIMENSION_ANOT0, anotscale, vspace * anotscale / maxppm, 0, "008800", null , "PPM [PAGES/MINUTE]");
chart.declareDimension(ChartPlotter.DIMENSION_ANOT1, vspace / 6, vspace / 6, 0, "888800", null , "URL");
chart.declareDimension(ChartPlotter.DIMENSION_ANOT2, 1, 1, 0, "888800", null , "PING");
// draw chart
long time;
final long now = System.currentTimeMillis();
long bytes;
int x0, x1, y0, y1, ppm, words;
int x0, x1, y0, y1;
try {
// draw urls
/*
@ -134,6 +135,7 @@ public class ProfilingGraph {
x0 = 1; y0 = 0;
if (events != null) {
EventTracker.Event event;
int words;
while (events.hasNext()) {
event = events.next();
time = event.time - now;
@ -153,6 +155,7 @@ public class ProfilingGraph {
x0 = 1; y0 = 0;
if (events != null) {
EventTracker.Event event;
int ppm;
while (events.hasNext()) {
event = events.next();
time = event.time - now;
@ -167,6 +170,24 @@ public class ProfilingGraph {
}
}
// draw peer ping
events = EventTracker.getHistory(EventTracker.EClass.PEERPING);
x0 = 1; y0 = 0;
if (events != null) {
EventTracker.Event event;
EventPing ping;
while (events.hasNext()) {
event = events.next();
time = event.time - now;
ping = (EventPing) event.payload;
x1 = (int) (time/1000);
y1 = Math.abs((ping.outgoing ? ping.toPeer : ping.fromPeer).hashCode()) % vspace;
chart.setColor("444444");
chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT2, x1, y1, 2, "PING " + ping.fromPeer.toUpperCase() + " -> " + ping.toPeer.toUpperCase() + (ping.newPeers > 0 ? "(+" + ping.newPeers + ")" : ""), 0);
x0 = x1; y0 = y1;
}
}
bufferChart = chart;
} catch (final ConcurrentModificationException cme) {
chart = bufferChart;
@ -175,14 +196,14 @@ public class ProfilingGraph {
return chart;
}
public static class searchEvent {
public static class EventSearch {
public SearchEvent.Type processName;
public String comment;
public String queryID;
public long duration;
public int resultCount;
public searchEvent(final String queryID, final SearchEvent.Type processName, final String comment, final int resultCount, final long duration) {
public EventSearch(final String queryID, final SearchEvent.Type processName, final String comment, final int resultCount, final long duration) {
this.queryID = queryID;
this.processName = processName;
this.comment = comment;
@ -191,4 +212,31 @@ public class ProfilingGraph {
}
}
public static class EventDHT {
public String fromPeer, toPeer;
public boolean outgoing;
public int totalReferences, newReferences;
public EventDHT(final String fromPeer, final String toPeer, final boolean outgoing, final int totalReferences, final int newReferences) {
this.fromPeer = fromPeer;
this.toPeer = toPeer;
this.outgoing = outgoing;
this.totalReferences = totalReferences;
this.newReferences = newReferences;
}
}
public static class EventPing {
public String fromPeer, toPeer;
public boolean outgoing;
public int newPeers;
public EventPing(final String fromPeer, final String toPeer, final boolean outgoing, final int newPeers) {
this.fromPeer = fromPeer;
this.toPeer = toPeer;
this.outgoing = outgoing;
this.newPeers = newPeers;
}
}
}

10
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -87,6 +87,7 @@ import net.yacy.kelondro.rwi.Reference;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.rwi.ReferenceContainerCache;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;
@ -104,6 +105,7 @@ import de.anomic.search.Switchboard;
import de.anomic.search.TextSnippet;
import de.anomic.server.serverCore;
import de.anomic.tools.crypt;
import de.anomic.yacy.graphics.ProfilingGraph;
import de.anomic.yacy.graphics.WebStructureGraph;
import de.anomic.yacy.graphics.WebStructureGraph.HostReference;
@ -143,7 +145,7 @@ public final class yacyClient {
*
* @return the number of new seeds
*/
public static int hello(final yacySeed mySeed, final yacyPeerActions peerActions, final String address, final String otherHash) {
public static int hello(final yacySeed mySeed, final yacyPeerActions peerActions, final String address, final String otherHash, final String otherName) {
Map<String, String> result = null;
final String salt = crypt.randomSalt();
@ -256,6 +258,7 @@ public final class yacyClient {
int count = 0;
String seedStr;
yacySeed s;
final int connectedBefore = peerActions.sizeConnected();
while ((seedStr = result.get("seed" + i++)) != null) {
// integrate new seed into own database
// the first seed, "seed0" is the seed of the responding peer
@ -277,6 +280,11 @@ public final class yacyClient {
}
}
}
final int connectedAfter = peerActions.sizeConnected();
// update event tracker
EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(mySeed.getName(), otherName, true, connectedAfter - connectedBefore), false);
return count;
}

106
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -80,7 +80,7 @@ public class yacyCore {
private static final int PING_MIN_DBSIZE = 5;
private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior
private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds
// public static yacyShare shareManager = null;
// public static boolean terminate = false;
@ -97,11 +97,11 @@ public class yacyCore {
this.sb = sb;
sb.setConfig("yacyStatus", "");
// create a peer news channel
final RSSFeed peernews = yacyChannel.channels(yacyChannel.PEERNEWS);
peernews.addMessage(new RSSMessage("YaCy started", "", ""));
// ensure that correct IP is used
final String staticIP = sb.getConfig("staticIP", "");
if (staticIP.length() != 0 && yacySeed.isProperIP(staticIP) == null) {
@ -124,7 +124,7 @@ public class yacyCore {
synchronized static public void triggerOnlineAction() {
lastOnlineTime = System.currentTimeMillis();
}
public final void publishSeedList() {
if (log.isFine()) log.logFine("yacyCore.publishSeedList: Triggered Seed Publish");
@ -137,11 +137,11 @@ public class yacyCore {
yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself");
*/
if ((sb.peers.lastSeedUpload_myIP.equals(sb.peers.mySeed().getIP())) &&
(sb.peers.lastSeedUpload_seedDBSize == sb.peers.sizeConnected()) &&
if ((this.sb.peers.lastSeedUpload_myIP.equals(this.sb.peers.mySeed().getIP())) &&
(this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected()) &&
(canReachMyself()) &&
(System.currentTimeMillis() - sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) &&
(sb.peers.mySeed().isPrincipal())
(System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) &&
(this.sb.peers.mySeed().isPrincipal())
) {
if (log.isFine()) log.logFine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP.");
return;
@ -164,7 +164,7 @@ public class yacyCore {
}
}
// we want to be a principal...
saveSeedList(sb);
saveSeedList(this.sb);
} else {
if (seedUploadMethod.equals("")) {
this.sb.setConfig("seedUploadMethod", "none");
@ -175,7 +175,7 @@ public class yacyCore {
}
public final void peerPing() {
if ((sb.isRobinsonMode()) && (sb.getConfig("cluster.mode", "").equals("privatepeer"))) {
if ((this.sb.isRobinsonMode()) && (this.sb.getConfig("cluster.mode", "").equals("privatepeer"))) {
// in case this peer is a privat peer we omit the peer ping
// all other robinson peer types do a peer ping:
// the privatecluster does the ping to the other cluster members
@ -185,34 +185,34 @@ public class yacyCore {
}
// before publishing, update some seed data
sb.updateMySeed();
this.sb.updateMySeed();
// publish own seed to other peer, this can every peer, but makes only sense for senior peers
if (sb.peers.sizeConnected() == 0) {
if (this.sb.peers.sizeConnected() == 0) {
// reload the seed lists
sb.loadSeedLists();
log.logInfo("re-initialized seed list. received " + sb.peers.sizeConnected() + " new peer(s)");
this.sb.loadSeedLists();
log.logInfo("re-initialized seed list. received " + this.sb.peers.sizeConnected() + " new peer(s)");
}
final int newSeeds = publishMySeed(false);
if (newSeeds > 0) {
log.logInfo("received " + newSeeds + " new peer(s), know a total of " + sb.peers.sizeConnected() + " different peers");
log.logInfo("received " + newSeeds + " new peer(s), know a total of " + this.sb.peers.sizeConnected() + " different peers");
}
}
private boolean canReachMyself() { // TODO: check if this method is necessary - depending on the used router it will not work
// returns true if we can reach ourself under our known peer address
// if we cannot reach ourself, we call a forced publishMySeed and return false
final long[] callback = yacyClient.queryUrlCount(sb.peers.mySeed());
final long[] callback = yacyClient.queryUrlCount(this.sb.peers.mySeed());
if (callback[0] >= 0 && callback[1] == magic) {
sb.peers.mySeed().setLastSeenUTC();
this.sb.peers.mySeed().setLastSeenUTC();
return true;
}
log.logInfo("re-connect own seed");
final String oldAddress = sb.peers.mySeed().getPublicAddress();
final String oldAddress = this.sb.peers.mySeed().getPublicAddress();
/*final int newSeeds =*/ publishMySeed(true);
return (oldAddress != null && oldAddress.equals(sb.peers.mySeed().getPublicAddress()));
return (oldAddress != null && oldAddress.equals(this.sb.peers.mySeed().getPublicAddress()));
}
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
@ -236,24 +236,24 @@ public class yacyCore {
public final void run() {
try {
this.added = yacyClient.hello(sb.peers.mySeed(), sb.peers.peerActions, seed.getClusterAddress(), seed.hash);
this.added = yacyClient.hello(yacyCore.this.sb.peers.mySeed(), yacyCore.this.sb.peers.peerActions, this.seed.getClusterAddress(), this.seed.hash, this.seed.getName());
if (this.added < 0) {
// no or wrong response, delete that address
final String cause = "peer ping to peer resulted in error response (added < 0)";
log.logInfo("publish: disconnected " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getPublicAddress() + ": " + cause);
sb.peers.peerActions.peerDeparture(this.seed, cause);
yacyCore.this.sb.peers.peerActions.peerDeparture(this.seed, cause);
} else {
// success! we have published our peer to a senior peer
// update latest news from the other peer
log.logInfo("publish: handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress());
// check if seed's lastSeen has been updated
final yacySeed newSeed = sb.peers.getConnected(this.seed.hash);
final yacySeed newSeed = yacyCore.this.sb.peers.getConnected(this.seed.hash);
if (newSeed != null) {
if (!newSeed.isOnline()) {
if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) +
" peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " is not online." +
" Removing Peer from connected");
sb.peers.peerActions.peerDeparture(newSeed, "peer not online");
yacyCore.this.sb.peers.peerActions.peerDeparture(newSeed, "peer not online");
} else
if (newSeed.getLastSeenUTC() < (System.currentTimeMillis() - 10000)) {
// update last seed date
@ -262,14 +262,14 @@ public class yacyCore {
" peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" +
my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "'");
newSeed.setLastSeenUTC();
sb.peers.peerActions.peerArrival(newSeed, true);
yacyCore.this.sb.peers.peerActions.peerArrival(newSeed, true);
} else {
if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) +
" peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" +
my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" +
my_SHORT_SECOND_FORMATTER.format(new Date(this.seed.getLastSeenUTC())) + "'");
this.seed.setLastSeenUTC();
sb.peers.peerActions.peerArrival(this.seed, true);
yacyCore.this.sb.peers.peerActions.peerArrival(this.seed, true);
}
}
} else {
@ -277,7 +277,7 @@ public class yacyCore {
}
}
} catch (final Exception e) {
log.logSevere("publishThread: error with target seed " + seed.toString() + ": " + e.getMessage(), e);
log.logSevere("publishThread: error with target seed " + this.seed.toString() + ": " + e.getMessage(), e);
} finally {
this.syncList.add(this);
this.sync.release();
@ -306,13 +306,13 @@ public class yacyCore {
// init yacyHello-process
Map<String, yacySeed> seeds; // hash/yacySeed relation
int attempts = sb.peers.sizeConnected();
int attempts = this.sb.peers.sizeConnected();
// getting a list of peers to contact
if (sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (this.sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (attempts > PING_INITIAL) { attempts = PING_INITIAL; }
final Map<byte[], String> ch = Switchboard.getSwitchboard().clusterhashes;
seeds = PeerSelection.seedsByAge(sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
seeds = PeerSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
if (ch != null) {
final Iterator<Map.Entry<byte[], String>> i = ch.entrySet().iterator();
@ -324,7 +324,7 @@ public class yacyCore {
hash = ASCII.String(entry.getKey());
seed = seeds.get(hash);
if (seed == null) {
seed = sb.peers.get(hash);
seed = this.sb.peers.get(hash);
if (seed == null) continue;
}
seed.setAlternativeAddress(entry.getValue());
@ -339,7 +339,7 @@ public class yacyCore {
} else {
if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; }
}
seeds = PeerSelection.seedsByAge(sb.peers, false, attempts); // best for seed list maintenance/cleaning
seeds = PeerSelection.seedsByAge(this.sb.peers, false, attempts); // best for seed list maintenance/cleaning
}
if (seeds == null || seeds.isEmpty()) { return 0; }
@ -351,16 +351,16 @@ public class yacyCore {
// include a YaCyNews record to my seed
try {
final yacyNewsDB.Record record = sb.peers.newsPool.myPublication();
final yacyNewsDB.Record record = this.sb.peers.newsPool.myPublication();
if (record == null) {
sb.peers.mySeed().put("news", "");
this.sb.peers.mySeed().put("news", "");
} else {
sb.peers.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
this.sb.peers.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
}
} catch (final Exception e) {
log.logSevere("publishMySeed: problem with news encoding", e);
}
sb.peers.mySeed().setUnusedFlags();
this.sb.peers.mySeed().setUnusedFlags();
int newSeeds = -1;
//if (seeds.length > 1) {
// holding a reference to all started threads
@ -383,7 +383,7 @@ public class yacyCore {
final String seederror = seed.isProper(false);
if ((address == null) || (seederror != null)) {
// we don't like that address, delete it
sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror);
this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror);
sync.acquire();
} else {
// starting a new publisher thread
@ -448,7 +448,7 @@ public class yacyCore {
if ((accessible >= PING_MIN_PEERSEEN) ||
(accessible >= notaccessible)) {
// We can be reached from a majority of other Peers
if (sb.peers.mySeed().isPrincipal()) {
if (this.sb.peers.mySeed().isPrincipal()) {
newPeerType = yacySeed.PEERTYPE_PRINCIPAL;
} else {
newPeerType = yacySeed.PEERTYPE_SENIOR;
@ -457,38 +457,38 @@ public class yacyCore {
// We cannot be reached from the outside
newPeerType = yacySeed.PEERTYPE_JUNIOR;
}
if (sb.peers.mySeed().orVirgin().equals(newPeerType)) {
log.logInfo("PeerPing: myType is " + sb.peers.mySeed().orVirgin());
if (this.sb.peers.mySeed().orVirgin().equals(newPeerType)) {
log.logInfo("PeerPing: myType is " + this.sb.peers.mySeed().orVirgin());
} else {
log.logInfo("PeerPing: changing myType from '" + sb.peers.mySeed().orVirgin() + "' to '" + newPeerType + "'");
sb.peers.mySeed().put(yacySeed.PEERTYPE, newPeerType);
log.logInfo("PeerPing: changing myType from '" + this.sb.peers.mySeed().orVirgin() + "' to '" + newPeerType + "'");
this.sb.peers.mySeed().put(yacySeed.PEERTYPE, newPeerType);
}
} else {
log.logInfo("PeerPing: No data, staying at myType: " + sb.peers.mySeed().orVirgin());
log.logInfo("PeerPing: No data, staying at myType: " + this.sb.peers.mySeed().orVirgin());
}
// success! we have published our peer to a senior peer
// update latest news from the other peer
// log.logInfo("publish: handshaked " + t.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + t.seed.getName() + "' at " + t.seed.getAddress());
sb.peers.saveMySeed();
this.sb.peers.saveMySeed();
// if we have an address, we do nothing
if (sb.peers.mySeed().isProper(true) == null && !force) { return 0; }
if (this.sb.peers.mySeed().isProper(true) == null && !force) { return 0; }
if (newSeeds > 0) return newSeeds;
// still no success: ask own NAT or internet responder
//final boolean DI604use = switchboard.getConfig("DI604use", "false").equals("true");
//final String DI604pw = switchboard.getConfig("DI604pw", "");
final String ip = sb.getConfig("staticIP", "");
final String ip = this.sb.getConfig("staticIP", "");
//if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw);
// yacyCore.log.logDebug("DEBUG: new IP=" + ip);
if (yacySeed.isProperIP(ip) == null) sb.peers.mySeed().setIP(ip);
if (sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ???????????????
sb.peers.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
if (yacySeed.isProperIP(ip) == null) this.sb.peers.mySeed().setIP(ip);
if (this.sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ???????????????
this.sb.peers.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
log.logInfo("publish: no recipient found, our address is " +
((sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : sb.peers.mySeed().getPublicAddress()));
sb.peers.saveMySeed();
((this.sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : this.sb.peers.mySeed().getPublicAddress()));
this.sb.peers.saveMySeed();
return 0;
} catch (final InterruptedException e) {
try {

62
source/de/anomic/yacy/yacyPeerActions.java
Unescape View File

@ -1,4 +1,4 @@
// yacyPeerActions.java
// yacyPeerActions.java
// -------------------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://yacy.net
@ -34,12 +34,12 @@ import net.yacy.kelondro.util.MapTools;
public class yacyPeerActions {
private final yacySeedDB seedDB;
private Map<String, String> userAgents;
public long disconnects;
private final yacyNewsPool newsPool;
public yacyPeerActions(final yacySeedDB seedDB, final yacyNewsPool newsPool) {
this.seedDB = seedDB;
this.newsPool = newsPool;
@ -49,10 +49,10 @@ public class yacyPeerActions {
public void close() {
// the seedDB and newsPool should be cleared elsewhere
if (userAgents != null) userAgents.clear();
userAgents = null;
if (this.userAgents != null) this.userAgents.clear();
this.userAgents = null;
}
public boolean connectPeer(final yacySeed seed, final boolean direct) {
// store a remote peer's seed
// returns true if the peer is new and previously unknown
@ -113,7 +113,7 @@ public class yacyPeerActions {
// disconnection time
long dtimeUTC0;
final yacySeed disconnectedSeed = seedDB.getDisconnected(seed.hash);
final yacySeed disconnectedSeed = this.seedDB.getDisconnected(seed.hash);
if (disconnectedSeed == null) {
dtimeUTC0 = 0; // never disconnected: virtually disconnected maximum time ago
} else {
@ -173,12 +173,12 @@ public class yacyPeerActions {
return false;
}
if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress());
seedDB.addConnected(seed);
this.seedDB.addConnected(seed);
return true;
}
// the seed is new
if ((seedDB.mySeedIsDefined()) && (seed.getIP().equals(this.seedDB.mySeed().getIP()))) {
if ((this.seedDB.mySeedIsDefined()) && (seed.getIP().equals(this.seedDB.mySeed().getIP()))) {
// seed from the same IP as the calling client: can be
// the case if there runs another one over a NAT
if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: saved NEW seed (myself IP) " + seed.getPublicAddress());
@ -195,33 +195,33 @@ public class yacyPeerActions {
final boolean res = connectPeer(peer, direct);
if (res) {
// perform all actions if peer is effective new
this.processPeerArrival(peer);
processPeerArrival(peer);
yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " joined the network", "", ""));
}
return res;
}
public void peerDeparture(final yacySeed peer, final String cause) {
if (peer == null) return;
// we do this if we did not get contact with the other peer
if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: no contact to a " + peer.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress() + ". Cause: " + cause);
synchronized (seedDB) {
if (!seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { disconnects++; }
synchronized (this.seedDB) {
if (!this.seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { this.disconnects++; }
peer.put("dct", Long.toString(System.currentTimeMillis()));
seedDB.addDisconnected(peer); // update info
this.seedDB.addDisconnected(peer); // update info
}
yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " left the network", "", ""));
}
public void peerPing(final yacySeed peer) {
if (peer == null) return;
// this is called only if the peer has junior status
seedDB.addPotential(peer);
this.seedDB.addPotential(peer);
// perform all actions
processPeerArrival(peer);
yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " sent me a ping", "", ""));
}
private void processPeerArrival(final yacySeed peer) {
final String recordString = peer.get("news", null);
//System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached"));
@ -243,14 +243,18 @@ public class yacyPeerActions {
}
}
}
public int sizeConnected() {
return this.seedDB.sizeConnected();
}
public void setUserAgent(final String IP, final String userAgent) {
if (userAgents == null) return; // case can happen during shutdown
userAgents.put(IP, userAgent);
if (this.userAgents == null) return; // case can happen during shutdown
this.userAgents.put(IP, userAgent);
}
public String getUserAgent(final String IP) {
final String userAgent = userAgents.get(IP);
final String userAgent = this.userAgents.get(IP);
return (userAgent == null) ? "" : userAgent;
}
@ -261,22 +265,22 @@ public class yacyPeerActions {
public static String formatInterval(final long millis) {
try {
final long mins = millis / 60000;
final StringBuilder uptime = new StringBuilder(40);
final int uptimeDays = (int) (Math.floor(mins/1440.0));
final int uptimeHours = (int) (Math.floor(mins/60.0)%24);
final int uptimeMins = (int) mins%60;
uptime.append(uptimeDays)
.append(((uptimeDays == 1)?" day ":" days "))
.append((uptimeHours < 10)?"0":"")
.append(uptimeHours)
.append(':')
.append((uptimeMins < 10)?"0":"")
.append(uptimeMins);
return uptime.toString();
.append(uptimeMins);
return uptime.toString();
} catch (final Exception e) {
return "unknown";
}

40
source/net/yacy/kelondro/util/EventTracker.java
Unescape View File

@ -34,59 +34,61 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import de.anomic.yacy.graphics.ProfilingGraph;
public class EventTracker {
private final static int maxQueueSize = 30000;
private final static long maxQueueAge = ProfilingGraph.maxTime;
public enum EClass {
WORDCACHE,
MEMORY,
PPM,
PEERPING,
DHT,
INDEX,
SEARCH;
}
private final static Map<EClass, ConcurrentLinkedQueue<Event>> historyMaps = new ConcurrentHashMap<EClass, ConcurrentLinkedQueue<Event>>();
private final static Map<EClass, Long> eventAccess = new ConcurrentHashMap<EClass, Long>(); // value: last time when this was accessed
public final static void delete(final EClass eventName) {
historyMaps.remove(eventName);
eventAccess.remove(eventName);
}
public final static void update(final EClass eventName, final Object eventPayload, boolean useProtection) {
public final static void update(final EClass eventName, final Object eventPayload, final boolean useProtection) {
// check protection against too heavy access
if (useProtection) {
Long lastAcc = eventAccess.get(eventName);
final Long lastAcc = eventAccess.get(eventName);
if (lastAcc == null) {
eventAccess.put(eventName, Long.valueOf(System.currentTimeMillis()));
} else {
long time = System.currentTimeMillis();
final long time = System.currentTimeMillis();
if (time - lastAcc.longValue() < 1000) {
return; // protect against too heavy load
}
eventAccess.put(eventName, Long.valueOf(time));
}
}
// get event history container
ConcurrentLinkedQueue<Event> history = historyMaps.get(eventName);
// create history
if (history == null) {
history = new ConcurrentLinkedQueue<Event>();
// update entry
history.offer(new Event(eventPayload));
// store map
historyMaps.put(eventName, history);
return;
}
// update history
history.offer(new Event(eventPayload));
// clean up too old entries
int tp = history.size() - maxQueueSize;
while (tp-- > 0) history.poll();
@ -104,24 +106,24 @@ public class EventTracker {
}
}
}
public final static Iterator<Event> getHistory(final EClass eventName) {
ConcurrentLinkedQueue<Event> list = historyMaps.get(eventName);
final ConcurrentLinkedQueue<Event> list = historyMaps.get(eventName);
if (list == null) return null;
return list.iterator();
}
public final static int countEvents(final EClass eventName, long time) {
Iterator<Event> event = getHistory(eventName);
public final static int countEvents(final EClass eventName, final long time) {
final Iterator<Event> event = getHistory(eventName);
if (event == null) return 0;
long now = System.currentTimeMillis();
final long now = System.currentTimeMillis();
int count = 0;
while (event.hasNext()) {
if (now - event.next().time < time) count++;
}
return count;
}
public final static class Event {
public Object payload;
public long time;

16
source/net/yacy/visualization/ChartPlotter.java
Unescape View File

@ -40,17 +40,19 @@ public class ChartPlotter extends RasterPlotter {
public static final int DIMENSION_BOTTOM = 3;
public static final int DIMENSION_ANOT0 = 4;
public static final int DIMENSION_ANOT1 = 5;
public static final int DIMENSION_ANOT2 = 6;
public static final int DIMENSION_ANOT3 = 7;
private final int leftborder;
private final int rightborder;
private final int topborder;
private final int bottomborder;
private final int[] scales = new int[]{0,0,0,0,0,0};
private final int[] pixels = new int[]{0,0,0,0,0,0};
private final int[] offsets = new int[]{0,0,0,0,0,0};
private final String[] colnames = new String[]{"FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF"};
private final String[] colscale = new String[]{null,null,null,null,null,null};
private final String[] tablenames = new String[]{"","","","","",""};
private final int[] scales = new int[]{0,0,0,0,0,0,0,0};
private final int[] pixels = new int[]{0,0,0,0,0,0,0,0};
private final int[] offsets = new int[]{0,0,0,0,0,0,0,0};
private final String[] colnames = new String[]{"FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF","FFFFFF"};
private final String[] colscale = new String[]{null,null,null,null,null,null,null,null};
private final String[] tablenames = new String[]{"","","","","","","",""};
public ChartPlotter(final int width, final int height, final String backgroundColor, final String foregroundColor, final String lightColor,
final int leftborder, final int rightborder, final int topborder, final int bottomborder,
@ -93,7 +95,7 @@ public class ChartPlotter extends RasterPlotter {
final int y = (coord_y - this.offsets[dimension_y]) * this.pixels[dimension_y] / this.scales[dimension_y];
if (dotsize == 1) plot(this.leftborder + x, this.height - this.bottomborder - y, 100);
else dot(this.leftborder + x, this.height - this.bottomborder - y, dotsize, true, 100);
if (anot != null) PrintTool.print(this, this.leftborder + x + dotsize + 2 + ((anotAngle == 315) ? -9 : 0), this.height - this.bottomborder - y + ((anotAngle == 315) ? -3 : 0), anotAngle, anot, (anotAngle == 0) ? -1 : ((anotAngle == 315) ? 1 : 0));
if (anot != null) PrintTool.print(this, this.leftborder + x + dotsize + 2 + ((anotAngle == 315) ? -9 : 0), this.height - this.bottomborder - y + ((anotAngle == 315) ? -3 : 0), anotAngle, anot, (anotAngle == 0) ? (anot.length() * 6 + x > this.width ? 1 : -1) : ((anotAngle == 315) ? 1 : 0));
}
public void chartLine(final int dimension_x, final int dimension_y, final int coord_x1, final int coord_y1, final int coord_x2, final int coord_y2) {

14
source/net/yacy/visualization/PrintTool.java
Unescape View File

@ -9,7 +9,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -29,7 +29,7 @@ package net.yacy.visualization;
public class PrintTool {
private static long[] font = new long[]{
0x00000000000000L,0x00300C03000030L,0x00CC3300000000L,0x00CCFFCCCFFCCCL,0x02FFCC2FE0CFFEL,0x00C3CF0FC3CF0CL,0x02FCE02ECCE2ECL,0x00300C00000000L,
0x00030380C03803L,0x0300B00C0B0000L,0x0000332BA03000L,0x00000C0FC0C000L,0x0000000302C0C0L,0x0000000FC00000L,0x00000000000030L,0x00030383838380L,
@ -84,7 +84,7 @@ public class PrintTool {
}
}
}
public static void print(final RasterPlotter matrix, final int x, final int y, final int angle, final String message, final int align) {
// align = -1 : left
// align = 1 : right
@ -107,8 +107,8 @@ public class PrintTool {
else if (angle == 315) {xx += 6; yy += 6;}
}
}
private static final int arcDist = 8;
public static void arcPrint(final RasterPlotter matrix, final int cx, final int cy, final int radius, final int angle, final String message) {
final int x = cx + (int) ((radius + 1) * Math.cos(RasterPlotter.PI180 * angle));
@ -123,6 +123,6 @@ public class PrintTool {
if ((angle < (90 - arcDist)) || (angle > (270 + arcDist))) xp = x;
print(matrix, xp, yp, 0, message, -1);
}
}

Write Preview
Loading…
Cancel
Save