From 71112b1fe63699aa0ea58662e251e818fbe21045 Mon Sep 17 00:00:00 2001 From: karlchenofhell Date: Tue, 16 Jan 2007 16:13:21 +0000 Subject: [PATCH] - added LogStatistics_p.html servlet based on the logalizer (indexing values not functional yet due to charset/regex problems) add the following to DATA/LOG/yacy.logging: --- # Properties for the LogalizerHandler de.anomic.server.logging.LogalizerHandler.enabled = true de.anomic.server.logging.LogalizerHandler.debug = false de.anomic.server.logging.LogalizerHandler.parserPackage = de.anomic.server.logging.logParsers --- and "de.anomic.server.logging.LogalizerHandler" to the list of global handlers git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3219 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/LogStatistics_p.html | 102 +++++++++ htroot/LogStatistics_p.java | 195 ++++++++++++++++++ htroot/env/base.css | 15 ++ skins/dark.css | 2 +- skins/default.css | 1 + skins/grey.css | 1 + skins/mint-green.css | 1 + skins/phosphor.css | 1 + .../server/logging/LogalizerHandler.java | 13 ++ .../logging/logParsers/LogParserPLASMA.java | 80 +++---- 10 files changed, 370 insertions(+), 41 deletions(-) create mode 100644 htroot/LogStatistics_p.html create mode 100644 htroot/LogStatistics_p.java diff --git a/htroot/LogStatistics_p.html b/htroot/LogStatistics_p.html new file mode 100644 index 000000000..e2ee9bb25 --- /dev/null +++ b/htroot/LogStatistics_p.html @@ -0,0 +1,102 @@ + + + + YaCy '#[clientname]#': Log Statistics + #%env/templates/metas.template%# + + + #%env/templates/header.template%# +

Log statistics

+ +
+ #(results)#:: +
DHT +
+ URLs +
+
URLs Received:
#[urlSum]#
+
URLs Requested:
#[urlReqSum]#
+
URLs Blocked:
#[blockedURLSum]#
+
Total time:
#[urlTimeSum]# #[urlTimeSumUnit]#
+
URLs Sent:
#[DHTSendURLs]#
+
+
+ +
+ RWIs / Words +
+
Words received:
#[wordsSum]#
+
RWIs reveived:
#[rwiSum]#
+
RWIs blocked:
#[blockedRWISum]#
+
Total time:
#[rwiTimeSum]# #[rwiTimeSumUnit]#
+
RWIs selected:
#[DHTSelectionWordsCount]#
+
Selection time:
#[DHTSelectionWordsTimeCount]#
+
+
+ +
+ Chunks / Protocol +
+
Sent traffic:
#[DHTSendTraffic]# #[DHTSendTrafficUnit]#
+
Total peers selected:
#[DHTSelectionTargetCount]#
+
DHT Distance: Min/Average/Max:
#[minDHTDist]# / #[avgDHTDist]# / #[maxDHTDist]#
+
Not enough peers found for distribution:
#[notEnoughDHTPeers]#
+
Peers busy:
#[busyPeerCount]#
+
Distribution failed:
#[failedIndexDistributionCount]#
+
Rejected index receives:
#[RWIRejectCount]#
+
Rejected index receives from:
+
+ #[DHTRejectPeers]# peers#(useDHTRejectPeers)#:: +
+
    #{DHTRejectPeers}# +
  • ( #[hash]# ) #[name]#
    • #{/DHTRejectPeers}# +
+
#(/useDHTRejectPeers)# +
+
Sent indexes:
+
+ #[DHTPeers]# peers#(useDHTPeers)#:: +
+
    #{DHTPeers}# +
  • ( #[hash]# ) #[name]#
    • #{/DHTPeers}# +
+
#(/useDHTPeers)# +
+
+
+
+ +
Ranking Distribution +
+
Submitted ranking files:
#[rankingDistributionCount]#
+
Total time submitting ranking files:
#[rankingDistributionTime]#
+
Failed ranking distributions:
#[rankingDistributionFailCount]#
+
+
+ +
Indexing +
+
Indexed sites:
#[indexedSites]#
+
Size of indexed sites:
#[indexedSiteSizeSum]# #[indexedSiteSizeSumUnit]#
+
Indexed words:
#[indexedWords]#
+
Indexed anchors:
#[indexedAnchors]#
+
Total stacking time:
#[indexedStackingTime]# #[indexedStackingTimeUnit]#
+
Total parsing time:
#[indexedParsingTime]# #[indexedParsingTimeUnit]#
+
Total indexing time:
#[indexedIndexingTime]# #[indexedIndexingTimeUnit]#
+
Total storage time:
#[indexedStorageTime]# #[indexedStorageTimeUnit]#
+
+
+ +
Errors +
+
Tried to create left child node twice
#[leftChildTwiceCount]#
+
Tried to create right child node twice
#[rightChildTwiceCount]#
+
Malformed URL Exception
#[malformedURLCount]#
+
+
+ #(/results)# +
+ + #%env/templates/footer.template%# + + \ No newline at end of file diff --git a/htroot/LogStatistics_p.java b/htroot/LogStatistics_p.java new file mode 100644 index 000000000..19471ae4c --- /dev/null +++ b/htroot/LogStatistics_p.java @@ -0,0 +1,195 @@ +// LogStatistic_p.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2005 +// Created 16.01.2007 +// +// This File is contributed by Franz Brauße +// +// $LastChangedDate: 2007-01-17 12:00:00 +0100 (Di, 17 Jan 2007) $ +// $LastChangedRevision: 3216 $ +// $LastChangedBy: karlchenofhell $ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notive above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +import java.util.HashSet; +import java.util.Hashtable; +import java.util.logging.Handler; +import java.util.logging.Logger; + +import de.anomic.http.httpHeader; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; +import de.anomic.server.logging.LogalizerHandler; +import de.anomic.server.logging.logParsers.LogParser; +import de.anomic.server.logging.logParsers.LogParserPLASMA; + +public class LogStatistics_p { + + private static final String RESULTS = "results_"; + + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { + + final serverObjects prop = new serverObjects(); + + Logger logger = Logger.getLogger(""); + Handler[] handlers = logger.getHandlers(); + Hashtable r = null; + for (int i=0; i 0) ? 1 : 0); + prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers", i); + + names = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_NAME)).toArray(); + hashes = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_HASH)).toArray(); + i = 0; + for (; i 0) ? 1 : 0); + prop.put(RESULTS + "useDHTPeers_DHTPeers", i); + + return prop; + } + } + + private static final String MILLISECONDS = "ms"; + private static final String SECONDS = "sec"; + private static final String MINUTES = "min"; + private static final String HOURS = "h"; + private static final String DAYS = "days"; + + private static final String[] units = new String[] { "Bytes", "KiloBytes", "MegaBytes", "GigaBytes" }; + + private static String[] transformTime(long timems) { + if (timems > 1000) timems /= 1000; else return new String[] { Long.toString(timems), MILLISECONDS }; + if (timems > 60) timems /= 60; else return new String[] { Long.toString(timems), SECONDS }; + if (timems > 60) timems /= 60; else return new String[] { Long.toString(timems), MINUTES }; + if (timems > 24) timems /= 24; else return new String[] { Long.toString(timems), HOURS }; + return new String[] { Long.toString(timems), DAYS }; + } + + private static String[] transformMem(long mem) { + int i; + for (i=0; i= 10240; i++) + mem /= 1024; + return new String[] { Long.toString(mem), units[i] }; + } +} + + + + + + + diff --git a/htroot/env/base.css b/htroot/env/base.css index 60056c5e3..c1b5f39da 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -488,4 +488,19 @@ div.CacheAdminInfo { width: 100px; border: 0px solid #aaaaaa; float: left; +} + +.hides .hoverShow { + display: none; +} + +.hides:hover .hoverShow { + display: block; + position: absolute; + border: 3px double; + margin: 0 0 0 2em; + padding: .3em; + height: 20em; + overflow: auto; + width: 30em; } \ No newline at end of file diff --git a/skins/dark.css b/skins/dark.css index 49b8ce394..787702c19 100644 --- a/skins/dark.css +++ b/skins/dark.css @@ -81,7 +81,7 @@ a.bookmarkAction { color: #999999; } .success { color: green; } .Message { background-color: #eeeeee; } .example { background-color: #D3D3D3; } - +.hides:hover .hoverShow { background-color: #111; border-color: #333; } /* Log */ body#ViewLog pre { color: #777; background-color: #333; } diff --git a/skins/default.css b/skins/default.css index 1f074ab4d..27498054a 100644 --- a/skins/default.css +++ b/skins/default.css @@ -208,6 +208,7 @@ div.ProgressBarFill { background-color:#D3D3D3; } +.hides:hover .hoverShow { background-color: #F8F8FF; } /* Log */ diff --git a/skins/grey.css b/skins/grey.css index a071eb997..ec10f345f 100644 --- a/skins/grey.css +++ b/skins/grey.css @@ -209,6 +209,7 @@ div.ProgressBarFill { background-color:#D3D3D3; } +.hides:hover .hoverShow { background-color: #F3F3F3; } /* Log */ diff --git a/skins/mint-green.css b/skins/mint-green.css index c726accf1..237c081fb 100644 --- a/skins/mint-green.css +++ b/skins/mint-green.css @@ -202,6 +202,7 @@ a.bookmarkAction { background-color:#D3D3D3; } +.hides:hover .hoverShow { background-color: #F8F8F8; } /* Log */ diff --git a/skins/phosphor.css b/skins/phosphor.css index 8d734356c..de5bc676f 100644 --- a/skins/phosphor.css +++ b/skins/phosphor.css @@ -218,6 +218,7 @@ div.ProgressBarFill { background-color:#D3D3D3; } +.hides:hover .hoverShow { background-color: #010; } /* Log */ diff --git a/source/de/anomic/server/logging/LogalizerHandler.java b/source/de/anomic/server/logging/LogalizerHandler.java index bf56ca7f2..4cf442240 100644 --- a/source/de/anomic/server/logging/LogalizerHandler.java +++ b/source/de/anomic/server/logging/LogalizerHandler.java @@ -50,6 +50,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.HashMap; import java.util.Hashtable; +import java.util.Iterator; import java.util.Set; import java.util.logging.Handler; import java.util.logging.LogManager; @@ -147,6 +148,18 @@ public class LogalizerHandler extends Handler { return parsers.keySet(); } + public LogParser getParser(int number) { + Object o; + Iterator it = parsers.keySet().iterator(); + int i = 0; + while (it.hasNext()) { + o = it.next(); + if (i++ == number) + return (LogParser)parsers.get(o); + } + return null; + } + public Hashtable getParserResults(LogParser parsername) { return parsername.getResults(); } diff --git a/source/de/anomic/server/logging/logParsers/LogParserPLASMA.java b/source/de/anomic/server/logging/logParsers/LogParserPLASMA.java index 150cce4db..a31b7f644 100644 --- a/source/de/anomic/server/logging/logParsers/LogParserPLASMA.java +++ b/source/de/anomic/server/logging/logParsers/LogParserPLASMA.java @@ -51,121 +51,121 @@ import java.util.regex.Pattern; public class LogParserPLASMA implements LogParser{ - /** the version of the LogParser */ + /** the version of the LogParser - Double*/ public static final String PARSER_VERSION = "version"; - /** the amount of URLs received during DHT */ + /** the amount of URLs received during DHT - Integer */ public static final String URLS_RECEIVED = "urlSum"; - /** the amount of URLs requested during DHT */ + /** the amount of URLs requested during DHT - Integer */ public static final String URLS_REQUESTED = "urlReqSum"; - /** the amount of URLs blocked during DHT because they match the peer's blacklist */ + /** the amount of URLs blocked during DHT because they match the peer's blacklist - Integer */ public static final String URLS_BLOCKED = "blockedURLSum"; - /** the amount of words received during DHT */ + /** the amount of words received during DHT - Integer */ public static final String WORDS_RECEIVED = "wordsSum"; - /** the amount of RWIs received during DHT */ + /** the amount of RWIs received during DHT - Integer */ public static final String RWIS_RECEIVED = "rwiSum"; - /** the amount of RWIs blocked during DHT because their entries match the peer's blacklist */ + /** the amount of RWIs blocked during DHT because their entries match the peer's blacklist - Integer */ public static final String RWIS_BLOCKED = "blockedRWISum"; - /** total time receiving RWIs during DHT in milli seconds */ + /** total time receiving RWIs during DHT in milli seconds - Long */ public static final String RWIS_RECEIVED_TIME = "rwiTimeSum"; - /** total time receiving URLs during DHT in milli seconds */ + /** total time receiving URLs during DHT in milli seconds - Long */ public static final String URLS_RECEIVED_TIME = "urlTimeSum"; - /** the traffic sent during DHT in bytes */ + /** the traffic sent during DHT in bytes - Long */ public static final String DHT_TRAFFIC_SENT = "DHTSendTraffic"; - /** the amount of URLs requested by other peers and sent by this one */ + /** the amount of URLs requested by other peers and sent by this one - Integer */ public static final String DHT_URLS_SENT = "DHTSendURLs"; - /** the amount of rejected DHT transfers from other peers (i.e. because this peer was busy) */ + /** the amount of rejected DHT transfers from other peers (i.e. because this peer was busy) - Integer */ public static final String DHT_REJECTED = "RWIRejectCount"; - /** the amount of different peer-names from whose DHT transfers were rejected */ + /** the peer-names from whose DHT transfers were rejected - HasSet */ public static final String DHT_REJECTED_PEERS_NAME = "DHTRejectPeerNames"; - /** the amount of different peer-hashes from whose DHT transfers were rejected */ + /** the peer-hashes from whose DHT transfers were rejected - HasSet */ public static final String DHT_REJECTED_PEERS_HASH = "DHTRejectPeerHashs"; - /** the amount of different peer-names this peer sent DHT chunks to */ + /** the peer-names this peer sent DHT chunks to - HasSet */ public static final String DHT_SENT_PEERS_NAME = "DHTPeerNames"; - /** the amount of different peer-hashes this peer sent DHT chunks to */ + /** the peer-hashes this peer sent DHT chunks to - HasSet */ public static final String DHT_SENT_PEERS_HASH = "DHTPeerHashs"; - /** total amount of selected peers for index distribution */ + /** total amount of selected peers for index distribution - Integer */ public static final String DHT_SELECTED = "DHTSelectionTargetCount"; - /** total amount of words selected for index distribution */ + /** total amount of words selected for index distribution - Integer */ public static final String DHT_WORDS_SELECTED = "DHTSelectionWordsCount"; - /** total time selecting words for index distribution */ + /** total time selecting words for index distribution - Integer */ public static final String DHT_WORDS_SELECTED_TIME = "DHTSelectionWordsTimeCount"; - /** the minimal DHT distance during peer-selection for index distribution */ + /** the minimal DHT distance during peer-selection for index distribution - Double */ public static final String DHT_DISTANCE_MIN = "minDHTDist"; - /** the maximal DHT distance during peer-selection for index distribution */ + /** the maximal DHT distance during peer-selection for index distribution - Double */ public static final String DHT_DISTANCE_MAX = "maxDHTDist"; - /** the average DHT distance during peer-selection for index distribution */ + /** the average DHT distance during peer-selection for index distribution - Double */ public static final String DHT_DISTANCE_AVERAGE = "avgDHTDist"; - /** how many times remote peers were too busy to accept the index transfer */ + /** how many times remote peers were too busy to accept the index transfer - Integer */ public static final String PEERS_BUSY = "busyPeerCount"; - /** how many times not enough peers for index distribution were found */ + /** how many times not enough peers for index distribution were found - Integer */ public static final String PEERS_TOO_LESS = "notEnoughDHTPeers"; - /** how many times the index distribution failed (i.e. due to time-out or other reasons) */ + /** how many times the index distribution failed (i.e. due to time-out or other reasons) - Integer */ public static final String DHT_SENT_FAILED = "failedIndexDistributionCount"; - /** how many times the error "tried to create left child-node twice" occured */ + /** how many times the error "tried to create left child-node twice" occured - Integer */ public static final String ERROR_CHILD_TWICE_LEFT = "leftChildTwiceCount"; - /** how many times the error "tried to create right child-node twice" occured */ + /** how many times the error "tried to create right child-node twice" occured - Integer */ public static final String ERROR_CHILD_TWICE_RIGHT = "rightChildTwiceCount"; - /** how many ranking distributions were executed successfully */ + /** how many ranking distributions were executed successfully - Integer */ public static final String RANKING_DIST = "rankingDistributionCount"; - /** total time the ranking distributions took */ + /** total time the ranking distributions took - Integer */ public static final String RANKING_DIST_TIME = "rankingDistributionTime"; - /** how many ranking distributions failed */ + /** how many ranking distributions failed - Integer */ public static final String RANKING_DIST_FAILED = "rankingDistributionFailCount"; - /** how many times the error "Malformed URL" occured */ + /** how many times the error "Malformed URL" occured - Integer */ public static final String ERROR_MALFORMED_URL = "malformedURLCount"; - /** the amount of indexed sites */ + /** the amount of indexed sites - Integer */ public static final String INDEXED_SITES = "indexedSites"; - /** total amount of indexed words */ + /** total amount of indexed words - Integer */ public static final String INDEXED_WORDS = "indexedWords"; - /** total size of all indexed sites */ + /** total size of all indexed sites - Integer */ public static final String INDEXED_SITES_SIZE = "indexedSiteSizeSum"; - /** total amount of indexed anchors */ + /** total amount of indexed anchors - Integer */ public static final String INDEXED_ANCHORS = "indexedAnchors"; - /** total time needed for stacking the site of an indexing */ + /** total time needed for stacking the site of an indexing - Integer */ public static final String INDEXED_STACK_TIME = "indexedStackingTime"; - /** total time needed for parsing during indexing */ + /** total time needed for parsing during indexing - Integer */ public static final String INDEXED_PARSE_TIME = "indexedParsingTime"; - /** total time needed for the actual indexing during indexing */ + /** total time needed for the actual indexing during indexing - Integer */ public static final String INDEXED_INDEX_TIME = "indexedIndexingTime"; - /** total time needed for storing the results of an indexing */ + /** total time needed for storing the results of an indexing - Integer */ public static final String INDEXED_STORE_TIME = "indexedStorageTime"; @@ -411,7 +411,7 @@ public class LogParserPLASMA implements LogParser{ results.put(DHT_WORDS_SELECTED_TIME , new Integer(DHTSelectionWordsTimeCount)); results.put(DHT_DISTANCE_MIN , new Double(minDHTDist)); results.put(DHT_DISTANCE_MAX , new Double(maxDHTDist)); - results.put(DHT_DISTANCE_AVERAGE , new Double(avgDHTDist)); + results.put(DHT_DISTANCE_AVERAGE , new Double(avgDHTDist / DHTSelectionTargetCount)); results.put(PEERS_BUSY , new Integer(busyPeerCount)); results.put(PEERS_TOO_LESS , new Integer(notEnoughDHTPeers)); results.put(DHT_SENT_FAILED , new Integer(failedIndexDistributionCount));