diff --git a/htroot/PerformanceSearch_p.java b/htroot/PerformanceSearch_p.java index 2cb0e5673..d53c596a0 100644 --- a/htroot/PerformanceSearch_p.java +++ b/htroot/PerformanceSearch_p.java @@ -51,11 +51,11 @@ public class PerformanceSearch_p { prop.put("table_" + c + "_event", search.processName.name()); prop.put("table_" + c + "_comment", search.comment); prop.putNum("table_" + c + "_count", search.resultCount); - prop.putNum("table_" + c + "_delta", event.time.getTime() - lastt); - prop.put("table_" + c + "_time", (event.time).toString()); + prop.putNum("table_" + c + "_delta", event.getTime() - lastt); + prop.put("table_" + c + "_time", event.getFormattedDate()); prop.putNum("table_" + c + "_duration", search.duration); c++; - lastt = event.time.getTime(); + lastt = event.getTime(); } } prop.put("table", c); diff --git a/htroot/api/timeline_p.java b/htroot/api/timeline_p.java index 2f3c83c66..783ff5056 100644 --- a/htroot/api/timeline_p.java +++ b/htroot/api/timeline_p.java @@ -24,45 +24,122 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import java.text.ParseException; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.TreeMap; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.sorting.OrderedScoreMap; +import net.yacy.search.EventTracker.Event; import net.yacy.search.EventTracker; -import net.yacy.search.Switchboard; +import net.yacy.search.query.AccessTracker; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; public final class timeline_p { + // example: + // http://localhost:8090/api/timeline_p.xml?from=20140601000000&to=20140629000000&data=queries&head=2&period=6h + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { - // return variable that accumulates replacements - final Switchboard sb = (Switchboard) env; - final serverObjects prop = new serverObjects(); if ((post == null) || (env == null)) return prop; - + // get type of data to be listed in the timeline + int maxeventsperperiod = post.getInt("head", 1); // the maximum number of events per period + String period = post.get("period", ""); // must be an integer with a character c at the end, c = Y|M|d|h|m|s + int periodlength = 0; + if (period.length() > 0) { + char c = period.charAt(period.length() - 1); + int p = Integer.parseInt(period.substring(0, period.length() - 1)); + if (c == 's') periodlength = p * 1000; + else if (c == 'm') periodlength = p * 1000 * 60; + else if (c == 'h') periodlength = p * 1000 * 60 * 60; + else if (c == 'd') periodlength = p * 1000 * 60 * 60 * 24; + else if (c == 'M') periodlength = p * 1000 * 60 * 60 * 24 * 30; + else if (c == 'Y') periodlength = p * 1000 * 60 * 60 * 24 * 365; + else periodlength = 0; + } final String[] data = post.get("data", "").split(","); // a string of word hashes that shall be searched and combined - Map> proc = new HashMap<>(); - for (String s: data) proc.put(s, null); + Map> proc = new HashMap<>(); + for (String s: data) if (s.length() > 0) proc.put(s, null); - /* - while (i.hasNext() && c < count) { - entry = i.next(); - lm = new Date(entry.lastModified()); - lms = GenericFormatter.ANSIC_FORMATTER.format(lm); - prop.put("event_" + c + "_time", lms); // like "Wed May 01 1963 00:00:00 GMT-0600" - prop.put("event_" + c + "_isDuration", 0); // 0 (only a point) or 1 (period of time) - prop.put("event_" + c + "_isDuration_duration", 0); // 0 (only a point) or 1 (period of time) - prop.putHTML("event_" + c + "_type", "type"); // short title of the event - prop.putHTML("event_" + c + "_description", ""); // long description of the event - c++; + // get a time period + Date fromDate = new Date(0); + Date toDate = new Date(); + try {fromDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("from", "20031215182700"));} catch (ParseException e) {} + try {toDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("to", GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())));} catch (ParseException e) {} + + // fill proc with events from the given data and time period + if (proc.containsKey("queries")) { + List events = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate); + proc.put("queries", events); } - prop.put("event", c); -*/ + + // mix all events into one event list + TreeMap eax = new TreeMap<>(); + for (List events: proc.values()) if (events != null) { + for (EventTracker.Event event: events) eax.put(event.getFormattedDate(), event); + } + proc.clear(); // we don't need that here any more + List ea = new ArrayList<>(); + for (Event event: eax.values()) ea.add(event); + + if (periodlength > 0 && ea.size() > 0) { + // create a statistical analysis; step by chunks of periodlength entries + Event firstEvent = ea.iterator().next(); + long startDate = fromDate.getTime(); + //TreeMap + OrderedScoreMap accumulation = new OrderedScoreMap<>(null); + List eap = new ArrayList<>(); + String limit = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(startDate + periodlength)); + for (Event event: ea) { + if (event.getFormattedDate().compareTo(limit) >= 0) { + // write accumulation of the score map into eap + stats(accumulation, eap, startDate, periodlength, maxeventsperperiod, firstEvent.type); + firstEvent = event; + startDate += periodlength; + limit = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(startDate + periodlength)); + } + accumulation.inc(event.payload.toString()); + } + stats(accumulation, eap, startDate, periodlength, maxeventsperperiod, firstEvent.type); + + // overwrite the old table for out + ea = eap; + } + + // create a list of these events + int count = 0; + for (Event event: ea) { + prop.put("event_" + count + "_time", event.getFormattedDate()); + prop.put("event_" + count + "_isPeriod", event.duration == 0 ? 0 : 1); + prop.put("event_" + count + "_isPeriod_duration", event.duration); + prop.put("event_" + count + "_isPeriod_count", event.count); + prop.putHTML("event_" + count + "_type", event.type); + prop.putXML("event_" + count + "_description", event.payload.toString()); + count++; + } + prop.put("event", count); + prop.put("count", count); return prop; } + private static void stats(OrderedScoreMap accumulation, List eap, long startDate, int periodlength, int head, String type) { + // write accumulation of the score map into eap + Iterator si = accumulation.keys(false); + int c = 0; + while (si.hasNext() && c++ < head) { + String key = si.next(); + eap.add(new Event(startDate, periodlength, type, key, accumulation.get(key))); + } + accumulation.clear(); + } + } diff --git a/htroot/api/timeline_p.xml b/htroot/api/timeline_p.xml index bc5cf0642..4bccc2092 100644 --- a/htroot/api/timeline_p.xml +++ b/htroot/api/timeline_p.xml @@ -1,7 +1,4 @@ - -#{event}# - - #[description]# - +#{event}# + #[description]# #{/event}# \ No newline at end of file diff --git a/source/net/yacy/peers/graphics/ProfilingGraph.java b/source/net/yacy/peers/graphics/ProfilingGraph.java index aa87caf8d..a78ce5d30 100644 --- a/source/net/yacy/peers/graphics/ProfilingGraph.java +++ b/source/net/yacy/peers/graphics/ProfilingGraph.java @@ -117,7 +117,7 @@ public class ProfilingGraph { EventTracker.Event event; while (events.hasNext()) { event = events.next(); - time = event.time.getTime() - now; + time = event.getTime() - now; bytes = ((Long) event.payload).longValue(); x1 = (int) (time/1000); y1 = (int) (bytes / 1024 / 1024); @@ -138,7 +138,7 @@ public class ProfilingGraph { int words; while (events.hasNext()) { event = events.next(); - time = event.time.getTime() - now; + time = event.getTime() - now; words = (int) ((Long) event.payload).longValue(); x1 = (int) (time/1000); y1 = words; @@ -158,7 +158,7 @@ public class ProfilingGraph { int ppm; while (events.hasNext()) { event = events.next(); - time = event.time.getTime() - now; + time = event.getTime() - now; ppm = (int) ((Long) event.payload).longValue(); x1 = (int) (time/1000); y1 = ppm; @@ -180,7 +180,7 @@ public class ProfilingGraph { String pingPeer; while (events.hasNext()) { event = events.next(); - time = event.time.getTime() - now; + time = event.getTime() - now; ping = (EventPing) event.payload; x1 = (int) (time/1000); y1 = Math.abs((ping.outgoing ? ping.toPeer : ping.fromPeer).hashCode()) % vspace; diff --git a/source/net/yacy/search/EventTracker.java b/source/net/yacy/search/EventTracker.java index d5486a365..2fba99a11 100644 --- a/source/net/yacy/search/EventTracker.java +++ b/source/net/yacy/search/EventTracker.java @@ -26,6 +26,7 @@ package net.yacy.search; +import java.text.ParseException; import java.util.Date; import java.util.Iterator; import java.util.Map; @@ -103,7 +104,7 @@ public class EventTracker { final long now = System.currentTimeMillis(); while (!history.isEmpty()) { e = history.peek(); - if (now - e.time.getTime() < maxQueueAge) break; + if (now - e.getTime() < maxQueueAge) break; history.poll(); } } @@ -123,13 +124,13 @@ public class EventTracker { final long now = System.currentTimeMillis(); int count = 0; while (event.hasNext()) { - if (now - event.next().time.getTime() < time) count++; + if (now - event.next().getTime() < time) count++; } return count; } public final static class Event { - final public Date time; + final private Object time; // either a String in SHORT_SECOND format, a Long with ms since epoch or Date; final public int duration; // ms final public String type; final public Object payload; @@ -137,9 +138,40 @@ public class EventTracker { public Event(final Date time, final int duration, final String type, final Object payload, final int count) { this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count; } + public Event(final Long time, final int duration, final String type, final Object payload, final int count) { + this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count; + } + public Event(final String time, final int duration, final String type, final Object payload, final int count) { + this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count; + } + public String getFormattedDate() { + if (this.time instanceof String) return (String) this.time; + if (this.time instanceof Long) return GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date((Long) this.time)); + if (this.time instanceof Date) return GenericFormatter.SHORT_SECOND_FORMATTER.format((Date) this.time); + return null; + } + public long getTime() { + if (this.time instanceof String) try { + return GenericFormatter.SHORT_SECOND_FORMATTER.parse((String) this.time).getTime(); + } catch (ParseException e) { + return -1L; + } + if (this.time instanceof Long) return (Long) this.time; + if (this.time instanceof Date) return ((Date) this.time).getTime(); + return -1L; + } + public Date getDate() { + if (this.time instanceof String) try { + return GenericFormatter.SHORT_SECOND_FORMATTER.parse((String) this.time); + } catch (ParseException e) { + return null; + }if (this.time instanceof Long) return new Date((Long) this.time); + if (this.time instanceof Date) return (Date) this.time; + return null; + } @Override public String toString() { - return type + " " + GenericFormatter.SHORT_SECOND_FORMATTER.format(time) + (duration == 0 ? " " : "(" + duration + "ms) ") + (count == 0 ? " " : "[" + count + "] ") + payload; + return type + " " + getFormattedDate() + (duration == 0 ? " " : "(" + duration + "ms) ") + (count == 0 ? " " : "[" + count + "] ") + payload; } } } diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index 17ce9e931..b1b1b86c2 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -25,15 +25,20 @@ package net.yacy.search.query; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStreamReader; import java.io.RandomAccessFile; import java.text.ParseException; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.LinkedList; +import java.util.List; +import java.util.regex.Pattern; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.WordCache; @@ -91,6 +96,10 @@ public class AccessTracker { dumpFile = f; } + public static File getDumpFile() { + return dumpFile; + } + public static void add(final Location location, final QueryParams query, int resultCount) { if (location == Location.local) synchronized (localSearches) {add(localSearches, query, resultCount);} if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query, resultCount);} @@ -208,8 +217,8 @@ public class AccessTracker { * @param to the right boundary of the sequence to search for (excluded) * @return a list of lines within the given dates */ - public static ArrayList readLog(File f, Date from, Date to) { - ArrayList list = new ArrayList<>(); + public static List readLog(File f, Date from, Date to) { + List events = new ArrayList<>(); RandomAccessFile raf = null; try { raf = new RandomAccessFile(f, "r"); @@ -217,23 +226,32 @@ public class AccessTracker { if (fd.after(from)) from = fd; long seekFrom = binarySearch(raf, from, 0, raf.length()); long seekTo = binarySearch(raf, to, seekFrom, raf.length()); - Date eDate = readDate(raf, seekTo); - if (eDate.before(to)) seekTo = raf.length(); + //Date eDate = readDate(raf, seekTo); + //if (eDate.before(to)) seekTo = raf.length(); raf.seek(seekFrom); + byte[] buffer = new byte[(int) (seekTo - seekFrom)]; + raf.readFully(buffer); // we make a copy because that dramatically speeds up reading lines; RandomAccessFile.readLine is very slow + raf.close(); + ByteArrayInputStream bais = new ByteArrayInputStream(buffer); + BufferedReader reader = new BufferedReader(new InputStreamReader(bais, "UTF-8")); String line; - while (raf.getFilePointer() < seekTo && (line = raf.readLine()) != null) { + Pattern sp = Pattern.compile(" "); + while ((line = reader.readLine()) != null) { // parse the line - String[] ls = line.split(" "); + String[] ls = sp.split(line); EventTracker.Event event; - try { - event = new EventTracker.Event(GenericFormatter.SHORT_SECOND_FORMATTER.parse(ls[0]), 0, "query", line.substring(ls[0].length() + ls[1].length() + 2), Integer.valueOf(ls[1])); - list.add(event); + if (ls.length > 1) try { + event = new EventTracker.Event(ls[0], 0, "query", line.substring(ls[0].length() + ls[1].length() + 2), Integer.valueOf(ls[1])); + events.add(event); } catch (NumberFormatException e) { continue; - } catch (ParseException e) { + } catch (Throwable e) { continue; } } + reader.close(); + bais.close(); + buffer = null; } catch (final FileNotFoundException e) { ConcurrentLog.logException(e); } catch (final IOException e) { @@ -241,7 +259,7 @@ public class AccessTracker { } finally { if (raf != null) try {raf.close();} catch (final IOException e) {} } - return list; + return events; } /** @@ -302,7 +320,7 @@ public class AccessTracker { try { from = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1]); Date to = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2]); - ArrayList dump = readLog(new File(file), from, to); + List dump = readLog(new File(file), from, to); for (EventTracker.Event s: dump) System.out.println(s.toString()); } catch (ParseException e) { e.printStackTrace();