boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring of AccessTracker events & timeline fix

Browse Source
pull/1/head
Michael Peter Christen 11 years ago
parent 5b94a257ce
commit 8c52f0651b
6 changed files with 172 additions and 48 deletions
Show Stats Download Patch File Download Diff File

6
htroot/PerformanceSearch_p.java
Unescape View File

@ -51,11 +51,11 @@ public class PerformanceSearch_p {
prop.put("table_" + c + "_event", search.processName.name());
prop.put("table_" + c + "_comment", search.comment);
prop.putNum("table_" + c + "_count", search.resultCount);
prop.putNum("table_" + c + "_delta", event.time.getTime() - lastt);
prop.put("table_" + c + "_time", (event.time).toString());
prop.putNum("table_" + c + "_delta", event.getTime() - lastt);
prop.put("table_" + c + "_time", event.getFormattedDate());
prop.putNum("table_" + c + "_duration", search.duration);
c++;
lastt = event.time.getTime();
lastt = event.getTime();
}
}
prop.put("table", c);

117
htroot/api/timeline_p.java
Unescape View File

@ -24,45 +24,122 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.OrderedScoreMap;
import net.yacy.search.EventTracker.Event;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard;
import net.yacy.search.query.AccessTracker;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public final class timeline_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
// example:
// http://localhost:8090/api/timeline_p.xml?from=20140601000000&to=20140629000000&data=queries&head=2&period=6h
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
// get type of data to be listed in the timeline
int maxeventsperperiod = post.getInt("head", 1); // the maximum number of events per period
String period = post.get("period", ""); // must be an integer with a character c at the end, c = Y|M|d|h|m|s
int periodlength = 0;
if (period.length() > 0) {
char c = period.charAt(period.length() - 1);
int p = Integer.parseInt(period.substring(0, period.length() - 1));
if (c == 's') periodlength = p * 1000;
else if (c == 'm') periodlength = p * 1000 * 60;
else if (c == 'h') periodlength = p * 1000 * 60 * 60;
else if (c == 'd') periodlength = p * 1000 * 60 * 60 * 24;
else if (c == 'M') periodlength = p * 1000 * 60 * 60 * 24 * 30;
else if (c == 'Y') periodlength = p * 1000 * 60 * 60 * 24 * 365;
else periodlength = 0;
}
final String[] data = post.get("data", "").split(","); // a string of word hashes that shall be searched and combined
Map<String, Map<Date, EventTracker.Event>> proc = new HashMap<>();
for (String s: data) proc.put(s, null);
Map<String, List<EventTracker.Event>> proc = new HashMap<>();
for (String s: data) if (s.length() > 0) proc.put(s, null);
/*
while (i.hasNext() && c < count) {
entry = i.next();
lm = new Date(entry.lastModified());
lms = GenericFormatter.ANSIC_FORMATTER.format(lm);
prop.put("event_" + c + "_time", lms); // like "Wed May 01 1963 00:00:00 GMT-0600"
prop.put("event_" + c + "_isDuration", 0); // 0 (only a point) or 1 (period of time)
prop.put("event_" + c + "_isDuration_duration", 0); // 0 (only a point) or 1 (period of time)
prop.putHTML("event_" + c + "_type", "type"); // short title of the event
prop.putHTML("event_" + c + "_description", ""); // long description of the event
c++;
// get a time period
Date fromDate = new Date(0);
Date toDate = new Date();
try {fromDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("from", "20031215182700"));} catch (ParseException e) {}
try {toDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("to", GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())));} catch (ParseException e) {}
// fill proc with events from the given data and time period
if (proc.containsKey("queries")) {
List<EventTracker.Event> events = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate);
proc.put("queries", events);
}
// mix all events into one event list
TreeMap<String, EventTracker.Event> eax = new TreeMap<>();
for (List<EventTracker.Event> events: proc.values()) if (events != null) {
for (EventTracker.Event event: events) eax.put(event.getFormattedDate(), event);
}
prop.put("event", c);
*/
proc.clear(); // we don't need that here any more
List<EventTracker.Event> ea = new ArrayList<>();
for (Event event: eax.values()) ea.add(event);
if (periodlength > 0 && ea.size() > 0) {
// create a statistical analysis; step by chunks of periodlength entries
Event firstEvent = ea.iterator().next();
long startDate = fromDate.getTime();
//TreeMap<Date, EventTracker.Event>
OrderedScoreMap<String> accumulation = new OrderedScoreMap<>(null);
List<EventTracker.Event> eap = new ArrayList<>();
String limit = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(startDate + periodlength));
for (Event event: ea) {
if (event.getFormattedDate().compareTo(limit) >= 0) {
// write accumulation of the score map into eap
stats(accumulation, eap, startDate, periodlength, maxeventsperperiod, firstEvent.type);
firstEvent = event;
startDate += periodlength;
limit = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(startDate + periodlength));
}
accumulation.inc(event.payload.toString());
}
stats(accumulation, eap, startDate, periodlength, maxeventsperperiod, firstEvent.type);
// overwrite the old table for out
ea = eap;
}
// create a list of these events
int count = 0;
for (Event event: ea) {
prop.put("event_" + count + "_time", event.getFormattedDate());
prop.put("event_" + count + "_isPeriod", event.duration == 0 ? 0 : 1);
prop.put("event_" + count + "_isPeriod_duration", event.duration);
prop.put("event_" + count + "_isPeriod_count", event.count);
prop.putHTML("event_" + count + "_type", event.type);
prop.putXML("event_" + count + "_description", event.payload.toString());
count++;
}
prop.put("event", count);
prop.put("count", count);
return prop;
}
private static void stats(OrderedScoreMap<String> accumulation, List<EventTracker.Event> eap, long startDate, int periodlength, int head, String type) {
// write accumulation of the score map into eap
Iterator<String> si = accumulation.keys(false);
int c = 0;
while (si.hasNext() && c++ < head) {
String key = si.next();
eap.add(new Event(startDate, periodlength, type, key, accumulation.get(key)));
}
accumulation.clear();
}
}

7
htroot/api/timeline_p.xml
Unescape View File

@ -1,7 +1,4 @@
<timeline>
#{event}#
<event time="#[time]#" isDuration=#(isDuration)#"false"::duration="#[duration]#"#(/isDuration)# type="#[type]#">
#[description]#
</event>
<timeline count="#[count]#">#{event}#
<event time="#[time]#" isPeriod=#(isPeriod)#"false"::"true" duration="#[duration]#" count="#[count]#"#(/isPeriod)# type="#[type]#">#[description]#</event>
#{/event}#
</timeline>

8
source/net/yacy/peers/graphics/ProfilingGraph.java
Unescape View File

@ -117,7 +117,7 @@ public class ProfilingGraph {
EventTracker.Event event;
while (events.hasNext()) {
event = events.next();
time = event.time.getTime() - now;
time = event.getTime() - now;
bytes = ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = (int) (bytes / 1024 / 1024);
@ -138,7 +138,7 @@ public class ProfilingGraph {
int words;
while (events.hasNext()) {
event = events.next();
time = event.time.getTime() - now;
time = event.getTime() - now;
words = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = words;
@ -158,7 +158,7 @@ public class ProfilingGraph {
int ppm;
while (events.hasNext()) {
event = events.next();
time = event.time.getTime() - now;
time = event.getTime() - now;
ppm = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = ppm;
@ -180,7 +180,7 @@ public class ProfilingGraph {
String pingPeer;
while (events.hasNext()) {
event = events.next();
time = event.time.getTime() - now;
time = event.getTime() - now;
ping = (EventPing) event.payload;
x1 = (int) (time/1000);
y1 = Math.abs((ping.outgoing ? ping.toPeer : ping.fromPeer).hashCode()) % vspace;

40
source/net/yacy/search/EventTracker.java
Unescape View File

@ -26,6 +26,7 @@
package net.yacy.search;
import java.text.ParseException;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
@ -103,7 +104,7 @@ public class EventTracker {
final long now = System.currentTimeMillis();
while (!history.isEmpty()) {
e = history.peek();
if (now - e.time.getTime() < maxQueueAge) break;
if (now - e.getTime() < maxQueueAge) break;
history.poll();
}
}
@ -123,13 +124,13 @@ public class EventTracker {
final long now = System.currentTimeMillis();
int count = 0;
while (event.hasNext()) {
if (now - event.next().time.getTime() < time) count++;
if (now - event.next().getTime() < time) count++;
}
return count;
}
public final static class Event {
final public Date time;
final private Object time; // either a String in SHORT_SECOND format, a Long with ms since epoch or Date;
final public int duration; // ms
final public String type;
final public Object payload;
@ -137,9 +138,40 @@ public class EventTracker {
public Event(final Date time, final int duration, final String type, final Object payload, final int count) {
this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count;
}
public Event(final Long time, final int duration, final String type, final Object payload, final int count) {
this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count;
}
public Event(final String time, final int duration, final String type, final Object payload, final int count) {
this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count;
}
public String getFormattedDate() {
if (this.time instanceof String) return (String) this.time;
if (this.time instanceof Long) return GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date((Long) this.time));
if (this.time instanceof Date) return GenericFormatter.SHORT_SECOND_FORMATTER.format((Date) this.time);
return null;
}
public long getTime() {
if (this.time instanceof String) try {
return GenericFormatter.SHORT_SECOND_FORMATTER.parse((String) this.time).getTime();
} catch (ParseException e) {
return -1L;
}
if (this.time instanceof Long) return (Long) this.time;
if (this.time instanceof Date) return ((Date) this.time).getTime();
return -1L;
}
public Date getDate() {
if (this.time instanceof String) try {
return GenericFormatter.SHORT_SECOND_FORMATTER.parse((String) this.time);
} catch (ParseException e) {
return null;
}if (this.time instanceof Long) return new Date((Long) this.time);
if (this.time instanceof Date) return (Date) this.time;
return null;
}
@Override
public String toString() {
return type + " " + GenericFormatter.SHORT_SECOND_FORMATTER.format(time) + (duration == 0 ? " " : "(" + duration + "ms) ") + (count == 0 ? " " : "[" + count + "] ") + payload;
return type + " " + getFormattedDate() + (duration == 0 ? " " : "(" + duration + "ms) ") + (count == 0 ? " " : "[" + count + "] ") + payload;
}
}
}

42
source/net/yacy/search/query/AccessTracker.java
Unescape View File

@ -25,15 +25,20 @@
package net.yacy.search.query;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.WordCache;
@ -91,6 +96,10 @@ public class AccessTracker {
dumpFile = f;
}
public static File getDumpFile() {
return dumpFile;
}
public static void add(final Location location, final QueryParams query, int resultCount) {
if (location == Location.local) synchronized (localSearches) {add(localSearches, query, resultCount);}
if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query, resultCount);}
@ -208,8 +217,8 @@ public class AccessTracker {
* @param to the right boundary of the sequence to search for (excluded)
* @return a list of lines within the given dates
*/
public static ArrayList<EventTracker.Event> readLog(File f, Date from, Date to) {
ArrayList<EventTracker.Event> list = new ArrayList<>();
public static List<EventTracker.Event> readLog(File f, Date from, Date to) {
List<EventTracker.Event> events = new ArrayList<>();
RandomAccessFile raf = null;
try {
raf = new RandomAccessFile(f, "r");
@ -217,23 +226,32 @@ public class AccessTracker {
if (fd.after(from)) from = fd;
long seekFrom = binarySearch(raf, from, 0, raf.length());
long seekTo = binarySearch(raf, to, seekFrom, raf.length());
Date eDate = readDate(raf, seekTo);
if (eDate.before(to)) seekTo = raf.length();
//Date eDate = readDate(raf, seekTo);
//if (eDate.before(to)) seekTo = raf.length();
raf.seek(seekFrom);
byte[] buffer = new byte[(int) (seekTo - seekFrom)];
raf.readFully(buffer); // we make a copy because that dramatically speeds up reading lines; RandomAccessFile.readLine is very slow
raf.close();
ByteArrayInputStream bais = new ByteArrayInputStream(buffer);
BufferedReader reader = new BufferedReader(new InputStreamReader(bais, "UTF-8"));
String line;
while (raf.getFilePointer() < seekTo && (line = raf.readLine()) != null) {
Pattern sp = Pattern.compile(" ");
while ((line = reader.readLine()) != null) {
// parse the line
String[] ls = line.split(" ");
String[] ls = sp.split(line);
EventTracker.Event event;
try {
event = new EventTracker.Event(GenericFormatter.SHORT_SECOND_FORMATTER.parse(ls[0]), 0, "query", line.substring(ls[0].length() + ls[1].length() + 2), Integer.valueOf(ls[1]));
list.add(event);
if (ls.length > 1) try {
event = new EventTracker.Event(ls[0], 0, "query", line.substring(ls[0].length() + ls[1].length() + 2), Integer.valueOf(ls[1]));
events.add(event);
} catch (NumberFormatException e) {
continue;
} catch (ParseException e) {
} catch (Throwable e) {
continue;
}
}
reader.close();
bais.close();
buffer = null;
} catch (final FileNotFoundException e) {
ConcurrentLog.logException(e);
} catch (final IOException e) {
@ -241,7 +259,7 @@ public class AccessTracker {
} finally {
if (raf != null) try {raf.close();} catch (final IOException e) {}
}
return list;
return events;
}
/**
@ -302,7 +320,7 @@ public class AccessTracker {
try {
from = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1]);
Date to = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2]);
ArrayList<EventTracker.Event> dump = readLog(new File(file), from, to);
List<EventTracker.Event> dump = readLog(new File(file), from, to);
for (EventTracker.Event s: dump) System.out.println(s.toString());
} catch (ParseException e) {
e.printStackTrace();

Write Preview
Loading…
Cancel
Save