boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring of AccessTracker events & timeline fix

Browse Source
pull/1/head
Michael Peter Christen 11 years ago
parent 5b94a257ce
commit 8c52f0651b
6 changed files with 172 additions and 48 deletions
Show Stats Download Patch File Download Diff File

6
htroot/PerformanceSearch_p.java
Unescape View File

@ -51,11 +51,11 @@ public class PerformanceSearch_p {
prop.put("table_" + c + "_event", search.processName.name()); prop.put("table_" + c + "_event", search.processName.name());
prop.put("table_" + c + "_comment", search.comment); prop.put("table_" + c + "_comment", search.comment);
prop.putNum("table_" + c + "_count", search.resultCount); prop.putNum("table_" + c + "_count", search.resultCount);
prop.putNum("table_" + c + "_delta", event.time.getTime() - lastt); prop.putNum("table_" + c + "_delta", event.getTime() - lastt);
prop.put("table_" + c + "_time", (event.time).toString()); prop.put("table_" + c + "_time", event.getFormattedDate());
prop.putNum("table_" + c + "_duration", search.duration); prop.putNum("table_" + c + "_duration", search.duration);
c++; c++;
lastt = event.time.getTime(); lastt = event.getTime();
} }
} }
prop.put("table", c); prop.put("table", c);

117
htroot/api/timeline_p.java
Unescape View File

@ -24,45 +24,122 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TreeMap;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.OrderedScoreMap;
import net.yacy.search.EventTracker.Event;
import net.yacy.search.EventTracker; import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.query.AccessTracker;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
public final class timeline_p { public final class timeline_p {
// example:
// http://localhost:8090/api/timeline_p.xml?from=20140601000000&to=20140629000000&data=queries&head=2&period=6h
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop; if ((post == null) || (env == null)) return prop;
// get type of data to be listed in the timeline
int maxeventsperperiod = post.getInt("head", 1); // the maximum number of events per period
String period = post.get("period", ""); // must be an integer with a character c at the end, c = Y|M|d|h|m|s
int periodlength = 0;
if (period.length() > 0) {
char c = period.charAt(period.length() - 1);
int p = Integer.parseInt(period.substring(0, period.length() - 1));
if (c == 's') periodlength = p * 1000;
else if (c == 'm') periodlength = p * 1000 * 60;
else if (c == 'h') periodlength = p * 1000 * 60 * 60;
else if (c == 'd') periodlength = p * 1000 * 60 * 60 * 24;
else if (c == 'M') periodlength = p * 1000 * 60 * 60 * 24 * 30;
else if (c == 'Y') periodlength = p * 1000 * 60 * 60 * 24 * 365;
else periodlength = 0;
}
final String[] data = post.get("data", "").split(","); // a string of word hashes that shall be searched and combined final String[] data = post.get("data", "").split(","); // a string of word hashes that shall be searched and combined
Map<String, Map<Date, EventTracker.Event>> proc = new HashMap<>(); Map<String, List<EventTracker.Event>> proc = new HashMap<>();
for (String s: data) proc.put(s, null); for (String s: data) if (s.length() > 0) proc.put(s, null);
/* // get a time period
while (i.hasNext() && c < count) { Date fromDate = new Date(0);
entry = i.next(); Date toDate = new Date();
lm = new Date(entry.lastModified()); try {fromDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("from", "20031215182700"));} catch (ParseException e) {}
lms = GenericFormatter.ANSIC_FORMATTER.format(lm); try {toDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("to", GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())));} catch (ParseException e) {}
prop.put("event_" + c + "_time", lms); // like "Wed May 01 1963 00:00:00 GMT-0600"
prop.put("event_" + c + "_isDuration", 0); // 0 (only a point) or 1 (period of time) // fill proc with events from the given data and time period
prop.put("event_" + c + "_isDuration_duration", 0); // 0 (only a point) or 1 (period of time) if (proc.containsKey("queries")) {
prop.putHTML("event_" + c + "_type", "type"); // short title of the event List<EventTracker.Event> events = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate);
prop.putHTML("event_" + c + "_description", ""); // long description of the event proc.put("queries", events);
c++;
} }
prop.put("event", c);
*/ // mix all events into one event list
TreeMap<String, EventTracker.Event> eax = new TreeMap<>();
for (List<EventTracker.Event> events: proc.values()) if (events != null) {
for (EventTracker.Event event: events) eax.put(event.getFormattedDate(), event);
}
proc.clear(); // we don't need that here any more
List<EventTracker.Event> ea = new ArrayList<>();
for (Event event: eax.values()) ea.add(event);
if (periodlength > 0 && ea.size() > 0) {
// create a statistical analysis; step by chunks of periodlength entries
Event firstEvent = ea.iterator().next();
long startDate = fromDate.getTime();
//TreeMap<Date, EventTracker.Event>
OrderedScoreMap<String> accumulation = new OrderedScoreMap<>(null);
List<EventTracker.Event> eap = new ArrayList<>();
String limit = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(startDate + periodlength));
for (Event event: ea) {
if (event.getFormattedDate().compareTo(limit) >= 0) {
// write accumulation of the score map into eap
stats(accumulation, eap, startDate, periodlength, maxeventsperperiod, firstEvent.type);
firstEvent = event;
startDate += periodlength;
limit = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(startDate + periodlength));
}
accumulation.inc(event.payload.toString());
}
stats(accumulation, eap, startDate, periodlength, maxeventsperperiod, firstEvent.type);
// overwrite the old table for out
ea = eap;
}
// create a list of these events
int count = 0;
for (Event event: ea) {
prop.put("event_" + count + "_time", event.getFormattedDate());
prop.put("event_" + count + "_isPeriod", event.duration == 0 ? 0 : 1);
prop.put("event_" + count + "_isPeriod_duration", event.duration);
prop.put("event_" + count + "_isPeriod_count", event.count);
prop.putHTML("event_" + count + "_type", event.type);
prop.putXML("event_" + count + "_description", event.payload.toString());
count++;
}
prop.put("event", count);
prop.put("count", count);
return prop; return prop;
} }
private static void stats(OrderedScoreMap<String> accumulation, List<EventTracker.Event> eap, long startDate, int periodlength, int head, String type) {
// write accumulation of the score map into eap
Iterator<String> si = accumulation.keys(false);
int c = 0;
while (si.hasNext() && c++ < head) {
String key = si.next();
eap.add(new Event(startDate, periodlength, type, key, accumulation.get(key)));
}
accumulation.clear();
}
} }

7
htroot/api/timeline_p.xml
Unescape View File

@ -1,7 +1,4 @@
<timeline> <timeline count="#[count]#">#{event}#
#{event}# <event time="#[time]#" isPeriod=#(isPeriod)#"false"::"true" duration="#[duration]#" count="#[count]#"#(/isPeriod)# type="#[type]#">#[description]#</event>
<event time="#[time]#" isDuration=#(isDuration)#"false"::duration="#[duration]#"#(/isDuration)# type="#[type]#">
#[description]#
</event>
#{/event}# #{/event}#
</timeline> </timeline>

8
source/net/yacy/peers/graphics/ProfilingGraph.java
Unescape View File

@ -117,7 +117,7 @@ public class ProfilingGraph {
EventTracker.Event event; EventTracker.Event event;
while (events.hasNext()) { while (events.hasNext()) {
event = events.next(); event = events.next();
time = event.time.getTime() - now; time = event.getTime() - now;
bytes = ((Long) event.payload).longValue(); bytes = ((Long) event.payload).longValue();
x1 = (int) (time/1000); x1 = (int) (time/1000);
y1 = (int) (bytes / 1024 / 1024); y1 = (int) (bytes / 1024 / 1024);
@ -138,7 +138,7 @@ public class ProfilingGraph {
int words; int words;
while (events.hasNext()) { while (events.hasNext()) {
event = events.next(); event = events.next();
time = event.time.getTime() - now; time = event.getTime() - now;
words = (int) ((Long) event.payload).longValue(); words = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000); x1 = (int) (time/1000);
y1 = words; y1 = words;
@ -158,7 +158,7 @@ public class ProfilingGraph {
int ppm; int ppm;
while (events.hasNext()) { while (events.hasNext()) {
event = events.next(); event = events.next();
time = event.time.getTime() - now; time = event.getTime() - now;
ppm = (int) ((Long) event.payload).longValue(); ppm = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000); x1 = (int) (time/1000);
y1 = ppm; y1 = ppm;
@ -180,7 +180,7 @@ public class ProfilingGraph {
String pingPeer; String pingPeer;
while (events.hasNext()) { while (events.hasNext()) {
event = events.next(); event = events.next();
time = event.time.getTime() - now; time = event.getTime() - now;
ping = (EventPing) event.payload; ping = (EventPing) event.payload;
x1 = (int) (time/1000); x1 = (int) (time/1000);
y1 = Math.abs((ping.outgoing ? ping.toPeer : ping.fromPeer).hashCode()) % vspace; y1 = Math.abs((ping.outgoing ? ping.toPeer : ping.fromPeer).hashCode()) % vspace;

40
source/net/yacy/search/EventTracker.java
Unescape View File

@ -26,6 +26,7 @@
package net.yacy.search; package net.yacy.search;
import java.text.ParseException;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
@ -103,7 +104,7 @@ public class EventTracker {
final long now = System.currentTimeMillis(); final long now = System.currentTimeMillis();
while (!history.isEmpty()) { while (!history.isEmpty()) {
e = history.peek(); e = history.peek();
if (now - e.time.getTime() < maxQueueAge) break; if (now - e.getTime() < maxQueueAge) break;
history.poll(); history.poll();
} }
} }
@ -123,13 +124,13 @@ public class EventTracker {
final long now = System.currentTimeMillis(); final long now = System.currentTimeMillis();
int count = 0; int count = 0;
while (event.hasNext()) { while (event.hasNext()) {
if (now - event.next().time.getTime() < time) count++; if (now - event.next().getTime() < time) count++;
} }
return count; return count;
} }
public final static class Event { public final static class Event {
final public Date time; final private Object time; // either a String in SHORT_SECOND format, a Long with ms since epoch or Date;
final public int duration; // ms final public int duration; // ms
final public String type; final public String type;
final public Object payload; final public Object payload;
@ -137,9 +138,40 @@ public class EventTracker {
public Event(final Date time, final int duration, final String type, final Object payload, final int count) { public Event(final Date time, final int duration, final String type, final Object payload, final int count) {
this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count; this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count;
} }
public Event(final Long time, final int duration, final String type, final Object payload, final int count) {
this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count;
}
public Event(final String time, final int duration, final String type, final Object payload, final int count) {
this.time = time; this.duration = duration; this.type = type; this.payload = payload; this.count = count;
}
public String getFormattedDate() {
if (this.time instanceof String) return (String) this.time;
if (this.time instanceof Long) return GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date((Long) this.time));
if (this.time instanceof Date) return GenericFormatter.SHORT_SECOND_FORMATTER.format((Date) this.time);
return null;
}
public long getTime() {
if (this.time instanceof String) try {
return GenericFormatter.SHORT_SECOND_FORMATTER.parse((String) this.time).getTime();
} catch (ParseException e) {
return -1L;
}
if (this.time instanceof Long) return (Long) this.time;
if (this.time instanceof Date) return ((Date) this.time).getTime();
return -1L;
}
public Date getDate() {
if (this.time instanceof String) try {
return GenericFormatter.SHORT_SECOND_FORMATTER.parse((String) this.time);
} catch (ParseException e) {
return null;
}if (this.time instanceof Long) return new Date((Long) this.time);
if (this.time instanceof Date) return (Date) this.time;
return null;
}
@Override @Override
public String toString() { public String toString() {
return type + " " + GenericFormatter.SHORT_SECOND_FORMATTER.format(time) + (duration == 0 ? " " : "(" + duration + "ms) ") + (count == 0 ? " " : "[" + count + "] ") + payload; return type + " " + getFormattedDate() + (duration == 0 ? " " : "(" + duration + "ms) ") + (count == 0 ? " " : "[" + count + "] ") + payload;
} }
} }
} }

42
source/net/yacy/search/query/AccessTracker.java
Unescape View File

@ -25,15 +25,20 @@
package net.yacy.search.query; package net.yacy.search.query;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;
import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.WordCache; import net.yacy.cora.document.WordCache;
@ -91,6 +96,10 @@ public class AccessTracker {
dumpFile = f; dumpFile = f;
} }
public static File getDumpFile() {
return dumpFile;
}
public static void add(final Location location, final QueryParams query, int resultCount) { public static void add(final Location location, final QueryParams query, int resultCount) {
if (location == Location.local) synchronized (localSearches) {add(localSearches, query, resultCount);} if (location == Location.local) synchronized (localSearches) {add(localSearches, query, resultCount);}
if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query, resultCount);} if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query, resultCount);}
@ -208,8 +217,8 @@ public class AccessTracker {
* @param to the right boundary of the sequence to search for (excluded) * @param to the right boundary of the sequence to search for (excluded)
* @return a list of lines within the given dates * @return a list of lines within the given dates
*/ */
public static ArrayList<EventTracker.Event> readLog(File f, Date from, Date to) { public static List<EventTracker.Event> readLog(File f, Date from, Date to) {
ArrayList<EventTracker.Event> list = new ArrayList<>(); List<EventTracker.Event> events = new ArrayList<>();
RandomAccessFile raf = null; RandomAccessFile raf = null;
try { try {
raf = new RandomAccessFile(f, "r"); raf = new RandomAccessFile(f, "r");
@ -217,23 +226,32 @@ public class AccessTracker {
if (fd.after(from)) from = fd; if (fd.after(from)) from = fd;
long seekFrom = binarySearch(raf, from, 0, raf.length()); long seekFrom = binarySearch(raf, from, 0, raf.length());
long seekTo = binarySearch(raf, to, seekFrom, raf.length()); long seekTo = binarySearch(raf, to, seekFrom, raf.length());
Date eDate = readDate(raf, seekTo); //Date eDate = readDate(raf, seekTo);
if (eDate.before(to)) seekTo = raf.length(); //if (eDate.before(to)) seekTo = raf.length();
raf.seek(seekFrom); raf.seek(seekFrom);
byte[] buffer = new byte[(int) (seekTo - seekFrom)];
raf.readFully(buffer); // we make a copy because that dramatically speeds up reading lines; RandomAccessFile.readLine is very slow
raf.close();
ByteArrayInputStream bais = new ByteArrayInputStream(buffer);
BufferedReader reader = new BufferedReader(new InputStreamReader(bais, "UTF-8"));
String line; String line;
while (raf.getFilePointer() < seekTo && (line = raf.readLine()) != null) { Pattern sp = Pattern.compile(" ");
while ((line = reader.readLine()) != null) {
// parse the line // parse the line
String[] ls = line.split(" "); String[] ls = sp.split(line);
EventTracker.Event event; EventTracker.Event event;
try { if (ls.length > 1) try {
event = new EventTracker.Event(GenericFormatter.SHORT_SECOND_FORMATTER.parse(ls[0]), 0, "query", line.substring(ls[0].length() + ls[1].length() + 2), Integer.valueOf(ls[1])); event = new EventTracker.Event(ls[0], 0, "query", line.substring(ls[0].length() + ls[1].length() + 2), Integer.valueOf(ls[1]));
list.add(event); events.add(event);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
continue; continue;
} catch (ParseException e) { } catch (Throwable e) {
continue; continue;
} }
} }
reader.close();
bais.close();
buffer = null;
} catch (final FileNotFoundException e) { } catch (final FileNotFoundException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} catch (final IOException e) { } catch (final IOException e) {
@ -241,7 +259,7 @@ public class AccessTracker {
} finally { } finally {
if (raf != null) try {raf.close();} catch (final IOException e) {} if (raf != null) try {raf.close();} catch (final IOException e) {}
} }
return list; return events;
} }
/** /**
@ -302,7 +320,7 @@ public class AccessTracker {
try { try {
from = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1]); from = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[1]);
Date to = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2]); Date to = GenericFormatter.SHORT_SECOND_FORMATTER.parse(args[2]);
ArrayList<EventTracker.Event> dump = readLog(new File(file), from, to); List<EventTracker.Event> dump = readLog(new File(file), from, to);
for (EventTracker.Event s: dump) System.out.println(s.toString()); for (EventTracker.Event s: dump) System.out.println(s.toString());
} catch (ParseException e) { } catch (ParseException e) {
e.printStackTrace(); e.printStackTrace();

Write Preview
Loading…
Cancel
Save