boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

new access tracking data type strategy; previous data types may have caused deadlocks of httpd when performing statistic cleanups

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6809 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent fc43f3028e
commit 0d04ab1422
3 changed files with 71 additions and 45 deletions
Show Stats Download Patch File Download Diff File

48
htroot/AccessTracker_p.java
Unescape View File

@ -28,10 +28,10 @@ import java.util.ArrayList;
import java.util.ConcurrentModificationException;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.text.SimpleDateFormat;
@ -40,23 +40,25 @@ import de.anomic.http.server.RequestHeader;
import de.anomic.net.natLib;
import de.anomic.search.QueryParams;
import de.anomic.search.Switchboard;
import de.anomic.server.serverAccessTracker;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.serverAccessTracker.Track;
import de.anomic.yacy.yacySeed;
public class AccessTracker_p {
private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US);
private static final SortedMap<Long, String> treemapclone(final SortedMap<Long, String> m) {
final TreeMap<Long, String> accessClone = new TreeMap<Long, String>();
try {
accessClone.putAll(m);
} catch (final ConcurrentModificationException e) {}
return accessClone;
private static final List<Track> listclone(final List<Track> m) {
final List<Track> accessClone = new LinkedList<Track>();
try {
accessClone.addAll(m);
} catch (final ConcurrentModificationException e) {}
return accessClone;
}
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
@ -72,17 +74,17 @@ public class AccessTracker_p {
if (page == 0) {
final Iterator<String> i = sb.accessHosts();
String host;
SortedMap<Long, String> access;
List<Track> access;
int entCount = 0;
try {
while ((entCount < maxCount) && (i.hasNext())) {
host = i.next();
access = sb.accessTrack(host);
prop.putHTML("page_list_" + entCount + "_host", host);
prop.putNum("page_list_" + entCount + "_countSecond", access.tailMap(Long.valueOf(System.currentTimeMillis() - 1000)).size());
prop.putNum("page_list_" + entCount + "_countMinute", access.tailMap(Long.valueOf(System.currentTimeMillis() - 1000 * 60)).size());
prop.putNum("page_list_" + entCount + "_count10Minutes", access.tailMap(Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 10)).size());
prop.putNum("page_list_" + entCount + "_countHour", access.tailMap(Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size());
prop.putNum("page_list_" + entCount + "_countSecond", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000)).size());
prop.putNum("page_list_" + entCount + "_countMinute", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60)).size());
prop.putNum("page_list_" + entCount + "_count10Minutes", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 10)).size());
prop.putNum("page_list_" + entCount + "_countHour", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size());
entCount++;
}
} catch (final ConcurrentModificationException e) {} // we don't want to synchronize this
@ -102,18 +104,18 @@ public class AccessTracker_p {
if (page == 1) {
String host = (post == null) ? "" : post.get("host", "");
int entCount = 0;
SortedMap<Long, String> access;
Map.Entry<Long, String> entry;
List<Track> access;
Track entry;
if (host.length() > 0) {
access = sb.accessTrack(host);
if (access != null) {
try {
final Iterator<Map.Entry<Long, String>> ii = treemapclone(access).entrySet().iterator();
final Iterator<Track> ii = listclone(access).iterator();
while (ii.hasNext()) {
entry = ii.next();
prop.putHTML("page_list_" + entCount + "_host", host);
prop.put("page_list_" + entCount + "_date", SimpleFormatter.format(new Date((entry.getKey()).longValue())));
prop.putHTML("page_list_" + entCount + "_path", entry.getValue());
prop.put("page_list_" + entCount + "_date", SimpleFormatter.format(new Date(entry.getTime())));
prop.putHTML("page_list_" + entCount + "_path", entry.getPath());
entCount++;
}
} catch (final ConcurrentModificationException e) {} // we don't want to synchronize this
@ -124,12 +126,12 @@ public class AccessTracker_p {
while ((entCount < maxCount) && (i.hasNext())) {
host = i.next();
access = sb.accessTrack(host);
final Iterator<Map.Entry<Long, String>> ii = treemapclone(access).entrySet().iterator();
final Iterator<Track> ii = listclone(access).iterator();
while (ii.hasNext()) {
entry = ii.next();
prop.putHTML("page_list_" + entCount + "_host", host);
prop.put("page_list_" + entCount + "_date", SimpleFormatter.format(new Date((entry.getKey()).longValue())));
prop.putHTML("page_list_" + entCount + "_path", entry.getValue());
prop.put("page_list_" + entCount + "_date", SimpleFormatter.format(new Date(entry.getTime())));
prop.putHTML("page_list_" + entCount + "_path", entry.getPath());
entCount++;
}
}

63
source/de/anomic/server/serverAccessTracker.java
Unescape View File

@ -22,9 +22,9 @@ package de.anomic.server;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.kelondro.logging.Log;
@ -36,14 +36,29 @@ public class serverAccessTracker {
private final long maxTrackingTime;
private final int maxTrackingCount;
private final int maxHostCount;
private final ConcurrentHashMap<String, SortedMap<Long, String>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
private final ConcurrentHashMap<String, List<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
private long lastCleanup;
public static class Track {
private long time;
private String path;
public Track(long time, String path) {
this.time = time;
this.path = path;
}
public long getTime() {
return this.time;
}
public String getPath() {
return this.path;
}
}
public serverAccessTracker(long maxTrackingTime, int maxTrackingCount, int maxTrackingHostCount) {
this.maxTrackingTime = maxTrackingTime;
this.maxTrackingCount = maxTrackingCount;
this.maxHostCount = maxTrackingHostCount;
this.accessTracker = new ConcurrentHashMap<String, SortedMap<Long, String>>();
this.accessTracker = new ConcurrentHashMap<String, List<Track>>();
}
/*
@ -54,18 +69,18 @@ public class serverAccessTracker {
if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return;
// clear entries which had no entry for the maxTrackingTime time
final Iterator<Map.Entry<String, SortedMap<Long, String>>> i = accessTracker.entrySet().iterator();
SortedMap<Long, String> track;
final Iterator<Map.Entry<String, List<Track>>> i = accessTracker.entrySet().iterator();
List<Track> track;
while (i.hasNext()) {
track = i.next().getValue();
if (track.tailMap(Long.valueOf(System.currentTimeMillis() - maxTrackingTime)).isEmpty()) {
if (tailList(track, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)).isEmpty()) {
// all entries are too old. delete the whole track
i.remove();
} else {
// check if the maxTrackingCount is exceeded
while (track.size() > this.maxTrackingCount) {
// delete the oldest entries
track.remove(track.firstKey());
track.remove(0);
}
}
}
@ -78,39 +93,47 @@ public class serverAccessTracker {
this.lastCleanup = System.currentTimeMillis();
}
public static List<Track> tailList(List<Track> timeList, long time) {
List<Track> t = new LinkedList<Track>();
for (Track l: timeList) if (l.getTime() > time) t.add(l);
return t;
}
private SortedMap<Long, String> clearTooOldAccess(final SortedMap<Long, String> access) {
private List<Track> clearTooOldAccess(final List<Track> access) {
try {
return access.tailMap(Long.valueOf(System.currentTimeMillis() - maxTrackingTime));
return tailList(access, Long.valueOf(System.currentTimeMillis() - maxTrackingTime));
} catch (IllegalArgumentException e) {
Log.logException(e);
return new TreeMap<Long, String>();
return new LinkedList<Track>();
}
}
public void track(final String host, String accessPath) {
// check storage size
if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) {
cleanupAccessTracker();
this.lastCleanup = System.currentTimeMillis();
if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) synchronized (this) {
if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) {
cleanupAccessTracker();
this.lastCleanup = System.currentTimeMillis();
}
}
// learn that a specific host has accessed a specific path
if (accessPath == null) accessPath="NULL";
SortedMap<Long, String> track = accessTracker.get(host);
if (track == null) track = new TreeMap<Long, String>();
List<Track> track = accessTracker.get(host);
if (track == null) track = new LinkedList<Track>();
synchronized (track) {
track.put(Long.valueOf(System.currentTimeMillis()), accessPath);
track.add(new Track(System.currentTimeMillis(), accessPath));
// write back to tracker
accessTracker.put(host, clearTooOldAccess(track));
}
}
public SortedMap<Long, String> accessTrack(final String host) {
public List<Track> accessTrack(final String host) {
// returns mapping from Long(accesstime) to path
SortedMap<Long, String> access = accessTracker.get(host);
List<Track> access = accessTracker.get(host);
if (access == null) return null;
// clear too old entries
synchronized (access) {
@ -128,7 +151,7 @@ public class serverAccessTracker {
public Iterator<String> accessHosts() {
// returns an iterator of hosts in tracker (String)
final HashMap<String, SortedMap<Long, String>> accessTrackerClone = new HashMap<String, SortedMap<Long, String>>();
final HashMap<String, List<Track>> accessTrackerClone = new HashMap<String, List<Track>>();
accessTrackerClone.putAll(accessTracker);
return accessTrackerClone.keySet().iterator();
}

5
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -27,8 +27,8 @@ import java.net.InetAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import net.yacy.kelondro.logging.Log;
@ -37,6 +37,7 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.WorkflowThread;
import de.anomic.server.serverAccessTracker.Track;
import de.anomic.server.serverCore.Session;
public class serverSwitch {
@ -531,7 +532,7 @@ public class serverSwitch {
this.accessTracker.track(host, accessPath);
}
public SortedMap<Long, String> accessTrack(String host) {
public List<Track> accessTrack(String host) {
return this.accessTracker.accessTrack(host);
}

Write Preview
Loading…
Cancel
Save