boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

new access tracking data type strategy; previous data types may have caused deadlocks of httpd when performing statistic cleanups

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6809 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent fc43f3028e
commit 0d04ab1422
3 changed files with 71 additions and 45 deletions
Show Stats Download Patch File Download Diff File

44
htroot/AccessTracker_p.java
Unescape View File

@ -28,10 +28,10 @@ import java.util.ArrayList;
import java.util.ConcurrentModificationException; import java.util.ConcurrentModificationException;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
@ -40,21 +40,23 @@ import de.anomic.http.server.RequestHeader;
import de.anomic.net.natLib; import de.anomic.net.natLib;
import de.anomic.search.QueryParams; import de.anomic.search.QueryParams;
import de.anomic.search.Switchboard; import de.anomic.search.Switchboard;
import de.anomic.server.serverAccessTracker;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.server.serverAccessTracker.Track;
import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeed;
public class AccessTracker_p { public class AccessTracker_p {
private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US); private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US);
private static final SortedMap<Long, String> treemapclone(final SortedMap<Long, String> m) { private static final List<Track> listclone(final List<Track> m) {
final TreeMap<Long, String> accessClone = new TreeMap<Long, String>(); final List<Track> accessClone = new LinkedList<Track>();
try { try {
accessClone.putAll(m); accessClone.addAll(m);
} catch (final ConcurrentModificationException e) {} } catch (final ConcurrentModificationException e) {}
return accessClone; return accessClone;
} }
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
@ -72,17 +74,17 @@ public class AccessTracker_p {
if (page == 0) { if (page == 0) {
final Iterator<String> i = sb.accessHosts(); final Iterator<String> i = sb.accessHosts();
String host; String host;
SortedMap<Long, String> access; List<Track> access;
int entCount = 0; int entCount = 0;
try { try {
while ((entCount < maxCount) && (i.hasNext())) { while ((entCount < maxCount) && (i.hasNext())) {
host = i.next(); host = i.next();
access = sb.accessTrack(host); access = sb.accessTrack(host);
prop.putHTML("page_list_" + entCount + "_host", host); prop.putHTML("page_list_" + entCount + "_host", host);
prop.putNum("page_list_" + entCount + "_countSecond", access.tailMap(Long.valueOf(System.currentTimeMillis() - 1000)).size()); prop.putNum("page_list_" + entCount + "_countSecond", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000)).size());
prop.putNum("page_list_" + entCount + "_countMinute", access.tailMap(Long.valueOf(System.currentTimeMillis() - 1000 * 60)).size()); prop.putNum("page_list_" + entCount + "_countMinute", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60)).size());
prop.putNum("page_list_" + entCount + "_count10Minutes", access.tailMap(Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 10)).size()); prop.putNum("page_list_" + entCount + "_count10Minutes", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 10)).size());
prop.putNum("page_list_" + entCount + "_countHour", access.tailMap(Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size()); prop.putNum("page_list_" + entCount + "_countHour", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size());
entCount++; entCount++;
} }
} catch (final ConcurrentModificationException e) {} // we don't want to synchronize this } catch (final ConcurrentModificationException e) {} // we don't want to synchronize this
@ -102,18 +104,18 @@ public class AccessTracker_p {
if (page == 1) { if (page == 1) {
String host = (post == null) ? "" : post.get("host", ""); String host = (post == null) ? "" : post.get("host", "");
int entCount = 0; int entCount = 0;
SortedMap<Long, String> access; List<Track> access;
Map.Entry<Long, String> entry; Track entry;
if (host.length() > 0) { if (host.length() > 0) {
access = sb.accessTrack(host); access = sb.accessTrack(host);
if (access != null) { if (access != null) {
try { try {
final Iterator<Map.Entry<Long, String>> ii = treemapclone(access).entrySet().iterator(); final Iterator<Track> ii = listclone(access).iterator();
while (ii.hasNext()) { while (ii.hasNext()) {
entry = ii.next(); entry = ii.next();
prop.putHTML("page_list_" + entCount + "_host", host); prop.putHTML("page_list_" + entCount + "_host", host);
prop.put("page_list_" + entCount + "_date", SimpleFormatter.format(new Date((entry.getKey()).longValue()))); prop.put("page_list_" + entCount + "_date", SimpleFormatter.format(new Date(entry.getTime())));
prop.putHTML("page_list_" + entCount + "_path", entry.getValue()); prop.putHTML("page_list_" + entCount + "_path", entry.getPath());
entCount++; entCount++;
} }
} catch (final ConcurrentModificationException e) {} // we don't want to synchronize this } catch (final ConcurrentModificationException e) {} // we don't want to synchronize this
@ -124,12 +126,12 @@ public class AccessTracker_p {
while ((entCount < maxCount) && (i.hasNext())) { while ((entCount < maxCount) && (i.hasNext())) {
host = i.next(); host = i.next();
access = sb.accessTrack(host); access = sb.accessTrack(host);
final Iterator<Map.Entry<Long, String>> ii = treemapclone(access).entrySet().iterator(); final Iterator<Track> ii = listclone(access).iterator();
while (ii.hasNext()) { while (ii.hasNext()) {
entry = ii.next(); entry = ii.next();
prop.putHTML("page_list_" + entCount + "_host", host); prop.putHTML("page_list_" + entCount + "_host", host);
prop.put("page_list_" + entCount + "_date", SimpleFormatter.format(new Date((entry.getKey()).longValue()))); prop.put("page_list_" + entCount + "_date", SimpleFormatter.format(new Date(entry.getTime())));
prop.putHTML("page_list_" + entCount + "_path", entry.getValue()); prop.putHTML("page_list_" + entCount + "_path", entry.getPath());
entCount++; entCount++;
} }
} }

63
source/de/anomic/server/serverAccessTracker.java
Unescape View File

@ -22,9 +22,9 @@ package de.anomic.server;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -36,14 +36,29 @@ public class serverAccessTracker {
private final long maxTrackingTime; private final long maxTrackingTime;
private final int maxTrackingCount; private final int maxTrackingCount;
private final int maxHostCount; private final int maxHostCount;
private final ConcurrentHashMap<String, SortedMap<Long, String>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries private final ConcurrentHashMap<String, List<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
private long lastCleanup; private long lastCleanup;
public static class Track {
private long time;
private String path;
public Track(long time, String path) {
this.time = time;
this.path = path;
}
public long getTime() {
return this.time;
}
public String getPath() {
return this.path;
}
}
public serverAccessTracker(long maxTrackingTime, int maxTrackingCount, int maxTrackingHostCount) { public serverAccessTracker(long maxTrackingTime, int maxTrackingCount, int maxTrackingHostCount) {
this.maxTrackingTime = maxTrackingTime; this.maxTrackingTime = maxTrackingTime;
this.maxTrackingCount = maxTrackingCount; this.maxTrackingCount = maxTrackingCount;
this.maxHostCount = maxTrackingHostCount; this.maxHostCount = maxTrackingHostCount;
this.accessTracker = new ConcurrentHashMap<String, SortedMap<Long, String>>(); this.accessTracker = new ConcurrentHashMap<String, List<Track>>();
} }
/* /*
@ -54,18 +69,18 @@ public class serverAccessTracker {
if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return;
// clear entries which had no entry for the maxTrackingTime time // clear entries which had no entry for the maxTrackingTime time
final Iterator<Map.Entry<String, SortedMap<Long, String>>> i = accessTracker.entrySet().iterator(); final Iterator<Map.Entry<String, List<Track>>> i = accessTracker.entrySet().iterator();
SortedMap<Long, String> track; List<Track> track;
while (i.hasNext()) { while (i.hasNext()) {
track = i.next().getValue(); track = i.next().getValue();
if (track.tailMap(Long.valueOf(System.currentTimeMillis() - maxTrackingTime)).isEmpty()) { if (tailList(track, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)).isEmpty()) {
// all entries are too old. delete the whole track // all entries are too old. delete the whole track
i.remove(); i.remove();
} else { } else {
// check if the maxTrackingCount is exceeded // check if the maxTrackingCount is exceeded
while (track.size() > this.maxTrackingCount) { while (track.size() > this.maxTrackingCount) {
// delete the oldest entries // delete the oldest entries
track.remove(track.firstKey()); track.remove(0);
} }
} }
} }
@ -79,38 +94,46 @@ public class serverAccessTracker {
this.lastCleanup = System.currentTimeMillis(); this.lastCleanup = System.currentTimeMillis();
} }
private SortedMap<Long, String> clearTooOldAccess(final SortedMap<Long, String> access) { public static List<Track> tailList(List<Track> timeList, long time) {
List<Track> t = new LinkedList<Track>();
for (Track l: timeList) if (l.getTime() > time) t.add(l);
return t;
}
private List<Track> clearTooOldAccess(final List<Track> access) {
try { try {
return access.tailMap(Long.valueOf(System.currentTimeMillis() - maxTrackingTime)); return tailList(access, Long.valueOf(System.currentTimeMillis() - maxTrackingTime));
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Log.logException(e); Log.logException(e);
return new TreeMap<Long, String>(); return new LinkedList<Track>();
} }
} }
public void track(final String host, String accessPath) { public void track(final String host, String accessPath) {
// check storage size // check storage size
if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) { if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) synchronized (this) {
cleanupAccessTracker(); if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) {
this.lastCleanup = System.currentTimeMillis(); cleanupAccessTracker();
this.lastCleanup = System.currentTimeMillis();
}
} }
// learn that a specific host has accessed a specific path // learn that a specific host has accessed a specific path
if (accessPath == null) accessPath="NULL"; if (accessPath == null) accessPath="NULL";
SortedMap<Long, String> track = accessTracker.get(host); List<Track> track = accessTracker.get(host);
if (track == null) track = new TreeMap<Long, String>(); if (track == null) track = new LinkedList<Track>();
synchronized (track) { synchronized (track) {
track.put(Long.valueOf(System.currentTimeMillis()), accessPath); track.add(new Track(System.currentTimeMillis(), accessPath));
// write back to tracker // write back to tracker
accessTracker.put(host, clearTooOldAccess(track)); accessTracker.put(host, clearTooOldAccess(track));
} }
} }
public SortedMap<Long, String> accessTrack(final String host) { public List<Track> accessTrack(final String host) {
// returns mapping from Long(accesstime) to path // returns mapping from Long(accesstime) to path
SortedMap<Long, String> access = accessTracker.get(host); List<Track> access = accessTracker.get(host);
if (access == null) return null; if (access == null) return null;
// clear too old entries // clear too old entries
synchronized (access) { synchronized (access) {
@ -128,7 +151,7 @@ public class serverAccessTracker {
public Iterator<String> accessHosts() { public Iterator<String> accessHosts() {
// returns an iterator of hosts in tracker (String) // returns an iterator of hosts in tracker (String)
final HashMap<String, SortedMap<Long, String>> accessTrackerClone = new HashMap<String, SortedMap<Long, String>>(); final HashMap<String, List<Track>> accessTrackerClone = new HashMap<String, List<Track>>();
accessTrackerClone.putAll(accessTracker); accessTrackerClone.putAll(accessTracker);
return accessTrackerClone.keySet().iterator(); return accessTrackerClone.keySet().iterator();
} }

5
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -27,8 +27,8 @@ import java.net.InetAddress;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap; import java.util.TreeMap;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -37,6 +37,7 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.workflow.BusyThread; import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.WorkflowThread; import net.yacy.kelondro.workflow.WorkflowThread;
import de.anomic.server.serverAccessTracker.Track;
import de.anomic.server.serverCore.Session; import de.anomic.server.serverCore.Session;
public class serverSwitch { public class serverSwitch {
@ -531,7 +532,7 @@ public class serverSwitch {
this.accessTracker.track(host, accessPath); this.accessTracker.track(host, accessPath);
} }
public SortedMap<Long, String> accessTrack(String host) { public List<Track> accessTrack(String host) {
return this.accessTracker.accessTrack(host); return this.accessTracker.accessTrack(host);
} }

Write Preview
Loading…
Cancel
Save