From a2f9974745f2588646c459e272878251285e9adf Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 19 Sep 2010 23:00:24 +0000 Subject: [PATCH] some redesign in the access tracker to realize sixcoolers question about "smartes way for deleting the first Object": - not so much abstraction for a collection, makes use of remove() (no operands) possible - different way to delete elements in track (destructive, not constructive (less copies of elements in new queue)) - more abstraction for class api since no static class must be used any more git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7169 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/AccessTracker_p.java | 11 +-- .../de/anomic/server/serverAccessTracker.java | 93 ++++++++++--------- source/de/anomic/server/serverSwitch.java | 4 + 3 files changed, 57 insertions(+), 51 deletions(-) diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.java index 926e2421a..8853b9c49 100644 --- a/htroot/AccessTracker_p.java +++ b/htroot/AccessTracker_p.java @@ -42,7 +42,6 @@ import net.yacy.cora.protocol.RequestHeader; import de.anomic.search.QueryParams; import de.anomic.search.Switchboard; -import de.anomic.server.serverAccessTracker; import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -76,17 +75,15 @@ public class AccessTracker_p { if (page == 0) { final Iterator i = sb.accessHosts(); String host; - Collection access; int entCount = 0; try { while ((entCount < maxCount) && (i.hasNext())) { host = i.next(); - access = sb.accessTrack(host); prop.putHTML("page_list_" + entCount + "_host", host); - prop.putNum("page_list_" + entCount + "_countSecond", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000)).size()); - prop.putNum("page_list_" + entCount + "_countMinute", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60)).size()); - prop.putNum("page_list_" + entCount + "_count10Minutes", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 10)).size()); - prop.putNum("page_list_" + entCount + "_countHour", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size()); + prop.putNum("page_list_" + entCount + "_countSecond", sb.latestAccessCount(host, 1000)); + prop.putNum("page_list_" + entCount + "_countMinute", sb.latestAccessCount(host, 1000 * 60)); + prop.putNum("page_list_" + entCount + "_count10Minutes", sb.latestAccessCount(host, 1000 * 60 * 10)); + prop.putNum("page_list_" + entCount + "_countHour", sb.latestAccessCount(host, 1000 * 60 * 60)); entCount++; } } catch (final ConcurrentModificationException e) {} // we don't want to synchronize this diff --git a/source/de/anomic/server/serverAccessTracker.java b/source/de/anomic/server/serverAccessTracker.java index 64edb41ed..c0d3eb0be 100644 --- a/source/de/anomic/server/serverAccessTracker.java +++ b/source/de/anomic/server/serverAccessTracker.java @@ -23,11 +23,10 @@ package de.anomic.server; import java.util.Collection; import java.util.Iterator; import java.util.Map; +import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import net.yacy.kelondro.logging.Log; - public class serverAccessTracker { private static final long cleanupCycle = 60000; // 1 minute @@ -35,7 +34,7 @@ public class serverAccessTracker { private final long maxTrackingTime; private final int maxTrackingCount; private final int maxHostCount; - private final ConcurrentHashMap> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries + private final ConcurrentHashMap> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries private long lastCleanup; public static class Track { @@ -57,7 +56,7 @@ public class serverAccessTracker { this.maxTrackingTime = maxTrackingTime; this.maxTrackingCount = maxTrackingCount; this.maxHostCount = maxTrackingHostCount; - this.accessTracker = new ConcurrentHashMap>(); + this.accessTracker = new ConcurrentHashMap>(); } /* @@ -65,50 +64,57 @@ public class serverAccessTracker { */ private synchronized void cleanupAccessTracker() { -// if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; + if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; // avoid too many scans of the queues this.lastCleanup = System.currentTimeMillis(); // clear entries which had no entry for the maxTrackingTime time - final Iterator>> i = accessTracker.entrySet().iterator(); - Iterator it; - Collection track; + final Iterator>> i = accessTracker.entrySet().iterator(); + ConcurrentLinkedQueue track; while (i.hasNext()) { track = i.next().getValue(); - if (tailList(track, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)).isEmpty()) { + clearTooOldAccess(track); + if (track.isEmpty()) { // all entries are too old. delete the whole track i.remove(); } else { // check if the maxTrackingCount is exceeded - it = track.iterator(); - while (track.size() > this.maxTrackingCount && it.hasNext()) { + while (track.size() > this.maxTrackingCount) try { // delete the oldest entries -// track.remove(0); - track.remove(it.next()); - } + track.remove(); + } catch (NoSuchElementException e) { break; } // concurrency may cause that the track is already empty } } // if there are more entries left than maxTrackingCount, delete some. while (accessTracker.size() > this.maxHostCount) { // delete just any - accessTracker.remove(accessTracker.keys().nextElement()); + String key = accessTracker.keys().nextElement(); + if (key == null) break; // may occur because of concurrency effects + accessTracker.remove(key); } + } -// this.lastCleanup = System.currentTimeMillis(); + /** + * compute the number of accesses to a given host in the latest time + * @param host the host that was accessed + * @param delta the time delta from now to the past where the access times shall be computed + * @return the number of accesses to the host in the given time span + */ + public int latestAccessCount(final String host, long delta) { + Collection timeList = accessTrack(host); + long time = System.currentTimeMillis() - delta; + int c = 0; + for (Track l: timeList) if (l.getTime() > time) c++; + return c; } - public static Collection tailList(Collection timeList, long time) { - Collection t = new ConcurrentLinkedQueue(); - for (Track l: timeList) if (l.getTime() > time) t.add(l); - return t; - } - - private Collection clearTooOldAccess(final Collection access) { - try { - return tailList(access, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)); - } catch (IllegalArgumentException e) { - Log.logException(e); - return new ConcurrentLinkedQueue(); + private void clearTooOldAccess(final ConcurrentLinkedQueue access) { + Long time = Long.valueOf(System.currentTimeMillis() - maxTrackingTime); + Iterator e = access.iterator(); + Track l; + while (e.hasNext()) { + l = e.next(); + if (l.getTime() <= time) e.remove(); } } @@ -120,35 +126,34 @@ public class serverAccessTracker { // learn that a specific host has accessed a specific path if (accessPath == null) accessPath="NULL"; - Collection track = accessTracker.get(host); - if (track == null) track = new ConcurrentLinkedQueue(); - track.add(new Track(System.currentTimeMillis(), accessPath)); - // write back to tracker - accessTracker.put(host, clearTooOldAccess(track)); + ConcurrentLinkedQueue track = accessTracker.get(host); + if (track == null) { + track = new ConcurrentLinkedQueue(); + track.add(new Track(System.currentTimeMillis(), accessPath)); + // add to tracker + accessTracker.put(host, track); + } else { + track.add(new Track(System.currentTimeMillis(), accessPath)); + clearTooOldAccess(track); + } } public Collection accessTrack(final String host) { // returns mapping from Long(accesstime) to path - Collection access = accessTracker.get(host); + ConcurrentLinkedQueue access = accessTracker.get(host); if (access == null) return null; // clear too old entries - synchronized (access) { - if (access.size() != (access = clearTooOldAccess(access)).size()) { - // write back to tracker - if (access.isEmpty()) { - accessTracker.remove(host); - } else { - accessTracker.put(host, access); - } - } + clearTooOldAccess(access); + if (access.isEmpty()) { + accessTracker.remove(host); } return access; } public Iterator accessHosts() { // returns an iterator of hosts in tracker (String) - final Map> accessTrackerClone = new ConcurrentHashMap>(); + final Map> accessTrackerClone = new ConcurrentHashMap>(); accessTrackerClone.putAll(accessTracker); return accessTrackerClone.keySet().iterator(); } diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index 0caddb10c..6ea69d639 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -544,6 +544,10 @@ public class serverSwitch { public Collection accessTrack(String host) { return this.accessTracker.accessTrack(host); } + + public int latestAccessCount(final String host, long timedelta) { + return this.accessTracker.latestAccessCount(host, timedelta); + } public Iterator accessHosts() { return this.accessTracker.accessHosts();