boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some redesign in the access tracker to realize sixcoolers question about "smartes way for deleting the first Object":

Browse Source 
- not so much abstraction for a collection, makes use of remove() (no operands) possible
- different way to delete elements in track (destructive, not constructive (less copies of elements in new queue))
- more abstraction for class api since no static class must be used any more

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7169 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 03f0414025
commit a2f9974745
3 changed files with 57 additions and 51 deletions
Show Stats Download Patch File Download Diff File

11
htroot/AccessTracker_p.java
Unescape View File

@ -42,7 +42,6 @@ import net.yacy.cora.protocol.RequestHeader;
import de.anomic.search.QueryParams; import de.anomic.search.QueryParams;
import de.anomic.search.Switchboard; import de.anomic.search.Switchboard;
import de.anomic.server.serverAccessTracker;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -76,17 +75,15 @@ public class AccessTracker_p {
if (page == 0) { if (page == 0) {
final Iterator<String> i = sb.accessHosts(); final Iterator<String> i = sb.accessHosts();
String host; String host;
Collection<Track> access;
int entCount = 0; int entCount = 0;
try { try {
while ((entCount < maxCount) && (i.hasNext())) { while ((entCount < maxCount) && (i.hasNext())) {
host = i.next(); host = i.next();
access = sb.accessTrack(host);
prop.putHTML("page_list_" + entCount + "_host", host); prop.putHTML("page_list_" + entCount + "_host", host);
prop.putNum("page_list_" + entCount + "_countSecond", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000)).size()); prop.putNum("page_list_" + entCount + "_countSecond", sb.latestAccessCount(host, 1000));
prop.putNum("page_list_" + entCount + "_countMinute", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60)).size()); prop.putNum("page_list_" + entCount + "_countMinute", sb.latestAccessCount(host, 1000 * 60));
prop.putNum("page_list_" + entCount + "_count10Minutes", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 10)).size()); prop.putNum("page_list_" + entCount + "_count10Minutes", sb.latestAccessCount(host, 1000 * 60 * 10));
prop.putNum("page_list_" + entCount + "_countHour", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size()); prop.putNum("page_list_" + entCount + "_countHour", sb.latestAccessCount(host, 1000 * 60 * 60));
entCount++; entCount++;
} }
} catch (final ConcurrentModificationException e) {} // we don't want to synchronize this } catch (final ConcurrentModificationException e) {} // we don't want to synchronize this

85
source/de/anomic/server/serverAccessTracker.java
Unescape View File

@ -23,11 +23,10 @@ package de.anomic.server;
import java.util.Collection; import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentLinkedQueue;
import net.yacy.kelondro.logging.Log;
public class serverAccessTracker { public class serverAccessTracker {
private static final long cleanupCycle = 60000; // 1 minute private static final long cleanupCycle = 60000; // 1 minute
@ -35,7 +34,7 @@ public class serverAccessTracker {
private final long maxTrackingTime; private final long maxTrackingTime;
private final int maxTrackingCount; private final int maxTrackingCount;
private final int maxHostCount; private final int maxHostCount;
private final ConcurrentHashMap<String, Collection<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries private final ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
private long lastCleanup; private long lastCleanup;
public static class Track { public static class Track {
@ -57,7 +56,7 @@ public class serverAccessTracker {
this.maxTrackingTime = maxTrackingTime; this.maxTrackingTime = maxTrackingTime;
this.maxTrackingCount = maxTrackingCount; this.maxTrackingCount = maxTrackingCount;
this.maxHostCount = maxTrackingHostCount; this.maxHostCount = maxTrackingHostCount;
this.accessTracker = new ConcurrentHashMap<String, Collection<Track>>(); this.accessTracker = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>();
} }
/* /*
@ -65,50 +64,57 @@ public class serverAccessTracker {
*/ */
private synchronized void cleanupAccessTracker() { private synchronized void cleanupAccessTracker() {
// if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; // avoid too many scans of the queues
this.lastCleanup = System.currentTimeMillis(); this.lastCleanup = System.currentTimeMillis();
// clear entries which had no entry for the maxTrackingTime time // clear entries which had no entry for the maxTrackingTime time
final Iterator<Map.Entry<String, Collection<Track>>> i = accessTracker.entrySet().iterator(); final Iterator<Map.Entry<String, ConcurrentLinkedQueue<Track>>> i = accessTracker.entrySet().iterator();
Iterator<Track> it; ConcurrentLinkedQueue<Track> track;
Collection<Track> track;
while (i.hasNext()) { while (i.hasNext()) {
track = i.next().getValue(); track = i.next().getValue();
if (tailList(track, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)).isEmpty()) { clearTooOldAccess(track);
if (track.isEmpty()) {
// all entries are too old. delete the whole track // all entries are too old. delete the whole track
i.remove(); i.remove();
} else { } else {
// check if the maxTrackingCount is exceeded // check if the maxTrackingCount is exceeded
it = track.iterator(); while (track.size() > this.maxTrackingCount) try {
while (track.size() > this.maxTrackingCount && it.hasNext()) {
// delete the oldest entries // delete the oldest entries
// track.remove(0); track.remove();
track.remove(it.next()); } catch (NoSuchElementException e) { break; } // concurrency may cause that the track is already empty
}
} }
} }
// if there are more entries left than maxTrackingCount, delete some. // if there are more entries left than maxTrackingCount, delete some.
while (accessTracker.size() > this.maxHostCount) { while (accessTracker.size() > this.maxHostCount) {
// delete just any // delete just any
accessTracker.remove(accessTracker.keys().nextElement()); String key = accessTracker.keys().nextElement();
if (key == null) break; // may occur because of concurrency effects
accessTracker.remove(key);
} }
// this.lastCleanup = System.currentTimeMillis();
} }
public static Collection<Track> tailList(Collection<Track> timeList, long time) { /**
Collection<Track> t = new ConcurrentLinkedQueue<Track>(); * compute the number of accesses to a given host in the latest time
for (Track l: timeList) if (l.getTime() > time) t.add(l); * @param host the host that was accessed
return t; * @param delta the time delta from now to the past where the access times shall be computed
* @return the number of accesses to the host in the given time span
*/
public int latestAccessCount(final String host, long delta) {
Collection<Track> timeList = accessTrack(host);
long time = System.currentTimeMillis() - delta;
int c = 0;
for (Track l: timeList) if (l.getTime() > time) c++;
return c;
} }
private Collection<Track> clearTooOldAccess(final Collection<Track> access) { private void clearTooOldAccess(final ConcurrentLinkedQueue<Track> access) {
try { Long time = Long.valueOf(System.currentTimeMillis() - maxTrackingTime);
return tailList(access, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)); Iterator<Track> e = access.iterator();
} catch (IllegalArgumentException e) { Track l;
Log.logException(e); while (e.hasNext()) {
return new ConcurrentLinkedQueue<Track>(); l = e.next();
if (l.getTime() <= time) e.remove();
} }
} }
@ -120,35 +126,34 @@ public class serverAccessTracker {
// learn that a specific host has accessed a specific path // learn that a specific host has accessed a specific path
if (accessPath == null) accessPath="NULL"; if (accessPath == null) accessPath="NULL";
Collection<Track> track = accessTracker.get(host); ConcurrentLinkedQueue<Track> track = accessTracker.get(host);
if (track == null) track = new ConcurrentLinkedQueue<Track>(); if (track == null) {
track = new ConcurrentLinkedQueue<Track>();
track.add(new Track(System.currentTimeMillis(), accessPath));
// add to tracker
accessTracker.put(host, track);
} else {
track.add(new Track(System.currentTimeMillis(), accessPath)); track.add(new Track(System.currentTimeMillis(), accessPath));
// write back to tracker clearTooOldAccess(track);
accessTracker.put(host, clearTooOldAccess(track)); }
} }
public Collection<Track> accessTrack(final String host) { public Collection<Track> accessTrack(final String host) {
// returns mapping from Long(accesstime) to path // returns mapping from Long(accesstime) to path
Collection<Track> access = accessTracker.get(host); ConcurrentLinkedQueue<Track> access = accessTracker.get(host);
if (access == null) return null; if (access == null) return null;
// clear too old entries // clear too old entries
synchronized (access) { clearTooOldAccess(access);
if (access.size() != (access = clearTooOldAccess(access)).size()) {
// write back to tracker
if (access.isEmpty()) { if (access.isEmpty()) {
accessTracker.remove(host); accessTracker.remove(host);
} else {
accessTracker.put(host, access);
}
}
} }
return access; return access;
} }
public Iterator<String> accessHosts() { public Iterator<String> accessHosts() {
// returns an iterator of hosts in tracker (String) // returns an iterator of hosts in tracker (String)
final Map<String, Collection<Track>> accessTrackerClone = new ConcurrentHashMap<String, Collection<Track>>(); final Map<String, ConcurrentLinkedQueue<Track>> accessTrackerClone = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>();
accessTrackerClone.putAll(accessTracker); accessTrackerClone.putAll(accessTracker);
return accessTrackerClone.keySet().iterator(); return accessTrackerClone.keySet().iterator();
} }

4
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -545,6 +545,10 @@ public class serverSwitch {
return this.accessTracker.accessTrack(host); return this.accessTracker.accessTrack(host);
} }
public int latestAccessCount(final String host, long timedelta) {
return this.accessTracker.latestAccessCount(host, timedelta);
}
public Iterator<String> accessHosts() { public Iterator<String> accessHosts() {
return this.accessTracker.accessHosts(); return this.accessTracker.accessHosts();
} }

Write Preview
Loading…
Cancel
Save