boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some redesign in the access tracker to realize sixcoolers question about "smartes way for deleting the first Object":

Browse Source 
- not so much abstraction for a collection, makes use of remove() (no operands) possible
- different way to delete elements in track (destructive, not constructive (less copies of elements in new queue))
- more abstraction for class api since no static class must be used any more

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7169 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 03f0414025
commit a2f9974745
3 changed files with 57 additions and 51 deletions
Show Stats Download Patch File Download Diff File

11
htroot/AccessTracker_p.java
Unescape View File

@ -42,7 +42,6 @@ import net.yacy.cora.protocol.RequestHeader;
import de.anomic.search.QueryParams;
import de.anomic.search.Switchboard;
import de.anomic.server.serverAccessTracker;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -76,17 +75,15 @@ public class AccessTracker_p {
if (page == 0) {
final Iterator<String> i = sb.accessHosts();
String host;
Collection<Track> access;
int entCount = 0;
try {
while ((entCount < maxCount) && (i.hasNext())) {
host = i.next();
access = sb.accessTrack(host);
prop.putHTML("page_list_" + entCount + "_host", host);
prop.putNum("page_list_" + entCount + "_countSecond", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000)).size());
prop.putNum("page_list_" + entCount + "_countMinute", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60)).size());
prop.putNum("page_list_" + entCount + "_count10Minutes", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 10)).size());
prop.putNum("page_list_" + entCount + "_countHour", serverAccessTracker.tailList(access, Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size());
prop.putNum("page_list_" + entCount + "_countSecond", sb.latestAccessCount(host, 1000));
prop.putNum("page_list_" + entCount + "_countMinute", sb.latestAccessCount(host, 1000 * 60));
prop.putNum("page_list_" + entCount + "_count10Minutes", sb.latestAccessCount(host, 1000 * 60 * 10));
prop.putNum("page_list_" + entCount + "_countHour", sb.latestAccessCount(host, 1000 * 60 * 60));
entCount++;
}
} catch (final ConcurrentModificationException e) {} // we don't want to synchronize this

93
source/de/anomic/server/serverAccessTracker.java
Unescape View File

@ -23,11 +23,10 @@ package de.anomic.server;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import net.yacy.kelondro.logging.Log;
public class serverAccessTracker {
private static final long cleanupCycle = 60000; // 1 minute
@ -35,7 +34,7 @@ public class serverAccessTracker {
private final long maxTrackingTime;
private final int maxTrackingCount;
private final int maxHostCount;
private final ConcurrentHashMap<String, Collection<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
private final ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
private long lastCleanup;
public static class Track {
@ -57,7 +56,7 @@ public class serverAccessTracker {
this.maxTrackingTime = maxTrackingTime;
this.maxTrackingCount = maxTrackingCount;
this.maxHostCount = maxTrackingHostCount;
this.accessTracker = new ConcurrentHashMap<String, Collection<Track>>();
this.accessTracker = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>();
}
/*
@ -65,50 +64,57 @@ public class serverAccessTracker {
*/
private synchronized void cleanupAccessTracker() {
// if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return;
if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; // avoid too many scans of the queues
this.lastCleanup = System.currentTimeMillis();
// clear entries which had no entry for the maxTrackingTime time
final Iterator<Map.Entry<String, Collection<Track>>> i = accessTracker.entrySet().iterator();
Iterator<Track> it;
Collection<Track> track;
final Iterator<Map.Entry<String, ConcurrentLinkedQueue<Track>>> i = accessTracker.entrySet().iterator();
ConcurrentLinkedQueue<Track> track;
while (i.hasNext()) {
track = i.next().getValue();
if (tailList(track, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)).isEmpty()) {
clearTooOldAccess(track);
if (track.isEmpty()) {
// all entries are too old. delete the whole track
i.remove();
} else {
// check if the maxTrackingCount is exceeded
it = track.iterator();
while (track.size() > this.maxTrackingCount && it.hasNext()) {
while (track.size() > this.maxTrackingCount) try {
// delete the oldest entries
// track.remove(0);
track.remove(it.next());
}
track.remove();
} catch (NoSuchElementException e) { break; } // concurrency may cause that the track is already empty
}
}
// if there are more entries left than maxTrackingCount, delete some.
while (accessTracker.size() > this.maxHostCount) {
// delete just any
accessTracker.remove(accessTracker.keys().nextElement());
String key = accessTracker.keys().nextElement();
if (key == null) break; // may occur because of concurrency effects
accessTracker.remove(key);
}
}
// this.lastCleanup = System.currentTimeMillis();
/**
* compute the number of accesses to a given host in the latest time
* @param host the host that was accessed
* @param delta the time delta from now to the past where the access times shall be computed
* @return the number of accesses to the host in the given time span
*/
public int latestAccessCount(final String host, long delta) {
Collection<Track> timeList = accessTrack(host);
long time = System.currentTimeMillis() - delta;
int c = 0;
for (Track l: timeList) if (l.getTime() > time) c++;
return c;
}
public static Collection<Track> tailList(Collection<Track> timeList, long time) {
Collection<Track> t = new ConcurrentLinkedQueue<Track>();
for (Track l: timeList) if (l.getTime() > time) t.add(l);
return t;
}
private Collection<Track> clearTooOldAccess(final Collection<Track> access) {
try {
return tailList(access, Long.valueOf(System.currentTimeMillis() - maxTrackingTime));
} catch (IllegalArgumentException e) {
Log.logException(e);
return new ConcurrentLinkedQueue<Track>();
private void clearTooOldAccess(final ConcurrentLinkedQueue<Track> access) {
Long time = Long.valueOf(System.currentTimeMillis() - maxTrackingTime);
Iterator<Track> e = access.iterator();
Track l;
while (e.hasNext()) {
l = e.next();
if (l.getTime() <= time) e.remove();
}
}
@ -120,35 +126,34 @@ public class serverAccessTracker {
// learn that a specific host has accessed a specific path
if (accessPath == null) accessPath="NULL";
Collection<Track> track = accessTracker.get(host);
if (track == null) track = new ConcurrentLinkedQueue<Track>();
track.add(new Track(System.currentTimeMillis(), accessPath));
// write back to tracker
accessTracker.put(host, clearTooOldAccess(track));
ConcurrentLinkedQueue<Track> track = accessTracker.get(host);
if (track == null) {
track = new ConcurrentLinkedQueue<Track>();
track.add(new Track(System.currentTimeMillis(), accessPath));
// add to tracker
accessTracker.put(host, track);
} else {
track.add(new Track(System.currentTimeMillis(), accessPath));
clearTooOldAccess(track);
}
}
public Collection<Track> accessTrack(final String host) {
// returns mapping from Long(accesstime) to path
Collection<Track> access = accessTracker.get(host);
ConcurrentLinkedQueue<Track> access = accessTracker.get(host);
if (access == null) return null;
// clear too old entries
synchronized (access) {
if (access.size() != (access = clearTooOldAccess(access)).size()) {
// write back to tracker
if (access.isEmpty()) {
accessTracker.remove(host);
} else {
accessTracker.put(host, access);
}
}
clearTooOldAccess(access);
if (access.isEmpty()) {
accessTracker.remove(host);
}
return access;
}
public Iterator<String> accessHosts() {
// returns an iterator of hosts in tracker (String)
final Map<String, Collection<Track>> accessTrackerClone = new ConcurrentHashMap<String, Collection<Track>>();
final Map<String, ConcurrentLinkedQueue<Track>> accessTrackerClone = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>();
accessTrackerClone.putAll(accessTracker);
return accessTrackerClone.keySet().iterator();
}

4
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -544,6 +544,10 @@ public class serverSwitch {
public Collection<Track> accessTrack(String host) {
return this.accessTracker.accessTrack(host);
}
public int latestAccessCount(final String host, long timedelta) {
return this.accessTracker.latestAccessCount(host, timedelta);
}
public Iterator<String> accessHosts() {
return this.accessTracker.accessHosts();

Write Preview
Loading…
Cancel
Save