made the access tracker class static because it shall be used by the

jetty auth module
pull/1/head
Michael Peter Christen 11 years ago
parent 1c56befb93
commit 9bd71fdbb4

@ -31,6 +31,7 @@ import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.server.serverAccessTracker;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
import net.yacy.visualization.HexGridPlotter; import net.yacy.visualization.HexGridPlotter;
@ -115,11 +116,11 @@ public class AccessPicture_p {
String host; String host;
int c, h; int c, h;
for (final int time2 : times) { for (final int time2 : times) {
final Iterator<String> i = sb.accessHosts(); final Iterator<String> i = serverAccessTracker.accessHosts();
try { try {
while (i.hasNext()) { while (i.hasNext()) {
host = i.next(); host = i.next();
c = sb.latestAccessCount(host, time2); c = serverAccessTracker.latestAccessCount(host, time2);
if (c > 0) { if (c > 0) {
h = (Math.abs(host.hashCode())) % hosts.length; h = (Math.abs(host.hashCode())) % hosts.length;
hosts[h] = host; hosts[h] = host;

@ -42,6 +42,7 @@ import net.yacy.peers.Seed;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.query.AccessTracker; import net.yacy.search.query.AccessTracker;
import net.yacy.search.query.QueryParams; import net.yacy.search.query.QueryParams;
import net.yacy.server.serverAccessTracker;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
import net.yacy.server.serverAccessTracker.Track; import net.yacy.server.serverAccessTracker.Track;
@ -71,17 +72,17 @@ public class AccessTracker_p {
final int maxCount = 1000; final int maxCount = 1000;
boolean dark = true; boolean dark = true;
if (page == 0) { if (page == 0) {
final Iterator<String> i = sb.accessHosts(); final Iterator<String> i = serverAccessTracker.accessHosts();
String host; String host;
int entCount = 0; int entCount = 0;
try { try {
while ((entCount < maxCount) && (i.hasNext())) { while ((entCount < maxCount) && (i.hasNext())) {
host = i.next(); host = i.next();
prop.putHTML("page_list_" + entCount + "_host", host); prop.putHTML("page_list_" + entCount + "_host", host);
prop.putNum("page_list_" + entCount + "_countSecond", sb.latestAccessCount(host, 1000)); prop.putNum("page_list_" + entCount + "_countSecond", serverAccessTracker.latestAccessCount(host, 1000));
prop.putNum("page_list_" + entCount + "_countMinute", sb.latestAccessCount(host, 1000 * 60)); prop.putNum("page_list_" + entCount + "_countMinute", serverAccessTracker.latestAccessCount(host, 1000 * 60));
prop.putNum("page_list_" + entCount + "_count10Minutes", sb.latestAccessCount(host, 1000 * 60 * 10)); prop.putNum("page_list_" + entCount + "_count10Minutes", serverAccessTracker.latestAccessCount(host, 1000 * 60 * 10));
prop.putNum("page_list_" + entCount + "_countHour", sb.latestAccessCount(host, 1000 * 60 * 60)); prop.putNum("page_list_" + entCount + "_countHour", serverAccessTracker.latestAccessCount(host, 1000 * 60 * 60));
entCount++; entCount++;
} }
} catch (final ConcurrentModificationException e) { } catch (final ConcurrentModificationException e) {
@ -99,7 +100,7 @@ public class AccessTracker_p {
Collection<Track> access; Collection<Track> access;
Track entry; Track entry;
if (host.length() > 0) { if (host.length() > 0) {
access = sb.accessTrack(host); access = serverAccessTracker.accessTrack(host);
if (access != null) { if (access != null) {
try { try {
final Iterator<Track> ii = listclone(access).iterator(); final Iterator<Track> ii = listclone(access).iterator();
@ -117,10 +118,10 @@ public class AccessTracker_p {
} }
} else { } else {
try { try {
final Iterator<String> i = sb.accessHosts(); final Iterator<String> i = serverAccessTracker.accessHosts();
while ((entCount < maxCount) && (i.hasNext())) { while ((entCount < maxCount) && (i.hasNext())) {
host = i.next(); host = i.next();
access = sb.accessTrack(host); access = serverAccessTracker.accessTrack(host);
final Iterator<Track> ii = listclone(access).iterator(); final Iterator<Track> ii = listclone(access).iterator();
while (ii.hasNext()) { while (ii.hasNext()) {
entry = ii.next(); entry = ii.next();

@ -34,6 +34,7 @@ import net.yacy.cora.protocol.Domains;
import net.yacy.data.UserDB.AccessRight; import net.yacy.data.UserDB.AccessRight;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverAccessTracker;
import org.eclipse.jetty.http.HttpSchemes; import org.eclipse.jetty.http.HttpSchemes;
import org.eclipse.jetty.security.RoleInfo; import org.eclipse.jetty.security.RoleInfo;
@ -175,7 +176,7 @@ public class Jetty8YaCySecurityHandler extends SecurityHandler {
String refererHost; String refererHost;
// update AccessTracker // update AccessTracker
refererHost = request.getRemoteAddr(); refererHost = request.getRemoteAddr();
sb.track(refererHost, pathInContext); serverAccessTracker.track(refererHost, pathInContext);
try { try {
refererHost = new MultiProtocolURL(request.getHeader("Referer")).getHost(); refererHost = new MultiProtocolURL(request.getHeader("Referer")).getHost();

@ -27,7 +27,6 @@ import org.apache.solr.common.params.MultiMapSolrParams;
import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.CommonPattern;
import net.yacy.search.index.Segment;
import net.yacy.search.schema.CollectionSchema; import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;

@ -28,15 +28,17 @@ import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import net.yacy.cora.protocol.Domains;
public class serverAccessTracker { public class serverAccessTracker {
private static final long cleanupCycle = 60000; // 1 minute private static final long cleanupCycle = 60000; // 1 minute
private static long maxTrackingTime = 3600000;
private final long maxTrackingTime; private static int maxTrackingCount = 1000;
private final int maxTrackingCount; private static int maxHostCount = 100;
private final int maxHostCount; private static final ConcurrentHashMap<String, Queue<Track>> accessTracker = new ConcurrentHashMap<String, Queue<Track>>(); // mappings from requesting host to an ArrayList of serverTrack-entries
private final ConcurrentHashMap<String, Queue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries private static long lastCleanup;
private long lastCleanup; private static long lastLocalhostAccess = 0;
public static class Track { public static class Track {
private final long time; private final long time;
@ -53,25 +55,22 @@ public class serverAccessTracker {
} }
} }
public serverAccessTracker(final long maxTrackingTime, final int maxTrackingCount, final int maxTrackingHostCount) { public static void init(final long mtt, final int mtc, final int mthc) {
this.maxTrackingTime = maxTrackingTime; maxTrackingTime = mtt;
this.maxTrackingCount = maxTrackingCount; maxTrackingCount = mtc;
this.maxHostCount = maxTrackingHostCount; maxHostCount = mthc;
this.accessTracker = new ConcurrentHashMap<String, Queue<Track>>();
} }
/* /*
* remove all entries from the access tracker where the age of the last access is greater than the given timeout * remove all entries from the access tracker where the age of the last access is greater than the given timeout
*/ */
private void cleanupAccessTracker() { private static void cleanupAccessTracker() {
synchronized (this) { if (System.currentTimeMillis() - lastCleanup < cleanupCycle) return; // avoid too many scans of the queues
if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; // avoid too many scans of the queues lastCleanup = System.currentTimeMillis();
this.lastCleanup = System.currentTimeMillis();
}
// clear entries which had no entry for the maxTrackingTime time // clear entries which had no entry for the maxTrackingTime time
final Iterator<Map.Entry<String, Queue<Track>>> i = this.accessTracker.entrySet().iterator(); final Iterator<Map.Entry<String, Queue<Track>>> i = accessTracker.entrySet().iterator();
Queue<Track> track; Queue<Track> track;
while (i.hasNext()) { while (i.hasNext()) {
track = i.next().getValue(); track = i.next().getValue();
@ -81,7 +80,7 @@ public class serverAccessTracker {
i.remove(); i.remove();
} else { } else {
// check if the maxTrackingCount is exceeded // check if the maxTrackingCount is exceeded
while (track.size() > this.maxTrackingCount) try { while (track.size() > maxTrackingCount) try {
// delete the oldest entries // delete the oldest entries
track.remove(); track.remove();
} catch (final NoSuchElementException e) { break; } // concurrency may cause that the track is already empty } catch (final NoSuchElementException e) { break; } // concurrency may cause that the track is already empty
@ -89,11 +88,11 @@ public class serverAccessTracker {
} }
// if there are more entries left than maxTrackingCount, delete some. // if there are more entries left than maxTrackingCount, delete some.
while (this.accessTracker.size() > this.maxHostCount) { while (accessTracker.size() > maxHostCount) {
// delete just any // delete just any
final String key = this.accessTracker.keys().nextElement(); final String key = accessTracker.keys().nextElement();
if (key == null) break; // may occur because of concurrency effects if (key == null) break; // may occur because of concurrency effects
this.accessTracker.remove(key); accessTracker.remove(key);
} }
} }
@ -103,7 +102,7 @@ public class serverAccessTracker {
* @param delta the time delta from now to the past where the access times shall be computed * @param delta the time delta from now to the past where the access times shall be computed
* @return the number of accesses to the host in the given time span * @return the number of accesses to the host in the given time span
*/ */
public int latestAccessCount(final String host, final long delta) { public static int latestAccessCount(final String host, final long delta) {
final Collection<Track> timeList = accessTrack(host); final Collection<Track> timeList = accessTrack(host);
if (timeList == null) return 0; if (timeList == null) return 0;
final long time = System.currentTimeMillis() - delta; final long time = System.currentTimeMillis() - delta;
@ -112,8 +111,8 @@ public class serverAccessTracker {
return c; return c;
} }
private void clearTooOldAccess(final Queue<Track> access) { private static void clearTooOldAccess(final Queue<Track> access) {
final long time = System.currentTimeMillis() - this.maxTrackingTime; final long time = System.currentTimeMillis() - maxTrackingTime;
final Iterator<Track> e = access.iterator(); final Iterator<Track> e = access.iterator();
Track l; Track l;
int max = access.size(); // ensure termination int max = access.size(); // ensure termination
@ -123,43 +122,48 @@ public class serverAccessTracker {
} }
} }
public void track(final String host, String accessPath) { public static void track(final String host, String accessPath) {
// check storage size // check storage size
if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) { if (System.currentTimeMillis() - lastCleanup > cleanupCycle) {
cleanupAccessTracker(); cleanupAccessTracker();
} }
// learn that a specific host has accessed a specific path // learn that a specific host has accessed a specific path
if (accessPath == null) accessPath="NULL"; if (accessPath == null) accessPath="NULL";
Queue<Track> track = this.accessTracker.get(host); Queue<Track> track = accessTracker.get(host);
if (track == null) { if (track == null) {
track = new LinkedBlockingQueue<Track>(); track = new LinkedBlockingQueue<Track>();
track.add(new Track(System.currentTimeMillis(), accessPath)); track.add(new Track(System.currentTimeMillis(), accessPath));
// add to tracker // add to tracker
this.accessTracker.put(host, track); accessTracker.put(host, track);
} else { } else {
track.add(new Track(System.currentTimeMillis(), accessPath)); track.add(new Track(System.currentTimeMillis(), accessPath));
clearTooOldAccess(track); clearTooOldAccess(track);
} }
if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis();
} }
public Collection<Track> accessTrack(final String host) { public static Collection<Track> accessTrack(final String host) {
// returns mapping from Long(accesstime) to path // returns mapping from Long(accesstime) to path
final Queue<Track> access = this.accessTracker.get(host); final Queue<Track> access = accessTracker.get(host);
if (access == null) return null; if (access == null) return null;
// clear too old entries // clear too old entries
clearTooOldAccess(access); clearTooOldAccess(access);
if (access.isEmpty()) { if (access.isEmpty()) {
this.accessTracker.remove(host); accessTracker.remove(host);
} }
return access; return access;
} }
public Iterator<String> accessHosts() { public static Iterator<String> accessHosts() {
// returns an iterator of hosts in tracker (String) // returns an iterator of hosts in tracker (String)
final Map<String, Queue<Track>> accessTrackerClone = new ConcurrentHashMap<String, Queue<Track>>(); final Map<String, Queue<Track>> accessTrackerClone = new ConcurrentHashMap<String, Queue<Track>>();
accessTrackerClone.putAll(this.accessTracker); accessTrackerClone.putAll(accessTracker);
return accessTrackerClone.keySet().iterator(); return accessTrackerClone.keySet().iterator();
} }
public static long timeSinceAccessFromLocalhost() {
return System.currentTimeMillis() - lastLocalhostAccess;
}
} }

@ -31,7 +31,6 @@ import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.Reader; import java.io.Reader;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.NavigableMap; import java.util.NavigableMap;
@ -51,7 +50,6 @@ import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.workflow.BusyThread; import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.WorkflowThread; import net.yacy.kelondro.workflow.WorkflowThread;
import net.yacy.server.serverAccessTracker.Track;
public class serverSwitch public class serverSwitch
{ {
@ -68,7 +66,6 @@ public class serverSwitch
private final ConcurrentMap<String, String> configRemoved; private final ConcurrentMap<String, String> configRemoved;
private final ConcurrentMap<InetAddress, String> authorization; private final ConcurrentMap<InetAddress, String> authorization;
private final NavigableMap<String, BusyThread> workerThreads; private final NavigableMap<String, BusyThread> workerThreads;
private final serverAccessTracker accessTracker;
private YaCyHttpServer httpserver; // implemented HttpServer private YaCyHttpServer httpserver; // implemented HttpServer
public serverSwitch( public serverSwitch(
@ -137,8 +134,7 @@ public class serverSwitch
this.serverJobs = 0; this.serverJobs = 0;
// init server tracking // init server tracking
this.accessTracker = serverAccessTracker.init(
new serverAccessTracker(
getConfigLong("server.maxTrackingTime", 60 * 60 * 1000), getConfigLong("server.maxTrackingTime", 60 * 60 * 1000),
(int) getConfigLong("server.maxTrackingCount", 1000), (int) getConfigLong("server.maxTrackingCount", 1000),
(int) getConfigLong("server.maxTrackingHostCount", 100)); (int) getConfigLong("server.maxTrackingHostCount", 100));
@ -520,22 +516,6 @@ public class serverSwitch
this.serverJobs = jobs; this.serverJobs = jobs;
} }
public void track(final String host, final String accessPath) {
this.accessTracker.track(host, accessPath);
}
public Collection<Track> accessTrack(final String host) {
return this.accessTracker.accessTrack(host);
}
public int latestAccessCount(final String host, final long timedelta) {
return this.accessTracker.latestAccessCount(host, timedelta);
}
public Iterator<String> accessHosts() {
return this.accessTracker.accessHosts();
}
/** /**
* Retrieve text data (e. g. config file) from file file may be an url or a filename with path relative to * Retrieve text data (e. g. config file) from file file may be an url or a filename with path relative to
* rootPath parameter * rootPath parameter

Loading…
Cancel
Save