From 9bd71fdbb4765bccfde4cc8d00b0104d0cc11906 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 5 Jan 2014 05:04:28 +0100 Subject: [PATCH] made the access tracker class static because it shall be used by the jetty auth module --- htroot/AccessPicture_p.java | 5 +- htroot/AccessTracker_p.java | 17 ++--- .../yacy/http/Jetty8YaCySecurityHandler.java | 3 +- .../net/yacy/search/query/QueryModifier.java | 1 - .../net/yacy/server/serverAccessTracker.java | 70 ++++++++++--------- source/net/yacy/server/serverSwitch.java | 22 +----- 6 files changed, 52 insertions(+), 66 deletions(-) diff --git a/htroot/AccessPicture_p.java b/htroot/AccessPicture_p.java index 35a025622..48e0bc650 100644 --- a/htroot/AccessPicture_p.java +++ b/htroot/AccessPicture_p.java @@ -31,6 +31,7 @@ import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.search.Switchboard; +import net.yacy.server.serverAccessTracker; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.visualization.HexGridPlotter; @@ -115,11 +116,11 @@ public class AccessPicture_p { String host; int c, h; for (final int time2 : times) { - final Iterator i = sb.accessHosts(); + final Iterator i = serverAccessTracker.accessHosts(); try { while (i.hasNext()) { host = i.next(); - c = sb.latestAccessCount(host, time2); + c = serverAccessTracker.latestAccessCount(host, time2); if (c > 0) { h = (Math.abs(host.hashCode())) % hosts.length; hosts[h] = host; diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.java index 75eea3e82..aaa42141a 100644 --- a/htroot/AccessTracker_p.java +++ b/htroot/AccessTracker_p.java @@ -42,6 +42,7 @@ import net.yacy.peers.Seed; import net.yacy.search.Switchboard; import net.yacy.search.query.AccessTracker; import net.yacy.search.query.QueryParams; +import net.yacy.server.serverAccessTracker; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.server.serverAccessTracker.Track; @@ -71,17 +72,17 @@ public class AccessTracker_p { final int maxCount = 1000; boolean dark = true; if (page == 0) { - final Iterator i = sb.accessHosts(); + final Iterator i = serverAccessTracker.accessHosts(); String host; int entCount = 0; try { while ((entCount < maxCount) && (i.hasNext())) { host = i.next(); prop.putHTML("page_list_" + entCount + "_host", host); - prop.putNum("page_list_" + entCount + "_countSecond", sb.latestAccessCount(host, 1000)); - prop.putNum("page_list_" + entCount + "_countMinute", sb.latestAccessCount(host, 1000 * 60)); - prop.putNum("page_list_" + entCount + "_count10Minutes", sb.latestAccessCount(host, 1000 * 60 * 10)); - prop.putNum("page_list_" + entCount + "_countHour", sb.latestAccessCount(host, 1000 * 60 * 60)); + prop.putNum("page_list_" + entCount + "_countSecond", serverAccessTracker.latestAccessCount(host, 1000)); + prop.putNum("page_list_" + entCount + "_countMinute", serverAccessTracker.latestAccessCount(host, 1000 * 60)); + prop.putNum("page_list_" + entCount + "_count10Minutes", serverAccessTracker.latestAccessCount(host, 1000 * 60 * 10)); + prop.putNum("page_list_" + entCount + "_countHour", serverAccessTracker.latestAccessCount(host, 1000 * 60 * 60)); entCount++; } } catch (final ConcurrentModificationException e) { @@ -99,7 +100,7 @@ public class AccessTracker_p { Collection access; Track entry; if (host.length() > 0) { - access = sb.accessTrack(host); + access = serverAccessTracker.accessTrack(host); if (access != null) { try { final Iterator ii = listclone(access).iterator(); @@ -117,10 +118,10 @@ public class AccessTracker_p { } } else { try { - final Iterator i = sb.accessHosts(); + final Iterator i = serverAccessTracker.accessHosts(); while ((entCount < maxCount) && (i.hasNext())) { host = i.next(); - access = sb.accessTrack(host); + access = serverAccessTracker.accessTrack(host); final Iterator ii = listclone(access).iterator(); while (ii.hasNext()) { entry = ii.next(); diff --git a/source/net/yacy/http/Jetty8YaCySecurityHandler.java b/source/net/yacy/http/Jetty8YaCySecurityHandler.java index eceaeffb6..78e93794a 100644 --- a/source/net/yacy/http/Jetty8YaCySecurityHandler.java +++ b/source/net/yacy/http/Jetty8YaCySecurityHandler.java @@ -34,6 +34,7 @@ import net.yacy.cora.protocol.Domains; import net.yacy.data.UserDB.AccessRight; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; +import net.yacy.server.serverAccessTracker; import org.eclipse.jetty.http.HttpSchemes; import org.eclipse.jetty.security.RoleInfo; @@ -175,7 +176,7 @@ public class Jetty8YaCySecurityHandler extends SecurityHandler { String refererHost; // update AccessTracker refererHost = request.getRemoteAddr(); - sb.track(refererHost, pathInContext); + serverAccessTracker.track(refererHost, pathInContext); try { refererHost = new MultiProtocolURL(request.getHeader("Referer")).getHost(); diff --git a/source/net/yacy/search/query/QueryModifier.java b/source/net/yacy/search/query/QueryModifier.java index d17871348..6255ff7af 100644 --- a/source/net/yacy/search/query/QueryModifier.java +++ b/source/net/yacy/search/query/QueryModifier.java @@ -27,7 +27,6 @@ import org.apache.solr.common.params.MultiMapSolrParams; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.CommonPattern; -import net.yacy.search.index.Segment; import net.yacy.search.schema.CollectionSchema; import net.yacy.server.serverObjects; diff --git a/source/net/yacy/server/serverAccessTracker.java b/source/net/yacy/server/serverAccessTracker.java index 2447fb952..d4f13817d 100644 --- a/source/net/yacy/server/serverAccessTracker.java +++ b/source/net/yacy/server/serverAccessTracker.java @@ -28,15 +28,17 @@ import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; +import net.yacy.cora.protocol.Domains; + public class serverAccessTracker { private static final long cleanupCycle = 60000; // 1 minute - - private final long maxTrackingTime; - private final int maxTrackingCount; - private final int maxHostCount; - private final ConcurrentHashMap> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries - private long lastCleanup; + private static long maxTrackingTime = 3600000; + private static int maxTrackingCount = 1000; + private static int maxHostCount = 100; + private static final ConcurrentHashMap> accessTracker = new ConcurrentHashMap>(); // mappings from requesting host to an ArrayList of serverTrack-entries + private static long lastCleanup; + private static long lastLocalhostAccess = 0; public static class Track { private final long time; @@ -53,25 +55,22 @@ public class serverAccessTracker { } } - public serverAccessTracker(final long maxTrackingTime, final int maxTrackingCount, final int maxTrackingHostCount) { - this.maxTrackingTime = maxTrackingTime; - this.maxTrackingCount = maxTrackingCount; - this.maxHostCount = maxTrackingHostCount; - this.accessTracker = new ConcurrentHashMap>(); + public static void init(final long mtt, final int mtc, final int mthc) { + maxTrackingTime = mtt; + maxTrackingCount = mtc; + maxHostCount = mthc; } /* * remove all entries from the access tracker where the age of the last access is greater than the given timeout */ - private void cleanupAccessTracker() { + private static void cleanupAccessTracker() { - synchronized (this) { - if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; // avoid too many scans of the queues - this.lastCleanup = System.currentTimeMillis(); - } + if (System.currentTimeMillis() - lastCleanup < cleanupCycle) return; // avoid too many scans of the queues + lastCleanup = System.currentTimeMillis(); // clear entries which had no entry for the maxTrackingTime time - final Iterator>> i = this.accessTracker.entrySet().iterator(); + final Iterator>> i = accessTracker.entrySet().iterator(); Queue track; while (i.hasNext()) { track = i.next().getValue(); @@ -81,7 +80,7 @@ public class serverAccessTracker { i.remove(); } else { // check if the maxTrackingCount is exceeded - while (track.size() > this.maxTrackingCount) try { + while (track.size() > maxTrackingCount) try { // delete the oldest entries track.remove(); } catch (final NoSuchElementException e) { break; } // concurrency may cause that the track is already empty @@ -89,11 +88,11 @@ public class serverAccessTracker { } // if there are more entries left than maxTrackingCount, delete some. - while (this.accessTracker.size() > this.maxHostCount) { + while (accessTracker.size() > maxHostCount) { // delete just any - final String key = this.accessTracker.keys().nextElement(); + final String key = accessTracker.keys().nextElement(); if (key == null) break; // may occur because of concurrency effects - this.accessTracker.remove(key); + accessTracker.remove(key); } } @@ -103,7 +102,7 @@ public class serverAccessTracker { * @param delta the time delta from now to the past where the access times shall be computed * @return the number of accesses to the host in the given time span */ - public int latestAccessCount(final String host, final long delta) { + public static int latestAccessCount(final String host, final long delta) { final Collection timeList = accessTrack(host); if (timeList == null) return 0; final long time = System.currentTimeMillis() - delta; @@ -112,8 +111,8 @@ public class serverAccessTracker { return c; } - private void clearTooOldAccess(final Queue access) { - final long time = System.currentTimeMillis() - this.maxTrackingTime; + private static void clearTooOldAccess(final Queue access) { + final long time = System.currentTimeMillis() - maxTrackingTime; final Iterator e = access.iterator(); Track l; int max = access.size(); // ensure termination @@ -123,43 +122,48 @@ public class serverAccessTracker { } } - public void track(final String host, String accessPath) { + public static void track(final String host, String accessPath) { // check storage size - if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) { + if (System.currentTimeMillis() - lastCleanup > cleanupCycle) { cleanupAccessTracker(); } // learn that a specific host has accessed a specific path if (accessPath == null) accessPath="NULL"; - Queue track = this.accessTracker.get(host); + Queue track = accessTracker.get(host); if (track == null) { track = new LinkedBlockingQueue(); track.add(new Track(System.currentTimeMillis(), accessPath)); // add to tracker - this.accessTracker.put(host, track); + accessTracker.put(host, track); } else { track.add(new Track(System.currentTimeMillis(), accessPath)); clearTooOldAccess(track); } + if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); } - public Collection accessTrack(final String host) { + public static Collection accessTrack(final String host) { // returns mapping from Long(accesstime) to path - final Queue access = this.accessTracker.get(host); + final Queue access = accessTracker.get(host); if (access == null) return null; // clear too old entries clearTooOldAccess(access); if (access.isEmpty()) { - this.accessTracker.remove(host); + accessTracker.remove(host); } return access; } - public Iterator accessHosts() { + public static Iterator accessHosts() { // returns an iterator of hosts in tracker (String) final Map> accessTrackerClone = new ConcurrentHashMap>(); - accessTrackerClone.putAll(this.accessTracker); + accessTrackerClone.putAll(accessTracker); return accessTrackerClone.keySet().iterator(); } + + public static long timeSinceAccessFromLocalhost() { + return System.currentTimeMillis() - lastLocalhostAccess; + } } diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index 6028bafb8..1f4208e69 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.net.InetAddress; -import java.util.Collection; import java.util.Iterator; import java.util.Map; import java.util.NavigableMap; @@ -51,7 +50,6 @@ import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.workflow.BusyThread; import net.yacy.kelondro.workflow.WorkflowThread; -import net.yacy.server.serverAccessTracker.Track; public class serverSwitch { @@ -68,7 +66,6 @@ public class serverSwitch private final ConcurrentMap configRemoved; private final ConcurrentMap authorization; private final NavigableMap workerThreads; - private final serverAccessTracker accessTracker; private YaCyHttpServer httpserver; // implemented HttpServer public serverSwitch( @@ -137,8 +134,7 @@ public class serverSwitch this.serverJobs = 0; // init server tracking - this.accessTracker = - new serverAccessTracker( + serverAccessTracker.init( getConfigLong("server.maxTrackingTime", 60 * 60 * 1000), (int) getConfigLong("server.maxTrackingCount", 1000), (int) getConfigLong("server.maxTrackingHostCount", 100)); @@ -520,22 +516,6 @@ public class serverSwitch this.serverJobs = jobs; } - public void track(final String host, final String accessPath) { - this.accessTracker.track(host, accessPath); - } - - public Collection accessTrack(final String host) { - return this.accessTracker.accessTrack(host); - } - - public int latestAccessCount(final String host, final long timedelta) { - return this.accessTracker.latestAccessCount(host, timedelta); - } - - public Iterator accessHosts() { - return this.accessTracker.accessHosts(); - } - /** * Retrieve text data (e. g. config file) from file file may be an url or a filename with path relative to * rootPath parameter