diff --git a/api/src/org/labkey/api/security/AuthenticationManager.java b/api/src/org/labkey/api/security/AuthenticationManager.java index b7964ad4c98..d8054bb6cc5 100644 --- a/api/src/org/labkey/api/security/AuthenticationManager.java +++ b/api/src/org/labkey/api/security/AuthenticationManager.java @@ -1273,10 +1273,10 @@ private static PrimaryAuthenticationResult _beforeAuthenticate(HttpServletReques // slow down login attempts when we detect more than 20/minute bad attempts per user, password, or ip address rl = addrLimiter.get(getIntCacheKey(request == null ? null : request.getRemoteAddr())); if (null != rl) - delay = Math.max(delay,rl.add(0, false)); + delay = Math.max(delay, rl.getDelay()); rl = pwdLimiter.get(getIntCacheKey(pwd)); if (null != rl) - delay = Math.max(delay, rl.add(0, false)); + delay = Math.max(delay, rl.getDelay()); try { @@ -1306,7 +1306,7 @@ private static long getDefaultUserLoginDelay(String id) { RateLimiter rl = userLimiter.get(getEmailCacheKey(id)); if (null != rl) - return rl.add(0, false); + return rl.getDelay(); return 0; } @@ -1318,9 +1318,9 @@ private static void _afterAuthenticate(HttpServletRequest request, String id, St { RateLimiter rl; rl = addrLimiter.get(getIntCacheKey(request.getRemoteAddr()),request, addrLoader); - rl.add(1, false); + rl.tryAdd(1); rl = pwdLimiter.get(getIntCacheKey(pwd),request, pwdLoader); - rl.add(1, false); + rl.tryAdd(1); addUserLoginDelay(request, id); } @@ -1349,7 +1349,7 @@ private static void addUserLoginDelay(HttpServletRequest request, String id) private static void addDefaultUserLoginDelay(HttpServletRequest request, String id) { RateLimiter rl = userLimiter.get(getEmailCacheKey(id),request, userLoader); - rl.add(1, false); + rl.tryAdd(1); } // Attempts to authenticate using only LoginFormAuthenticationProviders (e.g., DbLogin, LDAP). This is for the case diff --git a/api/src/org/labkey/api/util/RateLimiter.java b/api/src/org/labkey/api/util/RateLimiter.java index e5f0247d753..90453c32390 100644 --- a/api/src/org/labkey/api/util/RateLimiter.java +++ b/api/src/org/labkey/api/util/RateLimiter.java @@ -22,12 +22,43 @@ import static java.lang.Math.min; -/** - * User: matthewb - * Date: Jan 14, 2010 - * Time: 10:01:10 AM - */ - +/// Enforces a maximum throughput over a sliding time window. +/// +/// Callers accumulate units of work (bytes, requests, operations) against a target [Rate]. +/// Internally, time is divided into sub-windows that are rotated as time passes, so the +/// enforced rate reflects recent activity rather than an all-time average. +/// +/// ## Throttling a background thread +/// +/// Use [#add(long)] to block until the rate budget allows. The return value is the +/// number of milliseconds spent waiting. +/// +/// ```java +/// var limiter = new RateLimiter("file io", 1_000_000, TimeUnit.SECONDS); // 1 MB/s +/// for (File f : files) { +/// limiter.add(f.length()); // blocks if over rate +/// index(f); +/// } +/// ``` +/// +/// ## Recording without blocking +/// +/// Use [#tryAdd(long)] to accumulate without pausing — for threads that should track +/// rate but never stall: +/// +/// ```java +/// limiter.tryAdd(1); // record the event, return current delay in ms +/// ``` +/// +/// ## Probing the current delay +/// +/// Use [#getDelay()] to check how far ahead of the target rate the limiter is, +/// without accumulating or blocking: +/// +/// ```java +/// if (limiter.getDelay() > THRESHOLD_MS) +/// return TOO_MANY_REQUESTS; +/// ``` public class RateLimiter { final String _name; @@ -45,7 +76,7 @@ public class RateLimiter SimpleRateAccumulator _short; // collection of 'short' intervals - ArrayList _history = new ArrayList<>(4); + ArrayList _history = new ArrayList<>(4); public RateLimiter(String name, Rate rate) { @@ -90,7 +121,7 @@ private SimpleRateAccumulator aggregateRate(long now) { long start = now; long count = 0; - for (RateAccumulator a : _history) + for (var a : _history) { if (a.getStart() + historyInterval < now) continue; @@ -114,16 +145,23 @@ public String toString() } - /* - * RateLimiter.add() is thread-safe - * returns how far (in ms) we are ahead of the target rate - */ + /** Accumulate {@code count} units and block until the rate budget allows. Returns ms spent waiting. */ + public synchronized long add(long count) + { + return _pause(_updateCounts(count)); + } + + /** Accumulate {@code count} units without blocking. Returns how far ahead of the target rate we are (ms). */ + public synchronized long tryAdd(long count) + { + return _updateCounts(count); + } + + /** @deprecated Use {@link #add(long)} or {@link #tryAdd(long)} */ + @Deprecated public synchronized long add(long count, boolean wait) { - long delay = _updateCounts(count); - if (!wait) - return delay; - return _pause(delay); + return wait ? add(count) : tryAdd(count); } @@ -151,7 +189,7 @@ private synchronized long _updateCounts(long count) { while (!_history.isEmpty()) { - RateAccumulator last = _history.getLast(); + var last = _history.getLast(); if (last.getStart() + accumulateInterval > now - historyInterval) break; _history.removeLast(); @@ -170,8 +208,6 @@ public static class TestCase extends Assert { private static final double DELTA = 1E-8; - long _end = 0; - @org.junit.Test public void test() { @@ -180,51 +216,76 @@ public void test() assertEquals("RateLimiter:test 1/MILLISECOND", l.toString()); assertEquals(1000.0, l.getTarget().getRate(TimeUnit.SECONDS), DELTA); - Runnable run = new Runnable() + long end = System.currentTimeMillis() + 5000; + Runnable run = () -> { - @Override - public void run() + while (System.currentTimeMillis() < end) { - while (System.currentTimeMillis() < _end) - { - l.add(1,true); - l.add(4,true); - l.add(2,true); - l.add(5,true); - } + l.add(1); + l.add(4); + l.add(2); + l.add(5); } }; Thread[] threads = new Thread[4]; for (int i=0 ; i<4 ; i++) threads[i] = new Thread(run); - - _end = System.currentTimeMillis() + 5000; for (int i=0 ; i<4 ; i++) threads[i].start(); for (int i=0 ; i<4 ; i++) - try {threads[i].join(20000);}catch(InterruptedException x){} + try {threads[i].join(20000);} catch (InterruptedException x) {} - // count should be about 1.0 - RateAccumulator counter = l._long; - double a = counter.getRate(_end); - assertTrue(a < 2.0); - assertTrue(a > 0.1); + // target is 1/ms; after ~5s count should be roughly 5000 + double rate = (double) l.getCount() / 5000.0; + assertTrue(rate < 2.0); + assertTrue(rate > 0.1); } @org.junit.Test public void test2() { - final RateLimiter l = new RateLimiter("test",new Rate(1,TimeUnit.SECONDS),10000,500); + final RateLimiter l = new RateLimiter("test", new Rate(1, TimeUnit.SECONDS), 10000, 500); long start = System.currentTimeMillis(); for (int i=0 ; i<10 ; i++) - { - l.add(1,true); - } - long finish = System.currentTimeMillis(); - long duration = finish-start; + l.add(1); + long duration = System.currentTimeMillis() - start; assertTrue(duration > 5000); assertTrue(duration < 15000); } + + @org.junit.Test + public void testTryAdd() + { + // history < 20s so useSystem=true; accum=500 for fast sub-window turnover + RateLimiter l = new RateLimiter("test", new Rate(10, TimeUnit.SECONDS), 5000, 500); + + // Under rate: 5 units against a 10/s target — the 1s rate floor means this reads as under-rate + assertEquals(0, l.tryAdd(5)); + + // Way over rate: must return positive delay without blocking + long start = System.currentTimeMillis(); + long delay = l.tryAdd(10000); + assertTrue("tryAdd must not block", System.currentTimeMillis() - start < 200); + assertTrue("should report delay when over rate", delay > 0); + } + + @org.junit.Test + public void testGetDelay() + { + RateLimiter l = new RateLimiter("test", new Rate(10, TimeUnit.SECONDS), 5000, 500); + assertEquals("fresh limiter has no delay", 0, l.getDelay()); + l.tryAdd(10000); + assertTrue("over-rate limiter should report positive delay", l.getDelay() > 0); + } + + @org.junit.Test + public void testGetCount() + { + RateLimiter l = new RateLimiter("test", new Rate(1, TimeUnit.SECONDS), 5000, 500); + l.tryAdd(7); + l.tryAdd(3); + assertEquals(10, l.getCount()); + } } } \ No newline at end of file diff --git a/search/src/org/labkey/search/model/DavCrawler.java b/search/src/org/labkey/search/model/DavCrawler.java index 72a2d868ab2..0ee9393b904 100644 --- a/search/src/org/labkey/search/model/DavCrawler.java +++ b/search/src/org/labkey/search/model/DavCrawler.java @@ -289,7 +289,10 @@ public void run() { boolean isCrawlerThread = Thread.currentThread() == _crawlerThread; - _listingRateLimiter.add(1, isCrawlerThread); + if (isCrawlerThread) + _listingRateLimiter.add(1); + else + _listingRateLimiter.tryAdd(1); _log.debug("IndexDirectoryJob.run({})", _path); @@ -409,7 +412,10 @@ public void setLastIndexed(long ms, long modified) { if (!f.isFile()) continue; - _fileIORateLimiter.add(f.length(), isCrawlerThread); + if (isCrawlerThread) + _fileIORateLimiter.add(f.length()); + else + _fileIORateLimiter.tryAdd(f.length()); } _task.getQueue(null, SearchService.PRIORITY.crawl).addResource(child);