From c0ab2075325f54b9b238c13dc4b8ee53c5694d6a Mon Sep 17 00:00:00 2001 From: ZePedroResende Date: Sat, 25 Jul 2026 22:09:41 +0100 Subject: [PATCH 1/4] fix: cache api-key auth lookups on the proxy hot path Every proxied RPC request runs a DB query in ApiKeyAuth to validate the token before forwarding. Under a burst to a single stack this turns N concurrent requests into N concurrent DB checkouts, saturating the pool and producing DBConnection checkout timeouts (500s). Cache successful token lookups in an ETS table (60s TTL) so the hot path skips the DB. Measured: a 200-request burst to one token drops from 400 DB queries to 0. Co-Authored-By: Claude Opus 4.8 (1M context) --- server/lib/ethui/application.ex | 4 +++ server/lib/ethui_web/plugs/api_key_auth.ex | 28 ++++++++++++++++++- .../ethui_web/plugs/api_key_auth_test.exs | 25 +++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/server/lib/ethui/application.ex b/server/lib/ethui/application.ex index 8a172eb..cee99bf 100644 --- a/server/lib/ethui/application.ex +++ b/server/lib/ethui/application.ex @@ -7,6 +7,10 @@ defmodule Ethui.Application do @impl true def start(_type, _args) do + # Hot-path cache for api-key auth on proxied requests (see EthuiWeb.Plugs.ApiKeyAuth). + # Owned by the app master process, lives for the app lifetime. + :ets.new(:api_key_cache, [:named_table, :public, :set, read_concurrency: true]) + children = [ EthuiWeb.Telemetry, Ethui.Repo, diff --git a/server/lib/ethui_web/plugs/api_key_auth.ex b/server/lib/ethui_web/plugs/api_key_auth.ex index 8e340a2..9f58688 100644 --- a/server/lib/ethui_web/plugs/api_key_auth.ex +++ b/server/lib/ethui_web/plugs/api_key_auth.ex @@ -15,6 +15,10 @@ defmodule EthuiWeb.Plugs.ApiKeyAuth do @min_token_length 20 + # ponytail: 60s TTL cache; a revoked/deleted key keeps working up to 60s. + # Drop TTL or invalidate on api_key delete if that window matters. + @cache_ttl_ms :timer.seconds(60) + def init(opts), do: opts def call(conn, _opts) do @@ -29,7 +33,7 @@ defmodule EthuiWeb.Plugs.ApiKeyAuth do conn.path_info with [token | _] when byte_size(token) >= @min_token_length <- conn.path_info, - %ApiKey{} = api_key <- Accounts.get_api_key_by_token(token), + %ApiKey{} = api_key <- cached_api_key(token), true <- stack_matches?(conn, api_key) do conn |> Map.update!(:path_info, &tl/1) else @@ -41,6 +45,28 @@ defmodule EthuiWeb.Plugs.ApiKeyAuth do end end + # Caches successful token lookups in ETS so the proxy hot path doesn't hit the + # DB on every forwarded RPC request. Invalid tokens are not cached (they stay + # cheap-to-reject and can't fill the table). + defp cached_api_key(token) do + now = System.monotonic_time(:millisecond) + + case :ets.lookup(:api_key_cache, token) do + [{^token, %ApiKey{} = api_key, expiry}] when expiry > now -> + api_key + + _ -> + case Accounts.get_api_key_by_token(token) do + %ApiKey{} = api_key -> + :ets.insert(:api_key_cache, {token, api_key, now + @cache_ttl_ms}) + api_key + + other -> + other + end + end + end + defp stack_matches?(conn, api_key) do case conn.assigns[:proxy][:slug] do nil -> false diff --git a/server/test/ethui_web/plugs/api_key_auth_test.exs b/server/test/ethui_web/plugs/api_key_auth_test.exs index e4c09a9..6edc940 100644 --- a/server/test/ethui_web/plugs/api_key_auth_test.exs +++ b/server/test/ethui_web/plugs/api_key_auth_test.exs @@ -2,10 +2,12 @@ defmodule EthuiWeb.Plugs.ApiKeyAuthTest do use EthuiWeb.ConnCase, async: false import Plug.Test + import Ecto.Query, only: [from: 2] alias EthuiWeb.Plugs.{ApiKeyAuth, Authenticate, StackSubdomain} alias Ethui.Repo alias Ethui.Accounts + alias Ethui.Accounts.ApiKey alias Ethui.Stacks.Stack describe "Api key auth plug when enabled" do @@ -49,6 +51,29 @@ defmodule EthuiWeb.Plugs.ApiKeyAuthTest do assert conn.path_info == ["execute"] refute conn.halted end + + test "caches successful lookups so the proxy hot path skips the DB", %{ + slug: slug, + api_key: api_key + } do + call = fn -> + conn(:get, "/#{api_key}/execute") + |> Map.put(:host, "#{slug}.lvh.me") + |> StackSubdomain.call(StackSubdomain.init([])) + |> ApiKeyAuth.call(ApiKeyAuth.init([])) + end + + # first call populates the cache + refute call.().halted + + # deleting the DB row must NOT break auth while the cache entry is warm + Repo.delete_all(from(k in ApiKey, where: k.token == ^api_key)) + refute call.().halted + + # once the cache entry is dropped, the now-missing key is rejected + :ets.delete(:api_key_cache, api_key) + assert call.().halted + end end describe "Api key auth plug when disabled " do From 7b4f2c4eadb1e9b3622b794f50e507922c4da11b Mon Sep 17 00:00:00 2001 From: ZePedroResende Date: Sat, 25 Jul 2026 22:24:27 +0100 Subject: [PATCH 2/4] review fixes: guard ETS boot, drop dead line, cover TTL + invalid-token - application.ex: guard :ets.new against re-entrant start/2 (ArgumentError) - api_key_auth.ex: remove dead conn.path_info expression - tests: expired entry is ignored+re-checked; invalid tokens aren't cached Co-Authored-By: Claude Opus 4.8 (1M context) --- server/lib/ethui/application.ex | 5 ++- server/lib/ethui_web/plugs/api_key_auth.ex | 2 -- .../ethui_web/plugs/api_key_auth_test.exs | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/server/lib/ethui/application.ex b/server/lib/ethui/application.ex index cee99bf..aa67f07 100644 --- a/server/lib/ethui/application.ex +++ b/server/lib/ethui/application.ex @@ -9,7 +9,10 @@ defmodule Ethui.Application do def start(_type, _args) do # Hot-path cache for api-key auth on proxied requests (see EthuiWeb.Plugs.ApiKeyAuth). # Owned by the app master process, lives for the app lifetime. - :ets.new(:api_key_cache, [:named_table, :public, :set, read_concurrency: true]) + # Guarded so a re-entrant start/2 in the same VM doesn't raise on an existing table. + if :ets.whereis(:api_key_cache) == :undefined do + :ets.new(:api_key_cache, [:named_table, :public, :set, read_concurrency: true]) + end children = [ EthuiWeb.Telemetry, diff --git a/server/lib/ethui_web/plugs/api_key_auth.ex b/server/lib/ethui_web/plugs/api_key_auth.ex index 9f58688..e230d55 100644 --- a/server/lib/ethui_web/plugs/api_key_auth.ex +++ b/server/lib/ethui_web/plugs/api_key_auth.ex @@ -30,8 +30,6 @@ defmodule EthuiWeb.Plugs.ApiKeyAuth do end defp do_call(conn) do - conn.path_info - with [token | _] when byte_size(token) >= @min_token_length <- conn.path_info, %ApiKey{} = api_key <- cached_api_key(token), true <- stack_matches?(conn, api_key) do diff --git a/server/test/ethui_web/plugs/api_key_auth_test.exs b/server/test/ethui_web/plugs/api_key_auth_test.exs index 6edc940..aedc5bd 100644 --- a/server/test/ethui_web/plugs/api_key_auth_test.exs +++ b/server/test/ethui_web/plugs/api_key_auth_test.exs @@ -74,6 +74,38 @@ defmodule EthuiWeb.Plugs.ApiKeyAuthTest do :ets.delete(:api_key_cache, api_key) assert call.().halted end + + test "expired cache entries are ignored and re-checked against the DB", %{ + slug: slug, + api_key: api_key + } do + call = fn -> + conn(:get, "/#{api_key}/execute") + |> Map.put(:host, "#{slug}.lvh.me") + |> StackSubdomain.call(StackSubdomain.init([])) + |> ApiKeyAuth.call(ApiKeyAuth.init([])) + end + + struct = Ethui.Accounts.get_api_key_by_token(api_key) + Repo.delete_all(from(k in ApiKey, where: k.token == ^api_key)) + + # a warm (non-expired) entry would still authorize; an expired one must not + :ets.insert(:api_key_cache, {api_key, struct, System.monotonic_time(:millisecond) - 1}) + assert call.().halted + end + + test "invalid tokens are not cached", %{slug: slug} do + bad = String.duplicate("z", 30) + + conn = + conn(:get, "/#{bad}/execute") + |> Map.put(:host, "#{slug}.lvh.me") + |> StackSubdomain.call(StackSubdomain.init([])) + |> ApiKeyAuth.call(ApiKeyAuth.init([])) + + assert conn.halted + assert :ets.lookup(:api_key_cache, bad) == [] + end end describe "Api key auth plug when disabled " do From 1171ae8950b330ecdca12a70f24baae6ce01c35d Mon Sep 17 00:00:00 2001 From: ZePedroResende Date: Sat, 25 Jul 2026 22:32:53 +0100 Subject: [PATCH 3/4] fix dialyzer: bind unmatched :ets.new return in application start Co-Authored-By: Claude Opus 4.8 (1M context) --- server/lib/ethui/application.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/lib/ethui/application.ex b/server/lib/ethui/application.ex index aa67f07..e181fc6 100644 --- a/server/lib/ethui/application.ex +++ b/server/lib/ethui/application.ex @@ -10,9 +10,10 @@ defmodule Ethui.Application do # Hot-path cache for api-key auth on proxied requests (see EthuiWeb.Plugs.ApiKeyAuth). # Owned by the app master process, lives for the app lifetime. # Guarded so a re-entrant start/2 in the same VM doesn't raise on an existing table. - if :ets.whereis(:api_key_cache) == :undefined do - :ets.new(:api_key_cache, [:named_table, :public, :set, read_concurrency: true]) - end + _ = + if :ets.whereis(:api_key_cache) == :undefined do + :ets.new(:api_key_cache, [:named_table, :public, :set, read_concurrency: true]) + end children = [ EthuiWeb.Telemetry, From 3d9e1e79b2ba5a43fa10227755864396eda14f97 Mon Sep 17 00:00:00 2001 From: ZePedroResende Date: Sat, 25 Jul 2026 22:34:52 +0100 Subject: [PATCH 4/4] trim verbose comments; drop leftover ponytail: tag Co-Authored-By: Claude Opus 4.8 (1M context) --- server/lib/ethui/application.ex | 5 ++--- server/lib/ethui_web/plugs/api_key_auth.ex | 8 +++----- server/test/ethui_web/plugs/api_key_auth_test.exs | 5 ++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/server/lib/ethui/application.ex b/server/lib/ethui/application.ex index e181fc6..68ab259 100644 --- a/server/lib/ethui/application.ex +++ b/server/lib/ethui/application.ex @@ -7,9 +7,8 @@ defmodule Ethui.Application do @impl true def start(_type, _args) do - # Hot-path cache for api-key auth on proxied requests (see EthuiWeb.Plugs.ApiKeyAuth). - # Owned by the app master process, lives for the app lifetime. - # Guarded so a re-entrant start/2 in the same VM doesn't raise on an existing table. + # ETS cache backing EthuiWeb.Plugs.ApiKeyAuth; guard avoids raising if the + # table already exists on a re-entrant start/2. _ = if :ets.whereis(:api_key_cache) == :undefined do :ets.new(:api_key_cache, [:named_table, :public, :set, read_concurrency: true]) diff --git a/server/lib/ethui_web/plugs/api_key_auth.ex b/server/lib/ethui_web/plugs/api_key_auth.ex index e230d55..4d93c39 100644 --- a/server/lib/ethui_web/plugs/api_key_auth.ex +++ b/server/lib/ethui_web/plugs/api_key_auth.ex @@ -15,8 +15,7 @@ defmodule EthuiWeb.Plugs.ApiKeyAuth do @min_token_length 20 - # ponytail: 60s TTL cache; a revoked/deleted key keeps working up to 60s. - # Drop TTL or invalidate on api_key delete if that window matters. + # Successful lookups are cached for this window; a revoked key keeps working until it lapses. @cache_ttl_ms :timer.seconds(60) def init(opts), do: opts @@ -43,9 +42,8 @@ defmodule EthuiWeb.Plugs.ApiKeyAuth do end end - # Caches successful token lookups in ETS so the proxy hot path doesn't hit the - # DB on every forwarded RPC request. Invalid tokens are not cached (they stay - # cheap-to-reject and can't fill the table). + # Cache successful token lookups so the proxy hot path skips the DB per request. + # Misses (invalid tokens) are not cached. defp cached_api_key(token) do now = System.monotonic_time(:millisecond) diff --git a/server/test/ethui_web/plugs/api_key_auth_test.exs b/server/test/ethui_web/plugs/api_key_auth_test.exs index aedc5bd..b2edb5a 100644 --- a/server/test/ethui_web/plugs/api_key_auth_test.exs +++ b/server/test/ethui_web/plugs/api_key_auth_test.exs @@ -63,14 +63,13 @@ defmodule EthuiWeb.Plugs.ApiKeyAuthTest do |> ApiKeyAuth.call(ApiKeyAuth.init([])) end - # first call populates the cache refute call.().halted - # deleting the DB row must NOT break auth while the cache entry is warm + # warm cache authorizes even after the DB row is gone Repo.delete_all(from(k in ApiKey, where: k.token == ^api_key)) refute call.().halted - # once the cache entry is dropped, the now-missing key is rejected + # evicted -> re-checked -> now missing -> rejected :ets.delete(:api_key_cache, api_key) assert call.().halted end