From 5b068a6999baab1c172176c4d18917e3c6c714cd Mon Sep 17 00:00:00 2001 From: Daniel Truong Date: Mon, 7 Sep 2026 17:44:45 +0000 Subject: [PATCH] fix(ingest): resolve visitor IP behind APIM APIM stamps X-Client-Ip and appends nothing to X-Forwarded-For, so the last hop is the one Azure's Functions front end added for APIM itself. Reading that hop located every event in Toronto and put the whole OpenShift cluster in one rate-cap bucket, which is what gave eagle-api 429 bursts on prod. The visitor is the hop before it, read only when X-Client-Ip is an address in TRUSTED_PROXY_IPS: APIM sets that header with override and every route here sits behind the gateway guard, so a caller cannot forge it. A caller with no hop of its own is a server producer sharing the cluster's egress address with every browser behind it, and is exempt from the cap rather than sharing its bucket. --- README.md | 12 +++ azure/main.bicep | 4 + azure/main.prod.bicepparam | 5 + azure/main.test.bicepparam | 5 + azure/modules/api-function-flex.bicep | 7 ++ docs/EVENT-SCHEMA.md | 4 +- src/config.js | 7 ++ src/controllers/events.js | 10 +- src/ingest/enrich-geo.js | 54 ++++++++--- test/config.test.js | 15 +++ test/enrich-geo.test.js | 131 ++++++++++++++++++++++---- test/ingest-routes.test.js | 115 ++++++++++++++++++++++ 12 files changed, 334 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index f2117e2..e67effb 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,17 @@ An event is `{timestamp, eventType, sessionId, sourceApp, userId?, properties?}` sent. Everything else is kept in `Detail`. The caller's address is used to look up country, region and city, and is then thrown away. +Which address that is takes some care, because APIM is in the way. A browser event travels +`browser -> OpenShift router -> nginx -> demi-apim- -> Functions front end -> here`. The router +appends the visitor to `X-Forwarded-For`, APIM adds nothing to that header but stamps the address it +saw in `X-Client-Ip`, and Azure then appends APIM's own outbound address as the last hop — so the last +hop is Azure's, not the visitor's. When `X-Client-Ip` is one of the addresses in `TRUSTED_PROXY_IPS` +the request came through our cluster and the visitor is the hop before the Azure one. With no such hop +the caller is a server-side producer like eagle-api: it comes out of the same cluster addresses as +every browser behind it, so it is exempt from the per-address cap, which cannot tell them apart. +Everyone else is read as their own caller and capped normally. Nothing further left in +`X-Forwarded-For` is read, because a caller can write what it likes there. + A `POST /query` body names a measure, a bin, a range and optional dimensions and filters. Every one is a key into `src/query/schema.js`; no column, table or operator name can come out of a request body, and the time range never reaches the query text at all. @@ -104,6 +115,7 @@ instead of serving an open or a permanently-401 API. | `NODE_ENV` | empty | Set to `production` by the Function App, which picks the JSON log format over the readable one | | `SESSION_EVENT_CAP` | `2000` | Events one session may contribute per hour per instance; the rest are dropped | | `STORAGE_ACCOUNT_NAME` | empty | Holds the GeoLite2 database and the saved dashboards. Empty means no location lookup | +| `TRUSTED_PROXY_IPS` | empty | Comma list of the addresses our own proxies call out from, as APIM reports them in `X-Client-Ip`. Decides which requests are read one `X-Forwarded-For` hop further back for the visitor, and which producers are exempt from the per-address cap. Empty trusts nothing | `ALLOWED_SOURCE_APPS`, `SESSION_EVENT_CAP` and `IP_EVENT_CAP` are readable but unset by the template: `src/config.js` owns those defaults, and a second copy in the Bicep would drift from it. diff --git a/azure/main.bicep b/azure/main.bicep index 58b30f7..a3b337e 100644 --- a/azure/main.bicep +++ b/azure/main.bicep @@ -42,6 +42,9 @@ param keycloakAllowedClients string = '' @description('Front Door\'s own id, from `az afd profile show --query frontDoorId` on the eagle-edge profile. Only on a match is X-Azure-SocketIP trusted over the caller-supplied X-Forwarded-For.') param frontDoorId string = '' +@description('Comma list of the addresses our own proxies call out from, as APIM reports them in X-Client-Ip. Decides which requests are read one X-Forwarded-For hop further back for the visitor address, and which server producers are exempt from the per-address event cap. Empty trusts nothing.') +param trustedProxyIps string = '' + @description('Header name demi-apim- stamps on a forwarded request.') param apimSharedHeaderName string = 'X-Analytics-Gateway' @@ -177,6 +180,7 @@ module apiFunctionFlex './modules/api-function-flex.bicep' = { auditSharedHeaderValue: auditSharedHeaderValue allowedOrigins: allowedOrigins frontDoorId: frontDoorId + trustedProxyIps: trustedProxyIps } } diff --git a/azure/main.prod.bicepparam b/azure/main.prod.bicepparam index c92f9be..e80ab37 100644 --- a/azure/main.prod.bicepparam +++ b/azure/main.prod.bicepparam @@ -40,6 +40,11 @@ param keycloakAllowedClients = 'eagle-admin-console' // X-Azure-SocketIP is trusted, so a caller who knows it can choose its own client address. param frontDoorId = readEnvironmentVariable('FRONT_DOOR_ID') +// The OpenShift cluster's egress pool, measured 2026-09-07 from what APIM stamped in X-Client-Ip. The +// same four addresses in both environments. Unlike frontDoorId above these are safe in the open: APIM +// sets X-Client-Ip with `override`, so knowing them does not let a caller claim to be the cluster. +param trustedProxyIps = '142.34.194.121,142.34.194.122,142.34.194.123,142.34.194.124' + // Both the apex and the www host serve the public site; the AFD endpoint hostname is the origin the // site is still reachable on directly. No localhost here. param allowedOrigins = [ diff --git a/azure/main.test.bicepparam b/azure/main.test.bicepparam index eb36632..c01a4eb 100644 --- a/azure/main.test.bicepparam +++ b/azure/main.test.bicepparam @@ -33,6 +33,11 @@ param keycloakAllowedClients = 'eagle-admin-console' // X-Azure-SocketIP is trusted, so a caller who knows it can choose its own client address. param frontDoorId = readEnvironmentVariable('FRONT_DOOR_ID') +// The OpenShift cluster's egress pool, measured 2026-09-07 from what APIM stamped in X-Client-Ip. The +// same four addresses in both environments. Unlike frontDoorId above these are safe in the open: APIM +// sets X-Client-Ip with `override`, so knowing them does not let a caller claim to be the cluster. +param trustedProxyIps = '142.34.194.121,142.34.194.122,142.34.194.123,142.34.194.124' + // The two AFD hostnames carry a deploy-time hash and cannot be composed: eagle-public's is in // eagle-edge/README.md, eagle-demi-admin's in eagle-demi/azure/main.test.bicepparam. Third is // eagle-admin on OpenShift test. Both the apex and the www host serve the public site, same as prod. diff --git a/azure/modules/api-function-flex.bicep b/azure/modules/api-function-flex.bicep index a8249b0..26e6618 100644 --- a/azure/modules/api-function-flex.bicep +++ b/azure/modules/api-function-flex.bicep @@ -81,6 +81,9 @@ param allowedOrigins array = [] @description('Front Door\'s own id (`az afd profile show --query frontDoorId`). Only on an X-Azure-FDID match is X-Azure-SocketIP trusted over the caller-supplied X-Forwarded-For.') param frontDoorId string = '' +@description('Comma list of the addresses our own proxies call out from, as APIM reports them in X-Client-Ip (the OpenShift cluster egress pool). A request stamped with one of these is read one X-Forwarded-For hop further back for the visitor, and a server producer behind them is exempt from the per-address event cap. Empty trusts nothing.') +param trustedProxyIps string = '' + var apiAppName = 'analytics-api-fc-${environmentName}' var appServicePlanName = 'analytics-plan-fc-${environmentName}' var storageAccountName = take('analyticsfc${environmentName}${uniqueString(resourceGroup().id)}', 24) @@ -352,6 +355,10 @@ resource apiFunctionApp 'Microsoft.Web/sites@2023-12-01' = { name: 'FRONT_DOOR_ID' value: frontDoorId } + { + name: 'TRUSTED_PROXY_IPS' + value: trustedProxyIps + } // Picks the JSON log format in src/utils/logger.js. { name: 'NODE_ENV' diff --git a/docs/EVENT-SCHEMA.md b/docs/EVENT-SCHEMA.md index 6ac4469..b3075c2 100644 --- a/docs/EVENT-SCHEMA.md +++ b/docs/EVENT-SCHEMA.md @@ -18,7 +18,9 @@ comes back only when nothing at all was accepted, or when the envelope itself is buffered in the Function and posted to Log Analytics on a timer, so `202` means accepted, not stored. `POST /events` also answers `429` with `Retry-After: 60` once one client address has sent more than -`IP_EVENT_CAP` events in a minute. Audit rows are never refused for volume. +`IP_EVENT_CAP` events in a minute. Server-side producers are exempt: they reach the gateway from the +OpenShift cluster's egress addresses (`TRUSTED_PROXY_IPS`), which every browser behind the cluster +shares, so a per-address cap cannot separate them. Audit rows are never refused for volume. ## What the client sends diff --git a/src/config.js b/src/config.js index 8960358..69700c3 100644 --- a/src/config.js +++ b/src/config.js @@ -105,6 +105,13 @@ module.exports = { // ceiling is the Function's own (src/ingest/ip-cap.js). ipEventCap: intFromEnv('IP_EVENT_CAP', 600, 1), + // Addresses our own proxies call out from, as APIM reports them in X-Client-Ip: the OpenShift + // cluster's egress pool. A request stamped with one of these came through our own infrastructure, + // which is what lets src/ingest/enrich-geo.js look one hop further back in X-Forwarded-For for the + // visitor instead of geolocating and rate-capping the whole cluster as one caller. Empty trusts no + // proxy, so every caller is read as itself and capped. + trustedProxyIps: listFromEnv('TRUSTED_PROXY_IPS', ''), + // Read here rather than in src/utils/logger.js, so this file stays the only reader of process.env. get nodeEnv() { return process.env.NODE_ENV || ''; }, diff --git a/src/controllers/events.js b/src/controllers/events.js index 09458f8..10e9d3b 100644 --- a/src/controllers/events.js +++ b/src/controllers/events.js @@ -2,7 +2,7 @@ const { EVENTS_STREAM, enqueue } = require('../ingest/dcr-writer'); const { enrichDevice } = require('../ingest/enrich-device'); -const { clientIp, geoFields } = require('../ingest/enrich-geo'); +const { geoFields, resolveCaller } = require('../ingest/enrich-geo'); const ipCap = require('../ingest/ip-cap'); const { allow } = require('../ingest/session-cap'); const { toEventRow, validateEventBatch } = require('../ingest/validate'); @@ -21,13 +21,17 @@ const { toEventRow, validateEventBatch } = require('../ingest/validate'); */ async function events(req, res) { // One resolution of the address, used for the cap and then for the location lookup. - const ip = clientIp(req); + const { ip, trusted } = resolveCaller(req); // Before validation, deliberately: an address over its cap should not cost this instance the work // of parsing what it sent. 429 and not a silent drop, because a refused batch is the producer's to // retry. + // + // Our own server-side producers are exempt: they reach APIM from the cluster's egress pool, which + // every browser behind the same cluster also comes out of, so one address cap cannot separate them + // and capping it refused eagle-api's batches on prod. const offered = Array.isArray(req.body && req.body.events) ? req.body.events.length : 1; - if (!ipCap.allow(ip, offered)) { + if (!trusted && !ipCap.allow(ip, offered)) { res.set('Retry-After', String(ipCap.RETRY_AFTER_SECONDS)); res.status(429).json({ error: 'Too many events from this address. Retry in a minute.' }); return; diff --git a/src/ingest/enrich-geo.js b/src/ingest/enrich-geo.js index 63595a7..759781f 100644 --- a/src/ingest/enrich-geo.js +++ b/src/ingest/enrich-geo.js @@ -76,24 +76,55 @@ function isPrivateIp(value) { return IPV4_PRIVATE.some((range) => range.test(ip)); } +/** One of the addresses our own proxies call out from (config.trustedProxyIps). */ +function isTrustedProxy(ip) { + return Boolean(ip) && config.trustedProxyIps.includes(ip); +} + /** - * Who called, as far as it can be trusted. X-Azure-ClientIP is deliberately not read: Front Door - * derives it from the caller's own X-Forwarded-For, so the caller controls it. Same trust rule as - * eagle-api's rateLimitKey helper — the socket address only when the request really came through our - * Front Door profile. + * Who called, as far as it can be trusted. + * + * A browser event arrives through + * `browser -> OpenShift router -> rproxy -> demi-apim- -> Functions front end -> here`. The + * router appends the visitor to X-Forwarded-For; APIM appends nothing to it and instead stamps the + * address IT saw in X-Client-Ip, and the Functions front end then appends APIM's own outbound address. + * So the header reads `[…what the caller sent…, visitor, apim]` — hop -1 is Azure's, hop -2 is the + * visitor, and everything further left is caller-written and worth nothing. + * + * X-Client-Ip is only trustworthy because every route reaching here also carries `apimGuard` + * (src/auth/apim-header.js): APIM sets it with `override`, and a request that skipped the gateway is a + * 401 before a controller runs. X-Azure-ClientIP stays unread: Front Door derives it from the caller's + * own X-Forwarded-For. Same trust rule as eagle-api's rateLimitKey helper. + * + * @returns {{ip: string, trusted: boolean}} `trusted` marks one of our own server producers, which + * shares the cluster's egress address with every browser behind it and so cannot be capped by address. */ -function clientIp(req) { +function resolveCaller(req) { if (config.frontDoorId && safeEqual(req.header('x-azure-fdid'), config.frontDoorId)) { const socketIp = normalizeIp(req.header('x-azure-socketip')); - if (socketIp) return socketIp; + if (socketIp) return { ip: socketIp, trusted: false }; } - // The RIGHT-most hop, not the left-most: every entry to its left was written by whoever called us, - // and the last one is the address the proxy immediately in front of this app appended. - const forwarded = req.header('x-forwarded-for'); - if (forwarded) return normalizeIp(String(forwarded).split(',').at(-1)); + const hops = String(req.header('x-forwarded-for') || '').split(',').map(normalizeIp).filter(Boolean); + const gatewaySaw = normalizeIp(req.header('x-client-ip')); + + if (isTrustedProxy(gatewaySaw)) { + const visitor = hops.at(-2); + if (visitor && !isTrustedProxy(visitor)) return { ip: visitor, trusted: false }; - return ''; + // Nothing behind the proxy: an eagle-api pod or another server producer calling APIM itself. + return { ip: gatewaySaw, trusted: true }; + } + + // Somebody calling APIM straight off the internet. Their own address, and the cap applies. + if (gatewaySaw) return { ip: gatewaySaw, trusted: false }; + + return { ip: hops.at(-1) || '', trusted: false }; +} + +/** The resolved address on its own, for a caller that has no use for the trust flag. */ +function clientIp(req) { + return resolveCaller(req).ip; } /** The cached file if there is one, otherwise a fresh copy from the blob container. */ @@ -179,6 +210,7 @@ async function geoFields(ip) { module.exports = { clientIp, + resolveCaller, geoFields, isPrivateIp, // Test seam. One reader, one real implementation, so no abstraction over it. diff --git a/test/config.test.js b/test/config.test.js index 26c034f..f1046f8 100644 --- a/test/config.test.js +++ b/test/config.test.js @@ -65,3 +65,18 @@ test('ALLOWED_ORIGINS is read as a trimmed list', () => { test('an unset ALLOWED_ORIGINS admits no browser at all', () => { assert.deepStrictEqual(loadConfig({ ENVIRONMENT: 'test' }).allowedOrigins, []); }); + +test('TRUSTED_PROXY_IPS is read as a trimmed list', () => { + const config = loadConfig({ + ENVIRONMENT: 'test', + TRUSTED_PROXY_IPS: '142.34.194.121, 142.34.194.122' + }); + + assert.deepStrictEqual(config.trustedProxyIps, ['142.34.194.121', '142.34.194.122']); +}); + +// Nothing trusted is the safe default: every caller is located and capped by the last hop, which is +// what this service did before the cluster's egress addresses were known. +test('an unset TRUSTED_PROXY_IPS trusts no proxy', () => { + assert.deepStrictEqual(loadConfig({ ENVIRONMENT: 'test' }).trustedProxyIps, []); +}); diff --git a/test/enrich-geo.test.js b/test/enrich-geo.test.js index 323b2c8..4d54d48 100644 --- a/test/enrich-geo.test.js +++ b/test/enrich-geo.test.js @@ -1,12 +1,15 @@ 'use strict'; -// Read once by src/config.js, so it is set before the module under test loads. +// Read once by src/config.js, so these are set before the module under test loads. process.env.FRONT_DOOR_ID = 'front-door-id-for-tests'; +// Two of the four addresses the OpenShift cluster calls out from, as deployed. +process.env.TRUSTED_PROXY_IPS = '142.34.194.121,142.34.194.122'; const assert = require('node:assert'); const { test } = require('node:test'); const geo = require('../src/ingest/enrich-geo'); +const { loadModule } = require('./helpers/load-config'); // What a real GeoLite2 city record looks like, trimmed to the parts that are read plus the // coordinates, which must not come out the other end. @@ -70,51 +73,120 @@ for (const ip of PUBLIC) { }); } +// The address demi-apim- stamps in X-Client-Ip for a request that came through the OpenShift +// cluster, and the address the Functions front end appends for APIM's own outbound hop. +const CLUSTER = '142.34.194.121'; +const APIM_HOP = '20.104.10.20'; + const CALLERS = [ { - // The last hop, not the first: everything to its left was written by whoever called us, so a - // caller could otherwise name any address it liked and be located as that address. - name: 'the last hop of X-Forwarded-For is the client', - headers: { 'x-forwarded-for': '10.0.0.5, 10.0.0.6, 24.108.0.1' }, - expected: '24.108.0.1' + // Hop -2, not hop -1: APIM does not append to X-Forwarded-For, so the last hop is the one Azure's + // Functions front end added for APIM itself, and everybody would geolocate to APIM. + name: 'a browser behind the cluster is the hop before the Azure one', + headers: { + 'x-forwarded-for': `24.108.0.1, ${APIM_HOP}`, + 'x-client-ip': CLUSTER + }, + expected: { ip: '24.108.0.1', trusted: false } + }, + { + // Everything left of what the OpenShift router appended was written by the caller. + name: 'hops a caller put in front of its own are ignored', + headers: { + 'x-forwarded-for': `8.8.8.8, 1.1.1.1, 24.108.0.1, ${APIM_HOP}`, + 'x-client-ip': CLUSTER + }, + expected: { ip: '24.108.0.1', trusted: false } + }, + { + name: 'a server producer calling APIM itself is the cluster address, and is trusted', + headers: { + 'x-forwarded-for': APIM_HOP, + 'x-client-ip': CLUSTER + }, + expected: { ip: CLUSTER, trusted: true } + }, + { + // A second entry from the egress pool is still the cluster, not somebody behind it. + name: 'a repeated cluster hop is not mistaken for a visitor', + headers: { + 'x-forwarded-for': `142.34.194.122, ${APIM_HOP}`, + 'x-client-ip': CLUSTER + }, + expected: { ip: CLUSTER, trusted: true } }, { - name: 'an address the caller put at the front of X-Forwarded-For is ignored', - headers: { 'x-forwarded-for': '8.8.8.8, 24.108.0.1' }, - expected: '24.108.0.1' + name: 'a caller reaching APIM straight off the internet is its own address, and is capped', + headers: { + 'x-forwarded-for': APIM_HOP, + 'x-client-ip': '8.8.8.8' + }, + expected: { ip: '8.8.8.8', trusted: false } + }, + { + // X-Client-Ip is APIM's, set with `override`, and every route carrying this code carries + // apimGuard — so a request that reached here without it did not come through APIM at all. + name: 'with no X-Client-Ip the last hop of X-Forwarded-For is the client', + headers: { 'x-forwarded-for': '10.0.0.5, 10.0.0.6, 24.108.0.1' }, + expected: { ip: '24.108.0.1', trusted: false } }, { name: 'a port appended to an IPv4 hop is dropped', headers: { 'x-forwarded-for': '24.108.0.1:52344' }, - expected: '24.108.0.1' + expected: { ip: '24.108.0.1', trusted: false } + }, + { + name: 'a port appended to the visitor hop behind the cluster is dropped', + headers: { + 'x-forwarded-for': `24.108.0.1:52344, ${APIM_HOP}`, + 'x-client-ip': CLUSTER + }, + expected: { ip: '24.108.0.1', trusted: false } }, { name: 'a bracketed IPv6 hop is unwrapped', headers: { 'x-forwarded-for': '[2606:4700:4700::1111]:52344' }, - expected: '2606:4700:4700::1111' + expected: { ip: '2606:4700:4700::1111', trusted: false } + }, + { + name: 'a bracketed IPv6 visitor behind the cluster is unwrapped', + headers: { + 'x-forwarded-for': `[2606:4700:4700::1111]:52344, ${APIM_HOP}`, + 'x-client-ip': CLUSTER + }, + expected: { ip: '2606:4700:4700::1111', trusted: false } + }, + { + name: 'a bracketed X-Client-Ip is unwrapped', + headers: { 'x-client-ip': '[2606:4700:4700::1111]:52344' }, + expected: { ip: '2606:4700:4700::1111', trusted: false } }, { name: 'an unbracketed IPv6 hop keeps all of its colons', headers: { 'x-forwarded-for': '2606:4700:4700::1111' }, - expected: '2606:4700:4700::1111' + expected: { ip: '2606:4700:4700::1111', trusted: false } }, { + // Front Door is not in the path yet, but when it is, its own view of the socket beats everything + // below it. name: 'the socket address wins when the request really came through our Front Door', headers: { 'x-azure-fdid': 'front-door-id-for-tests', 'x-azure-socketip': '24.108.0.1', - 'x-forwarded-for': '8.8.8.8' + 'x-forwarded-for': `8.8.8.8, ${APIM_HOP}`, + 'x-client-ip': CLUSTER }, - expected: '24.108.0.1' + expected: { ip: '24.108.0.1', trusted: false } }, { name: 'a forged Front Door id falls back to the forwarding chain', headers: { 'x-azure-fdid': 'not-our-front-door', - 'x-azure-socketip': '24.108.0.1', - 'x-forwarded-for': '8.8.8.8' + 'x-azure-socketip': '8.8.8.8', + 'x-forwarded-for': `24.108.0.1, ${APIM_HOP}`, + 'x-client-ip': CLUSTER }, - expected: '8.8.8.8' + expected: { ip: '24.108.0.1', trusted: false } }, { name: 'X-Azure-ClientIP is ignored, because the caller controls what Front Door derives it from', @@ -122,21 +194,40 @@ const CALLERS = [ 'x-azure-fdid': 'front-door-id-for-tests', 'x-azure-clientip': '24.108.0.1' }, - expected: '' + expected: { ip: '', trusted: false } }, { name: 'a request with no forwarding headers has no client address', headers: {}, - expected: '' + expected: { ip: '', trusted: false } } ]; for (const caller of CALLERS) { test(caller.name, () => { - assert.strictEqual(geo.clientIp(request(caller.headers)), caller.expected); + assert.deepStrictEqual(geo.resolveCaller(request(caller.headers)), caller.expected); }); } +test('clientIp is the resolved address on its own', () => { + const headers = { 'x-forwarded-for': `24.108.0.1, ${APIM_HOP}`, 'x-client-ip': CLUSTER }; + + assert.strictEqual(geo.clientIp(request(headers)), '24.108.0.1'); +}); + +// An instance with nothing trusted cannot tell our cluster from any other caller, so it stops at the +// address APIM saw and nobody is exempt from the cap. Fail-closed: an unset setting narrows what is +// trusted, it never widens it. +test('with no trusted proxy the caller is the address APIM saw, and is not trusted', () => { + const untrusting = loadModule('ingest/enrich-geo', { TRUSTED_PROXY_IPS: '' }); + const headers = { 'x-forwarded-for': `24.108.0.1, ${APIM_HOP}`, 'x-client-ip': CLUSTER }; + + assert.deepStrictEqual(untrusting.resolveCaller(request(headers)), { + ip: CLUSTER, + trusted: false + }); +}); + test('a known address yields country, region and city, and nothing else', async (t) => { geo._setReader(READER); t.after(() => geo._reset()); diff --git a/test/ingest-routes.test.js b/test/ingest-routes.test.js index 4b769ce..cbe1262 100644 --- a/test/ingest-routes.test.js +++ b/test/ingest-routes.test.js @@ -12,6 +12,8 @@ process.env.AUDIT_SHARED_HEADER_VALUE = 'audit-value-for-tests'; process.env.ALLOWED_ORIGINS = 'https://eagle-public-test.example.invalid,http://localhost:4200'; process.env.SESSION_EVENT_CAP = '2'; process.env.IP_EVENT_CAP = '3'; +// Two of the four addresses the OpenShift cluster calls out from, as deployed. +process.env.TRUSTED_PROXY_IPS = '142.34.194.121,142.34.194.122'; const assert = require('node:assert'); const { test, beforeEach } = require('node:test'); @@ -30,6 +32,17 @@ const ORIGIN = { origin: 'https://eagle-public-test.example.invalid' }; // entry; everything to its left is what the caller sent. const CLIENT = { 'x-forwarded-for': '10.0.0.5, 24.108.0.1' }; +// What the real chain puts on a request: the cluster's egress address in X-Client-Ip, stamped by +// demi-apim-, and APIM's own outbound hop appended after the visitor's by the Functions front +// end. A browser is the hop before that one; a server producer has no hop of its own. +const CLUSTER = '142.34.194.121'; +const APIM_HOP = '20.104.10.20'; + +const throughCluster = (visitor) => ({ + 'x-forwarded-for': visitor ? `${visitor}, ${APIM_HOP}` : APIM_HOP, + 'x-client-ip': CLUSTER +}); + // Every case here posts to /events, and a request with no X-Forwarded-For shares one `unknown` bucket // with every other, so without this the cap one case fills is charged to the next. beforeEach(() => ipCap._reset()); @@ -309,6 +322,95 @@ test('a batch past the per-address cap is refused with a Retry-After', async (t) ); }); +// eagle-api reaches APIM from the cluster's egress pool, which every browser behind the same cluster +// also comes out of. Capping that one address refused eagle-api's batches on prod. +test('a server producer behind the cluster is not capped by address', async (t) => { + recordRows(t); + geo._setReader(null); + t.after(() => geo._reset()); + sessionCap._reset(); + t.after(() => sessionCap._reset()); + + // IP_EVENT_CAP is 3 above, so a capped caller would be refused from the fourth event on. + const answers = []; + for (const n of [0, 1, 2, 3, 4]) { + const body = { events: [event({ sourceApp: 'eagle-api', sessionId: `server-${n}` })] }; + const response = await call('POST', '/analytics/events', { + headers: { ...GATEWAY, ...throughCluster(null) }, + body + }); + answers.push(response.status); + } + + assert.deepStrictEqual(answers, [202, 202, 202, 202, 202]); +}); + +test('two browsers behind the cluster get a budget each', async (t) => { + recordRows(t); + geo._setReader(null); + t.after(() => geo._reset()); + sessionCap._reset(); + t.after(() => sessionCap._reset()); + + const filling = [0, 1, 2].map((n) => event({ sessionId: `busy-visitor-${n}` })); + const first = await call('POST', '/analytics/events', { + headers: { ...GATEWAY, ...throughCluster('24.108.0.1') }, + body: { events: filling } + }); + const second = await call('POST', '/analytics/events', { + headers: { ...GATEWAY, ...throughCluster('198.51.100.7') }, + body: { events: [event({ sessionId: 'quiet-visitor' })] } + }); + + assert.deepStrictEqual({ first: first.status, second: second.status }, { first: 202, second: 202 }); +}); + +test('a browser behind the cluster is still capped at its own address', async (t) => { + recordRows(t); + geo._setReader(null); + t.after(() => geo._reset()); + sessionCap._reset(); + t.after(() => sessionCap._reset()); + + const from = { ...GATEWAY, ...throughCluster('203.0.113.9') }; + const filling = [0, 1, 2].map((n) => event({ sessionId: `capped-visitor-${n}` })); + const first = await call('POST', '/analytics/events', { headers: from, body: { events: filling } }); + const second = await call('POST', '/analytics/events', { + headers: from, + body: { events: [event({ sessionId: 'capped-visitor-3' })] } + }); + + assert.deepStrictEqual( + { first: first.status, second: second.status, retryAfter: second.headers['retry-after'] }, + { first: 202, second: 429, retryAfter: '60' } + ); +}); + +// Every event geolocated to APIM's Toronto address before the visitor hop was read. +test('a browser behind the cluster is located by its own address, not the gateway hop', async (t) => { + const sent = recordRows(t); + geo._setReader({ + get: (ip) => (ip === '24.108.0.1' + ? { country: { iso_code: 'CA' }, subdivisions: [{ iso_code: 'BC' }], city: { names: { en: 'Victoria' } } } + : { country: { iso_code: 'US' }, city: { names: { en: 'Toronto' } } }) + }); + t.after(() => geo._reset()); + + const body = { events: [event({ sessionId: 'located-visitor' })] }; + await call('POST', '/analytics/events', { + headers: { ...GATEWAY, ...throughCluster('24.108.0.1') }, + body + }); + await writer.flush(); + + const { row } = sent[0]; + assert.deepStrictEqual({ Country: row.Country, Region: row.Region, City: row.City }, { + Country: 'CA', + Region: 'BC', + City: 'Victoria' + }); +}); + const AUDIT_ROW = Object.freeze({ action: 'project.updated', sourceApp: 'eagle-demi', @@ -380,3 +482,16 @@ test('an audit row reaches the audit stream, with the address it was called from } ); }); + +// The producer, not APIM's outbound hop: an audit trail that names the gateway names nobody. +test('an audit row from a server producer carries the address the gateway saw', async (t) => { + const sent = recordRows(t); + + await call('POST', '/analytics/audit', { + headers: { ...GATEWAY, ...AUDIT_HEADER, ...throughCluster(null) }, + body: { rows: [AUDIT_ROW] } + }); + await writer.flush(); + + assert.strictEqual(sent[0].row.SourceIp, CLUSTER); +});