fix(ingest): resolve visitor IP behind APIM

README.md

@@ -44,6 +44,17 @@ An event is `{timestamp, eventType, sessionId, sourceApp, userId?, properties?}`
 sent. Everything else is kept in `Detail`. The caller's address is used to look up country, region
 and city, and is then thrown away.
 
+Which address that is takes some care, because APIM is in the way. A browser event travels
+`browser -> OpenShift router -> nginx -> demi-apim-<env> -> Functions front end -> here`. The router
+appends the visitor to `X-Forwarded-For`, APIM adds nothing to that header but stamps the address it
+saw in `X-Client-Ip`, and Azure then appends APIM's own outbound address as the last hop — so the last
+hop is Azure's, not the visitor's. When `X-Client-Ip` is one of the addresses in `TRUSTED_PROXY_IPS`
+the request came through our cluster and the visitor is the hop before the Azure one. With no such hop
+the caller is a server-side producer like eagle-api: it comes out of the same cluster addresses as
+every browser behind it, so it is exempt from the per-address cap, which cannot tell them apart.
+Everyone else is read as their own caller and capped normally. Nothing further left in
+`X-Forwarded-For` is read, because a caller can write what it likes there.
+
 A `POST /query` body names a measure, a bin, a range and optional dimensions and filters. Every one
 is a key into `src/query/schema.js`; no column, table or operator name can come out of a request
 body, and the time range never reaches the query text at all.
@@ -104,6 +115,7 @@ instead of serving an open or a permanently-401 API.
 | `NODE_ENV` | empty | Set to `production` by the Function App, which picks the JSON log format over the readable one |
 | `SESSION_EVENT_CAP` | `2000` | Events one session may contribute per hour per instance; the rest are dropped |
 | `STORAGE_ACCOUNT_NAME` | empty | Holds the GeoLite2 database and the saved dashboards. Empty means no location lookup |
+| `TRUSTED_PROXY_IPS` | empty | Comma list of the addresses our own proxies call out from, as APIM reports them in `X-Client-Ip`. Decides which requests are read one `X-Forwarded-For` hop further back for the visitor, and which producers are exempt from the per-address cap. Empty trusts nothing |
 
 `ALLOWED_SOURCE_APPS`, `SESSION_EVENT_CAP` and `IP_EVENT_CAP` are readable but unset by the template:
 `src/config.js` owns those defaults, and a second copy in the Bicep would drift from it.

azure/main.bicep

@@ -42,6 +42,9 @@ param keycloakAllowedClients string = ''
 @description('Front Door\'s own id, from `az afd profile show --query frontDoorId` on the eagle-edge profile. Only on a match is X-Azure-SocketIP trusted over the caller-supplied X-Forwarded-For.')
 param frontDoorId string = ''
 
+@description('Comma list of the addresses our own proxies call out from, as APIM reports them in X-Client-Ip. Decides which requests are read one X-Forwarded-For hop further back for the visitor address, and which server producers are exempt from the per-address event cap. Empty trusts nothing.')
+param trustedProxyIps string = ''
+
 @description('Header name demi-apim-<env> stamps on a forwarded request.')
 param apimSharedHeaderName string = 'X-Analytics-Gateway'
 
@@ -177,6 +180,7 @@ module apiFunctionFlex './modules/api-function-flex.bicep' = {
     auditSharedHeaderValue: auditSharedHeaderValue
     allowedOrigins: allowedOrigins
     frontDoorId: frontDoorId
+    trustedProxyIps: trustedProxyIps
   }
 }
 

azure/main.prod.bicepparam

@@ -40,6 +40,11 @@ param keycloakAllowedClients = 'eagle-admin-console'
 // X-Azure-SocketIP is trusted, so a caller who knows it can choose its own client address.
 param frontDoorId = readEnvironmentVariable('FRONT_DOOR_ID')
 
+// The OpenShift cluster's egress pool, measured 2026-09-07 from what APIM stamped in X-Client-Ip. The
+// same four addresses in both environments. Unlike frontDoorId above these are safe in the open: APIM
+// sets X-Client-Ip with `override`, so knowing them does not let a caller claim to be the cluster.
+param trustedProxyIps = '142.34.194.121,142.34.194.122,142.34.194.123,142.34.194.124'
+
 // Both the apex and the www host serve the public site; the AFD endpoint hostname is the origin the
 // site is still reachable on directly. No localhost here.
 param allowedOrigins = [

azure/main.test.bicepparam

@@ -33,6 +33,11 @@ param keycloakAllowedClients = 'eagle-admin-console'
 // X-Azure-SocketIP is trusted, so a caller who knows it can choose its own client address.
 param frontDoorId = readEnvironmentVariable('FRONT_DOOR_ID')
 
+// The OpenShift cluster's egress pool, measured 2026-09-07 from what APIM stamped in X-Client-Ip. The
+// same four addresses in both environments. Unlike frontDoorId above these are safe in the open: APIM
+// sets X-Client-Ip with `override`, so knowing them does not let a caller claim to be the cluster.
+param trustedProxyIps = '142.34.194.121,142.34.194.122,142.34.194.123,142.34.194.124'
+
 // The two AFD hostnames carry a deploy-time hash and cannot be composed: eagle-public's is in
 // eagle-edge/README.md, eagle-demi-admin's in eagle-demi/azure/main.test.bicepparam. Third is
 // eagle-admin on OpenShift test. Both the apex and the www host serve the public site, same as prod.

azure/modules/api-function-flex.bicep

@@ -81,6 +81,9 @@ param allowedOrigins array = []
 @description('Front Door\'s own id (`az afd profile show --query frontDoorId`). Only on an X-Azure-FDID match is X-Azure-SocketIP trusted over the caller-supplied X-Forwarded-For.')
 param frontDoorId string = ''
 
+@description('Comma list of the addresses our own proxies call out from, as APIM reports them in X-Client-Ip (the OpenShift cluster egress pool). A request stamped with one of these is read one X-Forwarded-For hop further back for the visitor, and a server producer behind them is exempt from the per-address event cap. Empty trusts nothing.')
+param trustedProxyIps string = ''
+
 var apiAppName = 'analytics-api-fc-${environmentName}'
 var appServicePlanName = 'analytics-plan-fc-${environmentName}'
 var storageAccountName = take('analyticsfc${environmentName}${uniqueString(resourceGroup().id)}', 24)
@@ -352,6 +355,10 @@ resource apiFunctionApp 'Microsoft.Web/sites@2023-12-01' = {
           name: 'FRONT_DOOR_ID'
           value: frontDoorId
         }
+        {
+          name: 'TRUSTED_PROXY_IPS'
+          value: trustedProxyIps
+        }
         // Picks the JSON log format in src/utils/logger.js.
         {
           name: 'NODE_ENV'

docs/EVENT-SCHEMA.md

@@ -18,7 +18,9 @@ comes back only when nothing at all was accepted, or when the envelope itself is
 buffered in the Function and posted to Log Analytics on a timer, so `202` means accepted, not stored.
 
 `POST /events` also answers `429` with `Retry-After: 60` once one client address has sent more than
-`IP_EVENT_CAP` events in a minute. Audit rows are never refused for volume.
+`IP_EVENT_CAP` events in a minute. Server-side producers are exempt: they reach the gateway from the
+OpenShift cluster's egress addresses (`TRUSTED_PROXY_IPS`), which every browser behind the cluster
+shares, so a per-address cap cannot separate them. Audit rows are never refused for volume.
 
 ## What the client sends
 

src/config.js

@@ -105,6 +105,13 @@ module.exports = {
   // ceiling is the Function's own (src/ingest/ip-cap.js).
   ipEventCap: intFromEnv('IP_EVENT_CAP', 600, 1),
 
+  // Addresses our own proxies call out from, as APIM reports them in X-Client-Ip: the OpenShift
+  // cluster's egress pool. A request stamped with one of these came through our own infrastructure,
+  // which is what lets src/ingest/enrich-geo.js look one hop further back in X-Forwarded-For for the
+  // visitor instead of geolocating and rate-capping the whole cluster as one caller. Empty trusts no
+  // proxy, so every caller is read as itself and capped.
+  trustedProxyIps: listFromEnv('TRUSTED_PROXY_IPS', ''),
+
   // Read here rather than in src/utils/logger.js, so this file stays the only reader of process.env.
   get nodeEnv() { return process.env.NODE_ENV || ''; },
 

src/controllers/events.js

@@ -2,7 +2,7 @@
 
 const { EVENTS_STREAM, enqueue } = require('../ingest/dcr-writer');
 const { enrichDevice } = require('../ingest/enrich-device');
-const { clientIp, geoFields } = require('../ingest/enrich-geo');
+const { geoFields, resolveCaller } = require('../ingest/enrich-geo');
 const ipCap = require('../ingest/ip-cap');
 const { allow } = require('../ingest/session-cap');
 const { toEventRow, validateEventBatch } = require('../ingest/validate');
@@ -21,13 +21,17 @@ const { toEventRow, validateEventBatch } = require('../ingest/validate');
  */
 async function events(req, res) {
   // One resolution of the address, used for the cap and then for the location lookup.
-  const ip = clientIp(req);
+  const { ip, trusted } = resolveCaller(req);
 
   // Before validation, deliberately: an address over its cap should not cost this instance the work
   // of parsing what it sent. 429 and not a silent drop, because a refused batch is the producer's to
   // retry.
+  //
+  // Our own server-side producers are exempt: they reach APIM from the cluster's egress pool, which
+  // every browser behind the same cluster also comes out of, so one address cap cannot separate them
+  // and capping it refused eagle-api's batches on prod.
   const offered = Array.isArray(req.body && req.body.events) ? req.body.events.length : 1;
-  if (!ipCap.allow(ip, offered)) {
+  if (!trusted && !ipCap.allow(ip, offered)) {
     res.set('Retry-After', String(ipCap.RETRY_AFTER_SECONDS));
     res.status(429).json({ error: 'Too many events from this address. Retry in a minute.' });
     return;

src/ingest/enrich-geo.js

@@ -76,24 +76,55 @@ function isPrivateIp(value) {
   return IPV4_PRIVATE.some((range) => range.test(ip));
 }
 
+/** One of the addresses our own proxies call out from (config.trustedProxyIps). */
+function isTrustedProxy(ip) {
+  return Boolean(ip) && config.trustedProxyIps.includes(ip);
+}
+
 /**
- * Who called, as far as it can be trusted. X-Azure-ClientIP is deliberately not read: Front Door
- * derives it from the caller's own X-Forwarded-For, so the caller controls it. Same trust rule as
- * eagle-api's rateLimitKey helper — the socket address only when the request really came through our
- * Front Door profile.
+ * Who called, as far as it can be trusted.
+ *
+ * A browser event arrives through
+ * `browser -> OpenShift router -> rproxy -> demi-apim-<env> -> Functions front end -> here`. The
+ * router appends the visitor to X-Forwarded-For; APIM appends nothing to it and instead stamps the
+ * address IT saw in X-Client-Ip, and the Functions front end then appends APIM's own outbound address.
+ * So the header reads `[…what the caller sent…, visitor, apim]` — hop -1 is Azure's, hop -2 is the
+ * visitor, and everything further left is caller-written and worth nothing.
+ *
+ * X-Client-Ip is only trustworthy because every route reaching here also carries `apimGuard`
+ * (src/auth/apim-header.js): APIM sets it with `override`, and a request that skipped the gateway is a
+ * 401 before a controller runs. X-Azure-ClientIP stays unread: Front Door derives it from the caller's
+ * own X-Forwarded-For. Same trust rule as eagle-api's rateLimitKey helper.
+ *
+ * @returns {{ip: string, trusted: boolean}} `trusted` marks one of our own server producers, which
+ * shares the cluster's egress address with every browser behind it and so cannot be capped by address.
  */
-function clientIp(req) {
+function resolveCaller(req) {
   if (config.frontDoorId && safeEqual(req.header('x-azure-fdid'), config.frontDoorId)) {
     const socketIp = normalizeIp(req.header('x-azure-socketip'));
-    if (socketIp) return socketIp;
+    if (socketIp) return { ip: socketIp, trusted: false };
   }
 
-  // The RIGHT-most hop, not the left-most: every entry to its left was written by whoever called us,
-  // and the last one is the address the proxy immediately in front of this app appended.
-  const forwarded = req.header('x-forwarded-for');
-  if (forwarded) return normalizeIp(String(forwarded).split(',').at(-1));
+  const hops = String(req.header('x-forwarded-for') || '').split(',').map(normalizeIp).filter(Boolean);
+  const gatewaySaw = normalizeIp(req.header('x-client-ip'));
+
+  if (isTrustedProxy(gatewaySaw)) {
+    const visitor = hops.at(-2);
+    if (visitor && !isTrustedProxy(visitor)) return { ip: visitor, trusted: false };
 
-  return '';
+    // Nothing behind the proxy: an eagle-api pod or another server producer calling APIM itself.
+    return { ip: gatewaySaw, trusted: true };
+  }
+
+  // Somebody calling APIM straight off the internet. Their own address, and the cap applies.
+  if (gatewaySaw) return { ip: gatewaySaw, trusted: false };
+
+  return { ip: hops.at(-1) || '', trusted: false };
+}
+
+/** The resolved address on its own, for a caller that has no use for the trust flag. */
+function clientIp(req) {
+  return resolveCaller(req).ip;
 }
 
 /** The cached file if there is one, otherwise a fresh copy from the blob container. */
@@ -179,6 +210,7 @@ async function geoFields(ip) {
 
 module.exports = {
   clientIp,
+  resolveCaller,
   geoFields,
   isPrivateIp,
   // Test seam. One reader, one real implementation, so no abstraction over it.

test/config.test.js

@@ -65,3 +65,18 @@ test('ALLOWED_ORIGINS is read as a trimmed list', () => {
 test('an unset ALLOWED_ORIGINS admits no browser at all', () => {
   assert.deepStrictEqual(loadConfig({ ENVIRONMENT: 'test' }).allowedOrigins, []);
 });
+
+test('TRUSTED_PROXY_IPS is read as a trimmed list', () => {
+  const config = loadConfig({
+    ENVIRONMENT: 'test',
+    TRUSTED_PROXY_IPS: '142.34.194.121, 142.34.194.122'
+  });
+
+  assert.deepStrictEqual(config.trustedProxyIps, ['142.34.194.121', '142.34.194.122']);
+});
+
+// Nothing trusted is the safe default: every caller is located and capped by the last hop, which is
+// what this service did before the cluster's egress addresses were known.
+test('an unset TRUSTED_PROXY_IPS trusts no proxy', () => {
+  assert.deepStrictEqual(loadConfig({ ENVIRONMENT: 'test' }).trustedProxyIps, []);
+});