Rate limiting, audit and metrics for the osctrld endpoints

ARCHITECTURE.md

@@ -137,6 +137,7 @@ Request controls are composed at route registration:
 
 - `cmd/api/auth.go`: `handlerAuthCheck(...)` accepts a bearer JWT or the SPA's JWT cookie, verifies it against the user's currently stored token, enforces CSRF on cookie-authenticated mutations, and attaches the username to request context.
 - Login, pre-auth discovery, service-config apply, and TLS enrollment have configurable in-memory per-IP rate limits.
+- `osctrld` routes in `osctrl-tls` (`rateLimits.osctrldAuth`, `osctrld`, `osctrldDownload`; `cmd/tls/handlers/osctrld_limits.go`): every request is limited per client IP (default 300/min, extension downloads also 30/min), and wrong environment secrets are charged to a per-IP-and-environment budget (default 10/hour). Once that budget is spent the IP gets 429 with `Retry-After` on that environment's `osctrld` routes, even with the right secret: checked before the body is read, and again atomically with the secret comparison (`ratelimit.Limiter.Attempt`), so concurrent requests cannot exceed it. Correct secrets never charge it, so hosts sharing a NAT are unaffected. The client IP is `utils.GetIP`: behind a reverse proxy or load balancer, `service.trustedProxies` must list it, otherwise all clients share the proxy's budgets and one bad client can block the fleet. Past `maxBuckets` tracked keys, new keys share one overflow budget and are blocked together. Every wrong secret is audited (while blocked, one rejection per budget period per IP and environment), capped at 60 audit entries per hour per environment; drops are counted. Metrics: `osctrld_auth_failures_total{environment}`, `osctrld_rate_limited_total{route,limit}`, `osctrld_auth_audit_dropped_total{environment}`, and request duration/status by route pattern. Limits are per process.
 - Forwarded client IP headers are ignored unless the peer matches configured trusted-proxy CIDRs.
 - `cmd/tls/handlers/handlers.go`: `PrometheusMiddleware(...)` records request duration/status for enabled osquery-facing endpoints; handlers also apply endpoint-specific body-size caps.
 

cmd/tls/config_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/jmpsec/osctrl/pkg/config"
+	"github.com/spf13/viper"
 	"github.com/urfave/cli/v3"
 )
 
@@ -151,6 +152,20 @@ func TestSampleTLSConfigLoads(t *testing.T) {
 	if params.Osquery == nil {
 		t.Fatal("sample tls.yml did not load osquery")
 	}
+	// The sample documents every rate limit with its built-in default.
+	v := viper.New()
+	v.SetConfigFile(filepath.Join("..", "..", "deploy", "config", "tls.yml"))
+	if err := v.ReadInConfig(); err != nil {
+		t.Fatal(err)
+	}
+	for _, name := range config.TLSRateLimits {
+		if !v.IsSet("rateLimits." + name + ".burst") {
+			t.Errorf("sample tls.yml does not document rateLimits.%s", name)
+		}
+	}
+	if *params.RateLimits != config.DefaultRateLimits() {
+		t.Fatalf("sample tls.yml rateLimits = %+v, want the defaults", *params.RateLimits)
+	}
 }
 
 func TestConfigVersionMismatchDoesNotFailLoad(t *testing.T) {
@@ -175,3 +190,37 @@ redis:
 		t.Fatalf("cfg.Version = %d, want 999", cfg.Version)
 	}
 }
+
+// A tls.yml written before the osctrld limits existed has a rateLimits block
+// without them; it must still load, with the built-in osctrld defaults.
+func TestLoadedYAMLKeepsOsctrldRateLimitDefaults(t *testing.T) {
+	const body = `
+service:
+  auth: none
+rateLimits:
+  enroll:
+    burst: 9
+    period: 2m
+    evictAfter: 12m
+    retryAfter: 45
+  osctrld:
+    burst: 50
+`
+	cfg, err := loadYAMLConfiguration(writeTempTLSConfig(t, body))
+	if err != nil {
+		t.Fatalf("loadYAMLConfiguration: %v", err)
+	}
+	params := loadedYAMLToServiceParams(cfg, "tls.yml")
+	d := config.DefaultRateLimits()
+	if params.RateLimits.OsctrldAuth != d.OsctrldAuth || params.RateLimits.OsctrldDownload != d.OsctrldDownload {
+		t.Fatalf("osctrld limits = %+v / %+v, want defaults", params.RateLimits.OsctrldAuth, params.RateLimits.OsctrldDownload)
+	}
+	want := d.Osctrld
+	want.Burst = 50
+	if params.RateLimits.Osctrld != want {
+		t.Fatalf("osctrld = %+v, want %+v", params.RateLimits.Osctrld, want)
+	}
+	if params.RateLimits.Enroll.Burst != 9 {
+		t.Fatalf("enroll burst = %d, want 9", params.RateLimits.Enroll.Burst)
+	}
+}

cmd/tls/handlers/extensions.go

@@ -50,8 +50,7 @@ func (h *HandlersTLS) osctrldRequest(w http.ResponseWriter, r *http.Request, max
 		utils.HTTPResponse(w, "", http.StatusBadRequest, []byte(""))
 		return env, false
 	}
-	if !h.checkValidSecret(secret(), env) {
-		utils.HTTPResponse(w, "", http.StatusForbidden, []byte(""))
+	if !h.osctrldSecret(w, r, secret(), env) {
 		return env, false
 	}
 	return env, true

cmd/tls/handlers/handlers.go

@@ -83,6 +83,7 @@ type HandlersTLS struct {
 	NodeConfig      *nodeconfig.Manager      // nil: per-node config overrides are inert
 	FlagsOverrides  *nodeconfig.FlagsManager // nil: per-node flags overrides are inert
 	Extensions      *extensions.Manager      // nil: osctrld extension routes are not registered
+	OsctrldLimits   *OsctrldLimits           // nil: osctrld routes are not rate-limited
 	// DBHealth, when wired via WithDBHealth, lets HealthHandler
 	// report backend degradation. nil means "no monitor" and the
 	// health endpoint reports healthy as before.
@@ -438,6 +439,11 @@ func NewResponseWriter(w http.ResponseWriter) *responseWriter {
 	return &responseWriter{w, http.StatusOK}
 }
 
+// Unwrap lets http.ResponseController reach the underlying writer.
+func (rw *responseWriter) Unwrap() http.ResponseWriter {
+	return rw.ResponseWriter
+}
+
 func (rw *responseWriter) WriteHeader(code int) {
 	rw.statusCode = code
 	rw.ResponseWriter.WriteHeader(code)

cmd/tls/handlers/metrics.go

@@ -37,6 +37,24 @@ var (
 		Help:    "The duration of batch data flushing to backend",
 		Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5},
 	}, []string{"operation"})
+	// osctrldAuthFailures counts wrong osctrld secrets per environment name
+	// (only resolved environments, so cardinality stays bounded).
+	osctrldAuthFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "osctrld_auth_failures_total",
+		Help: "osctrld requests rejected for a wrong environment secret",
+	}, []string{"environment"})
+	// osctrldRateLimited counts osctrld requests rejected with 429, by
+	// route pattern and rate limit (osctrldAuth, osctrld, osctrldDownload).
+	osctrldRateLimited = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "osctrld_rate_limited_total",
+		Help: "osctrld requests rejected by a rate limit",
+	}, []string{"route", "limit"})
+	// osctrldAuthAuditDropped counts failed-secret audit entries dropped by
+	// the per-environment audit cap.
+	osctrldAuthAuditDropped = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "osctrld_auth_audit_dropped_total",
+		Help: "osctrld failed-secret audit entries dropped by the per-environment cap",
+	}, []string{"environment"})
 	// DBDegraded is 1 when the DB health monitor reports the database
 	// as degraded (caches are in stale-serve mode), 0 when healthy.
 	dbDegraded = prometheus.NewGauge(prometheus.GaugeOpts{
@@ -52,6 +70,9 @@ func RegisterMetrics(reg prometheus.Registerer) {
 	reg.MustRegister(distributedQueryProcessingDuration)
 	reg.MustRegister(batchFlushDuration)
 	reg.MustRegister(dbDegraded)
+	reg.MustRegister(osctrldAuthFailures)
+	reg.MustRegister(osctrldRateLimited)
+	reg.MustRegister(osctrldAuthAuditDropped)
 }
 
 // SetDBDegraded updates the db_degraded gauge. Called by the DB

cmd/tls/handlers/osctrld_limits.go

@@ -0,0 +1,168 @@
+package handlers
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/jmpsec/osctrl/pkg/config"
+	"github.com/jmpsec/osctrl/pkg/environments"
+	"github.com/jmpsec/osctrl/pkg/ratelimit"
+	"github.com/jmpsec/osctrl/pkg/utils"
+)
+
+// osctrldAuditPerHour caps failed-secret audit writes per environment, so
+// a spray from many addresses cannot flood audit_logs.
+const osctrldAuditPerHour = 60
+
+// OsctrldLimits holds the per-process rate limiters of the osctrld routes.
+// They key on utils.GetIP: the TCP peer, or the forwarded client when the
+// peer is a configured trusted proxy (otherwise every client behind a proxy
+// shares the proxy's budgets).
+type OsctrldLimits struct {
+	// Auth is charged only by wrong secrets, per address and environment.
+	Auth *ratelimit.Limiter
+	// Traffic is charged by every osctrld request, per address.
+	Traffic *ratelimit.Limiter
+	// Download is charged by every extension download, per address.
+	Download *ratelimit.Limiter
+	// authAudit lets one blocked rejection per Auth period reach the
+	// audit log, per address and environment.
+	authAudit *ratelimit.Limiter
+	// auditCap bounds failed-secret audit writes per environment.
+	auditCap *ratelimit.Limiter
+}
+
+// NewOsctrldLimits builds the osctrld limiters from the rate-limit config.
+func NewOsctrldLimits(cfg config.YAMLConfigurationRateLimits) *OsctrldLimits {
+	audit := cfg.OsctrldAuth
+	audit.Burst = 1
+	return &OsctrldLimits{
+		Auth:      ratelimit.NewFromConfig(cfg.OsctrldAuth),
+		Traffic:   ratelimit.NewFromConfig(cfg.Osctrld),
+		Download:  ratelimit.NewFromConfig(cfg.OsctrldDownload),
+		authAudit: ratelimit.NewFromConfig(audit),
+		auditCap:  ratelimit.New(osctrldAuditPerHour, time.Hour, 2*time.Hour),
+	}
+}
+
+// WithOsctrldLimits rate-limits the osctrld routes; nil leaves them unlimited.
+func WithOsctrldLimits(l *OsctrldLimits) Option {
+	return func(h *HandlersTLS) {
+		h.OsctrldLimits = l
+	}
+}
+
+// osctrldAuthKey keys the failed-secret budget by address and environment
+// UUID, case-folded so spellings of one environment share a budget.
+func osctrldAuthKey(ip, envUUID string) string {
+	return ip + "|" + strings.ToLower(envUUID)
+}
+
+// OsctrldMiddleware wraps an osctrld route: the per-address traffic limit
+// (and the download limit when download is set), then the failed-secret
+// block, which turns a blocked address away before its body is read.
+// Requests and latency are recorded by route pattern.
+func (h *HandlersTLS) OsctrldMiddleware(download bool, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		rw := NewResponseWriter(w)
+		h.osctrldLimit(rw, r, download, next)
+		requestDuration.WithLabelValues(r.Method, r.Pattern, strconv.Itoa(rw.statusCode)).Observe(time.Since(start).Seconds())
+	})
+}
+
+func (h *HandlersTLS) osctrldLimit(w http.ResponseWriter, r *http.Request, download bool, next http.Handler) {
+	l := h.OsctrldLimits
+	if l == nil {
+		next.ServeHTTP(w, r)
+		return
+	}
+	reject := func(lim *ratelimit.Limiter, name string) {
+		osctrldRateLimited.WithLabelValues(r.Pattern, name).Inc()
+		lim.Reject(w)
+	}
+	ip := utils.GetIP(r)
+	if !l.Traffic.Allow(ip) {
+		reject(l.Traffic, "osctrld")
+		return
+	}
+	if download && !l.Download.Allow(ip) {
+		reject(l.Download, "osctrldDownload")
+		return
+	}
+	// Cheap early rejection; osctrldSecret re-checks atomically.
+	envVar := r.PathValue("env")
+	if l.Auth.Exhausted(osctrldAuthKey(ip, envVar)) {
+		h.osctrldBlocked(w, r, ip, envVar, nil)
+		return
+	}
+	next.ServeHTTP(w, r)
+}
+
+// osctrldBlocked answers 429 for an address whose failed-secret budget for
+// the environment is spent. The first rejection per Auth period is audited;
+// env is resolved from envVar when nil.
+func (h *HandlersTLS) osctrldBlocked(w http.ResponseWriter, r *http.Request, ip, envVar string, env *environments.TLSEnvironment) {
+	l := h.OsctrldLimits
+	if l.authAudit.Allow(osctrldAuthKey(ip, envVar)) {
+		if env == nil {
+			resolved := h.osctrldAuditEnv(r, envVar)
+			env = &resolved
+		}
+		h.osctrldAudit(ip, *env, "blocked after too many failed secrets")
+	}
+	osctrldRateLimited.WithLabelValues(r.Pattern, "osctrldAuth").Inc()
+	l.Auth.Reject(w)
+}
+
+// osctrldAudit writes a failed-secret audit entry unless the environment's
+// audit cap is spent, in which case the drop is counted.
+func (h *HandlersTLS) osctrldAudit(ip string, env environments.TLSEnvironment, reason string) {
+	if l := h.OsctrldLimits; l != nil && !l.auditCap.Allow(strings.ToLower(env.UUID)) {
+		osctrldAuthAuditDropped.WithLabelValues(env.Name).Inc()
+		return
+	}
+	h.AuditLog.FailedOsctrld(ip, env.Name, reason, env.ID)
+}
+
+// osctrldAuditEnv resolves the environment in the path for an audit entry,
+// falling back to the (UUID-validated) path value with no ID.
+func (h *HandlersTLS) osctrldAuditEnv(r *http.Request, envVar string) environments.TLSEnvironment {
+	if h.EnvCache != nil {
+		if env, err := h.EnvCache.GetByUUID(r.Context(), strings.ToLower(envVar)); err == nil {
+			return env
+		}
+	}
+	if !utils.CheckUUID(envVar) {
+		envVar = "<invalid>"
+	}
+	return environments.TLSEnvironment{UUID: envVar, Name: envVar}
+}
+
+// osctrldSecret checks the secret of an osctrld request. The budget check,
+// the comparison and the charge for a wrong secret happen atomically, so
+// requests that passed the middleware together cannot exceed the budget:
+// once it is spent the request gets 429 without a comparison. A wrong
+// secret is counted, audited (capped per environment) and answered 403.
+func (h *HandlersTLS) osctrldSecret(w http.ResponseWriter, r *http.Request, secret string, env environments.TLSEnvironment) bool {
+	valid := func() bool { return h.checkValidSecret(secret, env) }
+	ip := utils.GetIP(r)
+	if h.OsctrldLimits != nil {
+		allowed, ok := h.OsctrldLimits.Auth.Attempt(osctrldAuthKey(ip, env.UUID), valid)
+		if !allowed {
+			h.osctrldBlocked(w, r, ip, env.UUID, &env)
+			return false
+		}
+		if ok {
+			return true
+		}
+	} else if valid() {
+		return true
+	}
+	osctrldAuthFailures.WithLabelValues(env.Name).Inc()
+	h.osctrldAudit(ip, env, "invalid secret")
+	utils.HTTPResponse(w, "", http.StatusForbidden, []byte(""))
+	return false
+}