diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 461b567d1..7cbe39ee0 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -522,16 +522,13 @@ impl PlatformHttpClient for AxumPlatformHttpClient { /// /// # Degraded features in dev /// -/// KV store is [`trusted_server_core::platform::UnavailableKvStore`] — any route -/// touching synthetic-ID or consent KV will degrade gracefully. A `warn` log is +/// The generic runtime KV slot uses +/// [`trusted_server_core::platform::UnavailableKvStore`]. A `warn` log is /// emitted once per process. pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> RuntimeServices { static KV_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); KV_WARNED.get_or_init(|| { - log::warn!( - "Axum dev server: KV store is unavailable (UnavailableKvStore). \ - Routes that depend on synthetic-ID or consent KV will degrade gracefully." - ); + log::warn!("Axum dev server: generic runtime KV is unavailable (UnavailableKvStore)."); }); let client_ip = edgezero_adapter_axum::context::AxumRequestContext::get(ctx.request()) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 12b2fd164..3f7eaef4b 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -131,8 +131,8 @@ use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester} use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ - open_kv_store, FastlyPlatformBackend, FastlyPlatformConfigStore, FastlyPlatformGeo, - FastlyPlatformHttpClient, FastlyPlatformSecretStore, UnavailableKvStore, + FastlyPlatformBackend, FastlyPlatformConfigStore, FastlyPlatformGeo, FastlyPlatformHttpClient, + FastlyPlatformSecretStore, UnavailableKvStore, }; // --------------------------------------------------------------------------- @@ -195,36 +195,6 @@ fn warn_if_certificate_check_disabled(settings: &Settings) { } } -/// Resolves per-request consent KV store services for routes that read consent data. -/// -/// When `settings.consent.consent_store` is configured and the named KV store cannot -/// be opened, returns `Err` so the caller can respond with 503 (fail-closed). This is -/// intentional hardening over the legacy `route_request` path, which builds -/// `runtime_services` with `UnavailableKvStore` and never opens the named consent -/// store, so it never fails closed — the `EdgeZero` path instead makes consent-dependent -/// routes unavailable rather than proceeding without consent. -/// -/// # Errors -/// -/// Returns an error when the configured consent store cannot be opened. -pub(crate) fn runtime_services_for_consent_route( - settings: &Settings, - runtime_services: &RuntimeServices, -) -> Result> { - let Some(store_name) = settings.consent.consent_store.as_deref() else { - return Ok(runtime_services.clone()); - }; - - open_kv_store(store_name) - .map(|store| runtime_services.clone().with_kv_store(store)) - .map_err(|e| { - Report::new(TrustedServerError::KvStore { - store_name: store_name.to_string(), - message: e.to_string(), - }) - }) -} - // --------------------------------------------------------------------------- // Per-request RuntimeServices // --------------------------------------------------------------------------- @@ -589,10 +559,6 @@ async fn run_named_route( NamedRouteHandler::SetTester => handle_set_tester(&state.settings), NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { - // The auction reads consent data, so the consent KV store must be - // available — fail closed with 503 when it is configured but - // cannot be opened, matching legacy behavior. - let consent_services = runtime_services_for_consent_route(&state.settings, services)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let registry_ref = if partner_registry.is_empty() { None @@ -605,7 +571,7 @@ async fn run_named_route( ec.kv_graph.as_ref(), registry_ref, &ec.ec_context, - &consent_services, + services, req, ) .await @@ -617,10 +583,6 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - // Like the auction, page-bids reads consent data, so the consent KV - // store must be available — fail closed with 503 when configured but - // unopenable, matching legacy. - let consent_services = runtime_services_for_consent_route(&state.settings, services)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let registry_ref = if partner_registry.is_empty() { None @@ -634,7 +596,7 @@ async fn run_named_route( }; handle_page_bids( &state.settings, - &consent_services, + services, ec.kv_graph.as_ref(), auction, &ec.ec_context, @@ -768,49 +730,41 @@ async fn dispatch_fallback( } } - // Publisher pages read consent data, so the consent KV store must be - // available — fail closed with 503 when it is configured but cannot - // be opened, matching legacy behavior. - match runtime_services_for_consent_route(&state.settings, services) { - Ok(publisher_services) => { - // Run the server-side auction with the configured creative- - // opportunity slots and collect the dispatched bids in the - // buffered finalize (`buffer_publisher_response_async`), matching - // the legacy streaming path. `handle_publisher_request` matches the - // slots against the request path. The partner registry plus the - // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with - // server-side EIDs, same as the legacy auction. - let slots = state.settings.creative_opportunity_slots(); - match PartnerRegistry::from_config(&state.settings.ec.partners) { - Ok(partner_registry) => { - let auction = AuctionDispatch { - orchestrator: &state.orchestrator, - slots, - registry: Some(&partner_registry), - }; - match handle_publisher_request( + // Run the server-side auction with the configured creative- + // opportunity slots and collect the dispatched bids in the + // buffered finalize (`buffer_publisher_response_async`), matching + // the legacy streaming path. `handle_publisher_request` matches the + // slots against the request path. The partner registry plus the + // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with + // server-side EIDs, same as the legacy auction. + let slots = state.settings.creative_opportunity_slots(); + match PartnerRegistry::from_config(&state.settings.ec.partners) { + Ok(partner_registry) => { + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots, + registry: Some(&partner_registry), + }; + match handle_publisher_request( + &state.settings, + services, + ec.kv_graph.as_ref(), + &mut ec.ec_context, + auction, + req, + ) + .await + { + Ok(pub_response) => { + buffer_publisher_response_async( + pub_response, + &method, &state.settings, - &publisher_services, - ec.kv_graph.as_ref(), - &mut ec.ec_context, - auction, - req, + &state.registry, + &state.orchestrator, + services, ) .await - { - Ok(pub_response) => { - buffer_publisher_response_async( - pub_response, - &method, - &state.settings, - &state.registry, - &state.orchestrator, - &publisher_services, - ) - .await - } - Err(e) => Err(e), - } } Err(e) => Err(e), } @@ -1225,7 +1179,6 @@ mod tests { use error_stack::Report; use futures::executor::block_on; - use serde_json::json; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::error::TrustedServerError; @@ -1240,51 +1193,6 @@ mod tests { }; use trusted_server_core::settings::Settings; - fn settings_with_missing_consent_store() -> Settings { - Settings::from_toml( - r#" - [[handlers]] - path = "^/(_ts/)?admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [proxy] - allowed_domains = ["*.example", "*.example.com"] - - [ec] - passphrase = "test-passphrase-at-least-32-bytes!!" - - [request_signing] - enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - - [consent] - consent_store = "missing-consent-store" - - [integrations.prebid] - enabled = true - server_url = "https://test-prebid.com/openrtb2/auction" - external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" - - [integrations.datadome] - enabled = true - - [auction] - enabled = true - providers = ["prebid"] - timeout_ms = 2000 - "#, - ) - .expect("should parse EdgeZero app test settings") - } - fn app_state_for_settings(settings: Settings) -> Arc { build_state_from_settings(settings).expect("should build app state from settings") } @@ -1979,27 +1887,6 @@ mod tests { ); } - #[test] - fn dispatch_auction_with_missing_consent_store_returns_503() { - let state = app_state_for_settings(settings_with_missing_consent_store()); - let router = TrustedServerApp::routes_for_state(&state); - let body = json!({ "adUnits": [] }).to_string(); - let req = request_builder() - .method(Method::POST) - .uri("/auction") - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(body)) - .expect("should build auction request"); - - let response = route(&router, req); - - assert_eq!( - response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "auction route should fail closed when configured consent store cannot be opened" - ); - } - #[test] fn dispatch_unregistered_method_returns_405_at_router_level() { // Documents the known router-level behavior for verbs outside the @@ -2031,54 +1918,6 @@ mod tests { ); } - #[test] - fn edgezero_missing_consent_store_breaks_only_consent_routes() { - let state = app_state_for_settings(settings_with_missing_consent_store()); - let router = TrustedServerApp::routes_for_state(&state); - - let admin_response = route( - &router, - empty_request(Method::POST, "/_ts/admin/keys/rotate"), - ); - assert_eq!( - admin_response.status(), - StatusCode::UNAUTHORIZED, - "admin auth behavior should not depend on consent KV availability" - ); - - let auction_request = request_builder() - .method(Method::POST) - .uri("/auction") - .body(Body::from(r#"{"adUnits":[]}"#)) - .expect("should build auction request"); - let auction_response = route(&router, auction_request); - assert_eq!( - auction_response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "auction should fail closed when configured consent KV cannot be opened" - ); - - let publisher_response = route(&router, empty_request(Method::GET, "/articles/example")); - assert_eq!( - publisher_response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "publisher fallback should fail closed when configured consent KV cannot be opened" - ); - - // Integration routes must NOT require the consent KV — runtime_services_for_consent_route - // is wired only into the publisher and auction branches of dispatch_fallback, not into - // the integration proxy branch. A missing consent store must not 503 integration routes. - let integration_response = route( - &router, - empty_request(Method::GET, "/integrations/datadome/tags.js"), - ); - assert_ne!( - integration_response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "integration routes should be unaffected by a missing consent KV store" - ); - } - #[test] fn dispatch_fallback_asset_route_skips_ec_finalization() { // Parity guard for the configured asset-route fallback: a GET matching a diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 106ced787..347c7fe14 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -1,16 +1,12 @@ //! Fastly-backed implementations of the platform traits defined in //! `trusted-server-core::platform`. -use std::io::Read as _; -use std::net::IpAddr; -use std::sync::Arc; - use bytes::Bytes; -use edgezero_adapter_fastly::key_value_store::FastlyKvStore; -use edgezero_core::key_value_store::KvError; use error_stack::{Report, ResultExt}; use fastly::geo::{geo_lookup, Geo}; use fastly::{ConfigStore, Request, SecretStore}; +use std::io::Read as _; +use std::net::IpAddr; use crate::backend::BackendConfig; pub(crate) use trusted_server_core::platform::UnavailableKvStore; @@ -18,8 +14,8 @@ use trusted_server_core::platform::{ ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, PlatformImageOptimizerParams, - PlatformImageOptimizerRegion, PlatformKvStore, PlatformPendingRequest, PlatformResponse, - PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, + PlatformImageOptimizerRegion, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, + PlatformSelectResult, StoreId, StoreName, }; // --------------------------------------------------------------------------- @@ -583,16 +579,6 @@ pub fn client_info_from_request(req: &Request) -> ClientInfo { } } -/// Open a named KV store as a [`PlatformKvStore`] implementation. -/// -/// # Errors -/// -/// Returns [`KvError::Unavailable`] when the store does not exist, or -/// [`KvError::Internal`] when the Fastly SDK fails to open it. -pub fn open_kv_store(store_name: &str) -> Result, KvError> { - FastlyKvStore::open(store_name).map(|store| Arc::new(store) as Arc) -} - // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 1e13ca300..deb59e926 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -202,14 +202,9 @@ fn spin_secret_variable_name( /// /// Delegates all operations through `KvHandle`'s raw-bytes API. Spin KV has no /// native TTL support, so [`put_bytes_with_ttl`](KvStore::put_bytes_with_ttl) -/// *errors* (`KvError::Validation`) rather than silently writing a non-expiring -/// record — the privacy-safe failure mode. Consequently TTL-backed consent -/// persistence (`save_consent_to_kv`) is unavailable on Spin: each write returns -/// the error, which the core caller logs and treats as non-fatal (consistent -/// with all adapters — failing to persist consent never breaks the request, and -/// not persisting is the safe direction). Operators configuring -/// `settings.consent.consent_store` on Spin should expect stored-consent -/// fallback not to function. +/// returns `KvError::Validation` rather than silently writing a non-expiring +/// record. Callers of the generic platform KV interface must handle that +/// capability difference explicitly. struct KvHandleAdapter(KvHandle); #[async_trait::async_trait(?Send)] diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index fb49d513c..4540483d8 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -8,10 +8,8 @@ //! auction pipeline and populates `OpenRTB` bid requests. //! //! Consent is interpreted from request cookies, headers, geolocation, and -//! publisher policy defaults. When the caller supplies an EC ID and a KV -//! store via [`ConsentPipelineInput`], the pipeline also loads persisted -//! consent as a fallback and persists cookie-sourced consent on change. EC -//! identity lifecycle state is managed separately by the EC identity graph. +//! publisher policy defaults. EC identity lifecycle and withdrawal state are +//! managed separately by the EC identity graph. //! //! # Supported signals //! @@ -38,7 +36,6 @@ pub mod tcf; pub mod types; pub mod us_privacy; -pub use crate::storage::kv_store as kv; pub use extraction::extract_consent_signals; pub use types::{ ConsentContext, ConsentSource, PrivacyFlag, RawConsentSignals, TcfConsent, UsPrivacy, @@ -76,17 +73,6 @@ pub struct ConsentPipelineInput<'a> { pub config: &'a ConsentConfig, /// Geolocation data from the request (for jurisdiction detection). pub geo: Option<&'a GeoInfo>, - /// EC ID for KV Store consent persistence. - /// - /// When set along with `kv_store`, enables: - /// - **Read fallback**: loads consent from KV when cookies are absent. - /// - **Write-on-change**: persists cookie-sourced consent to KV. - pub ec_id: Option<&'a str>, - /// KV store for consent persistence. - /// - /// `None` when consent persistence is not configured for this request, or - /// when the caller intentionally skips consent KV access. - pub kv_store: Option<&'a dyn crate::platform::PlatformKvStore>, } /// Extracts, decodes, and normalizes consent signals from a request. @@ -101,16 +87,8 @@ pub struct ConsentPipelineInput<'a> { /// 6. Builds a [`ConsentContext`] with both raw and decoded data. /// 7. Logs a summary for observability. /// -/// When [`ConsentPipelineInput::ec_id`] and [`ConsentPipelineInput::kv_store`] -/// are both set, the pipeline also: -/// -/// - **Read fallback**: loads consent persisted in KV for the EC ID when the -/// request carries no consent signals. -/// - **Write-on-change**: persists cookie-sourced consent to KV after the -/// context is built (skipping empty contexts and unchanged fingerprints). -/// -/// Without those inputs the returned context reflects request-local consent -/// signals plus policy defaults only. +/// The returned context reflects request-local consent signals plus policy +/// defaults only. /// /// Decoding failures are logged and the corresponding decoded field is set to /// `None` — the raw string is still preserved for proxy-mode forwarding. @@ -122,20 +100,6 @@ pub fn build_consent_context(input: &ConsentPipelineInput<'_>) -> ConsentContext log_missing_geo_warning_once(); } - // Read fallback: when the request carries no consent signals, fall back - // to consent persisted in KV for this EC ID (when persistence is wired). - if signals.is_empty() { - if let (Some(ec_id), Some(store)) = (input.ec_id, input.kv_store) { - if let Some(mut ctx) = kv::load_consent_from_kv(store, ec_id) { - // Jurisdiction is request-local: derive it from the current - // geo rather than the value stored with the persisted entry. - ctx.jurisdiction = jurisdiction::detect_jurisdiction(input.geo, input.config); - log_consent_context(&ctx); - return ctx; - } - } - } - // In proxy mode, skip decoding entirely. if input.config.mode == ConsentMode::Proxy { let jur = jurisdiction::detect_jurisdiction(input.geo, input.config); @@ -169,13 +133,6 @@ pub fn build_consent_context(input: &ConsentPipelineInput<'_>) -> ConsentContext apply_expiration_check(&mut ctx, input.config); apply_gpc_us_privacy(&mut ctx, input.config); - // Write-on-change: persist cookie-sourced consent for this EC ID (when - // persistence is wired). The helper skips empty contexts and unchanged - // fingerprints internally. - if let (Some(ec_id), Some(store)) = (input.ec_id, input.kv_store) { - kv::save_consent_to_kv(store, ec_id, &ctx, input.config.max_consent_age_days); - } - log_consent_context(&ctx); ctx } @@ -904,8 +861,6 @@ mod tests { req: &req, config: &config, geo: None, - ec_id: None, - kv_store: None, }); assert_eq!( @@ -933,8 +888,6 @@ mod tests { req: &req, config: &config, geo: None, - ec_id: None, - kv_store: None, }); assert!( @@ -963,8 +916,6 @@ mod tests { req: &req, config: &config, geo: None, - ec_id: None, - kv_store: None, }); assert!( @@ -1473,161 +1424,4 @@ mod tests { "GPP without US section should fall through to us_privacy" ); } - - // ----------------------------------------------------------------------- - // Consent KV read-fallback / write-on-change pipeline tests - // ----------------------------------------------------------------------- - - struct InMemoryKvStore { - entries: std::sync::Mutex>, - } - - impl InMemoryKvStore { - fn new() -> Self { - Self { - entries: std::sync::Mutex::new(std::collections::HashMap::new()), - } - } - } - - #[async_trait::async_trait(?Send)] - impl crate::platform::PlatformKvStore for InMemoryKvStore { - async fn get_bytes( - &self, - key: &str, - ) -> Result, crate::platform::KvError> { - Ok(self - .entries - .lock() - .expect("should lock entries") - .get(key) - .cloned()) - } - - async fn put_bytes( - &self, - key: &str, - value: bytes::Bytes, - ) -> Result<(), crate::platform::KvError> { - self.entries - .lock() - .expect("should lock entries") - .insert(key.to_owned(), value); - Ok(()) - } - - async fn put_bytes_with_ttl( - &self, - key: &str, - value: bytes::Bytes, - _ttl: std::time::Duration, - ) -> Result<(), crate::platform::KvError> { - self.put_bytes(key, value).await - } - - async fn delete(&self, key: &str) -> Result<(), crate::platform::KvError> { - self.entries - .lock() - .expect("should lock entries") - .remove(key); - Ok(()) - } - - async fn list_keys_page( - &self, - _prefix: &str, - _cursor: Option<&str>, - _limit: usize, - ) -> Result { - Ok(edgezero_core::key_value_store::KvPage::default()) - } - } - - #[test] - fn pipeline_persists_cookie_sourced_consent_to_kv() { - let jar = parse_cookies_to_jar("us_privacy=1YNN"); - let req = build_request(); - let config = ConsentConfig::default(); - let store = InMemoryKvStore::new(); - - let ctx = build_consent_context(&ConsentPipelineInput { - jar: Some(&jar), - req: &req, - config: &config, - geo: None, - ec_id: Some("test-ec-id"), - kv_store: Some(&store), - }); - - assert_eq!( - ctx.raw_us_privacy.as_deref(), - Some("1YNN"), - "should build cookie-sourced consent" - ); - let persisted = crate::consent::kv::load_consent_from_kv(&store, "test-ec-id") - .expect("should persist cookie-sourced consent to KV"); - assert_eq!( - persisted.raw_us_privacy.as_deref(), - Some("1YNN"), - "persisted consent should round-trip the cookie signal" - ); - } - - #[test] - fn pipeline_falls_back_to_kv_consent_when_request_has_no_signals() { - let config = ConsentConfig::default(); - let store = InMemoryKvStore::new(); - - // First request carries a consent cookie — persisted to KV. - let jar = parse_cookies_to_jar("us_privacy=1YNN"); - let req = build_request(); - build_consent_context(&ConsentPipelineInput { - jar: Some(&jar), - req: &req, - config: &config, - geo: None, - ec_id: Some("test-ec-id"), - kv_store: Some(&store), - }); - - // Second request has no consent signals — must fall back to KV. - let bare_req = build_request(); - let ctx = build_consent_context(&ConsentPipelineInput { - jar: None, - req: &bare_req, - config: &config, - geo: None, - ec_id: Some("test-ec-id"), - kv_store: Some(&store), - }); - - assert_eq!( - ctx.raw_us_privacy.as_deref(), - Some("1YNN"), - "should load persisted consent when the request carries no signals" - ); - } - - #[test] - fn pipeline_skips_kv_when_persistence_not_wired() { - let jar = parse_cookies_to_jar("us_privacy=1YNN"); - let req = build_request(); - let config = ConsentConfig::default(); - let store = InMemoryKvStore::new(); - - // ec_id is absent, so the pipeline must not touch the KV store. - build_consent_context(&ConsentPipelineInput { - jar: Some(&jar), - req: &req, - config: &config, - geo: None, - ec_id: None, - kv_store: Some(&store), - }); - - assert!( - crate::consent::kv::load_consent_from_kv(&store, "test-ec-id").is_none(), - "should not persist consent without an EC ID" - ); - } } diff --git a/crates/trusted-server-core/src/consent/types.rs b/crates/trusted-server-core/src/consent/types.rs index ac82aaf2e..bf3f0f426 100644 --- a/crates/trusted-server-core/src/consent/types.rs +++ b/crates/trusted-server-core/src/consent/types.rs @@ -7,7 +7,7 @@ //! - [`UsPrivacy`] / [`PrivacyFlag`] — decoded US Privacy (CCPA) 4-char string //! - [`TcfConsent`] — decoded TCF v2 core consent data //! - [`GppConsent`] — decoded GPP consent data -//! - [`ConsentSource`] — how consent was sourced (cookie, KV store, etc.) +//! - [`ConsentSource`] — how consent was sourced (cookie or policy default) use core::fmt; @@ -383,8 +383,6 @@ impl fmt::Display for UsPrivacy { pub enum ConsentSource { /// Read from cookies on the incoming request. Cookie, - /// Loaded from KV store via Edge Cookie (EC) ID lookup. - KvStore, /// Applied from explicit publisher policy defaults. PolicyDefault, /// No consent data available. diff --git a/crates/trusted-server-core/src/consent_config.rs b/crates/trusted-server-core/src/consent_config.rs index 3b5ff2570..89249d150 100644 --- a/crates/trusted-server-core/src/consent_config.rs +++ b/crates/trusted-server-core/src/consent_config.rs @@ -73,11 +73,6 @@ pub struct ConsentConfig { /// but disagree on consent status. #[serde(default)] pub conflict_resolution: ConflictResolutionConfig, - /// When set, consent data is persisted per Edge Cookie (EC) ID so that - /// returning users without consent cookies can still have their - /// consent preferences applied. Set to `None` to disable. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub consent_store: Option, } impl Default for ConsentConfig { @@ -90,7 +85,6 @@ impl Default for ConsentConfig { us_states: UsStatesConfig::default(), us_privacy_defaults: UsPrivacyDefaultsConfig::default(), conflict_resolution: ConflictResolutionConfig::default(), - consent_store: None, } } } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 584bd9190..a178521ed 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -299,8 +299,6 @@ impl EcContext { req, config: &settings.consent, geo: geo_info, - ec_id: None, - kv_store: None, }); log::info!( diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 32a1c2a85..f37f95e4f 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1504,8 +1504,8 @@ impl PrebidAuctionProvider { // Build user object — populate consent at both OpenRTB 2.6 top-level // and Prebid ext-based locations (dual placement). // In cookies_only mode, cookie-sourced consent travels through the - // forwarded Cookie header. KV/policy-sourced consent has no inbound - // cookie to forward, so carry it in the OpenRTB body instead. + // forwarded Cookie header. Policy-sourced consent has no inbound cookie + // to forward, so carry it in the OpenRTB body instead. let consent_ctx = request.user.consent.as_ref().filter(|ctx| { self.config.consent_forwarding.includes_body_consent() || !matches!(ctx.source, crate::consent::ConsentSource::Cookie) @@ -3830,16 +3830,16 @@ external_bundle_sri = "sha384-AAAA" } #[test] - fn to_openrtb_includes_kv_consent_when_cookies_only_has_no_cookie_to_forward() { + fn to_openrtb_includes_policy_default_consent_when_cookies_only_has_no_cookie_to_forward() { let mut config = base_config(); config.consent_forwarding = ConsentForwardingMode::CookiesOnly; let provider = PrebidAuctionProvider::new(config); let mut auction_request = create_test_auction_request(); auction_request.user.consent = Some(ConsentContext { - raw_tc_string: Some("BOkv-backed-consent-string".to_string()), + raw_tc_string: Some("BOpolicy-consent-string".to_string()), raw_us_privacy: Some("1YNN".to_string()), gdpr_applies: true, - source: ConsentSource::KvStore, + source: ConsentSource::PolicyDefault, ..Default::default() }); @@ -3860,15 +3860,15 @@ external_bundle_sri = "sha384-AAAA" assert_eq!( openrtb.user.as_ref().and_then(|u| u.consent.as_deref()), - Some("BOkv-backed-consent-string"), - "cookies_only should fall back to body consent when consent came from KV" + Some("BOpolicy-consent-string"), + "cookies_only should carry policy-sourced consent in the body" ); let regs = openrtb.regs.as_ref().expect("should include consent regs"); assert_eq!(regs.gdpr, Some(true), "should carry GDPR applicability"); assert_eq!( regs.us_privacy.as_deref(), Some("1YNN"), - "should carry non-cookie consent strings from KV" + "should carry policy-sourced consent strings" ); } diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 70a4d6cfd..468bae8d9 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -65,7 +65,6 @@ pub mod rsc_flight; pub(crate) mod s3_sigv4; pub mod settings; pub mod settings_data; -pub mod storage; pub mod streaming_processor; pub mod streaming_replacer; pub mod test_support; diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 24c899f39..7d50d9643 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -212,8 +212,6 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("s3_sigv4.rs", include_str!("s3_sigv4.rs")), ("settings.rs", include_str!("settings.rs")), ("settings_data.rs", include_str!("settings_data.rs")), - ("storage/kv_store.rs", include_str!("storage/kv_store.rs")), - ("storage/mod.rs", include_str!("storage/mod.rs")), ( "streaming_processor.rs", include_str!("streaming_processor.rs"), diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 9cbb2a546..7ddf16b63 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2659,6 +2659,38 @@ mod tests { ); } + #[test] + fn settings_rejects_removed_consent_store_toml() { + let toml = format!( + "{}\n[consent]\nconsent_store = \"legacy-consent-store\"\n", + crate_test_settings_str() + ); + + let err = Settings::from_toml(&toml) + .expect_err("should reject the removed consent_store TOML field"); + + assert!( + format!("{err:?}").contains("consent_store"), + "should identify the removed field: {err:?}" + ); + } + + #[test] + fn settings_rejects_removed_consent_store_json() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should parse baseline settings"); + let mut value = serde_json::to_value(settings).expect("should serialize baseline settings"); + value["consent"]["consent_store"] = json!("legacy-consent-store"); + + let err = Settings::from_json_value(value) + .expect_err("should reject the removed consent_store JSON field"); + + assert!( + format!("{err:?}").contains("consent_store"), + "should identify the removed field: {err:?}" + ); + } + #[test] fn test_settings_from_valid_toml() { let toml_str = crate_test_settings_str(); diff --git a/crates/trusted-server-core/src/storage/kv_store.rs b/crates/trusted-server-core/src/storage/kv_store.rs deleted file mode 100644 index cf8665b2a..000000000 --- a/crates/trusted-server-core/src/storage/kv_store.rs +++ /dev/null @@ -1,570 +0,0 @@ -//! KV Store consent persistence. -//! -//! Stores and retrieves consent data from a KV Store, keyed by EC ID. This -//! provides consent continuity for returning users whose browsers may not -//! have consent cookies on every request. -//! -//! # Storage layout -//! -//! Each entry uses a single JSON body ([`KvConsentEntry`]) containing the raw -//! consent strings, context flags, and a fingerprint for write-on-change -//! detection. -//! -//! # Change detection -//! -//! Writes only occur when consent signals have actually changed. -//! [`consent_fingerprint`] hashes the raw strings into a compact fingerprint -//! stored in the body's `fp` field. On the next request, the existing -//! fingerprint is compared before writing. - -use bytes::Bytes; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; - -use crate::consent::jurisdiction::Jurisdiction; -use crate::consent::types::{ConsentContext, ConsentSource}; -use crate::platform::PlatformKvStore; - -// --------------------------------------------------------------------------- -// KV body (JSON, stored as value) -// --------------------------------------------------------------------------- - -/// Consent data stored in the KV Store body. -/// -/// Contains the raw consent strings needed to reconstruct a [`ConsentContext`]. -/// Decoded data (TCF, GPP, US Privacy) is not stored — it is re-decoded on -/// read to avoid stale decoded state. -/// -/// The `fp` field holds the consent fingerprint for write-on-change detection. -/// Entries written before PR5 lack this field; `#[serde(default)]` treats them -/// as having an empty fingerprint, which always triggers a self-healing -/// re-write. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct KvConsentEntry { - /// Fingerprint of consent signals for write-on-change detection. - /// - /// Written by [`save_consent_to_kv`]. Entries written before PR5 lack - /// this field; `#[serde(default)]` treats them as having an empty - /// fingerprint, which always triggers a self-healing re-write. - #[serde(default)] - pub fp: String, - /// Raw TC String from `euconsent-v2` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_tc_string: Option, - /// Raw GPP string from `__gpp` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_gpp_string: Option, - /// GPP section IDs (decoded or from `__gpp_sid` cookie). - #[serde(skip_serializing_if = "Option::is_none")] - pub gpp_section_ids: Option>, - /// Raw US Privacy string from `us_privacy` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_us_privacy: Option, - /// Raw Google Additional Consent (AC) string. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_ac_string: Option, - - /// Whether GDPR applies to this request. - pub gdpr_applies: bool, - /// Global Privacy Control signal. - pub gpc: bool, - /// Serialized jurisdiction (e.g. `"GDPR"`, `"US-CA"`, `"unknown"`). - pub jurisdiction: String, - - /// When this entry was stored (deciseconds since Unix epoch). - pub stored_at_ds: u64, -} - -// --------------------------------------------------------------------------- -// Conversions -// --------------------------------------------------------------------------- - -/// Builds a [`KvConsentEntry`] from a [`ConsentContext`]. -/// -/// Captures only the raw strings and contextual flags. Decoded data is -/// intentionally omitted — it will be re-decoded on read. The `fp` field is -/// initialized to an empty string and must be set by the caller before writing. -#[must_use] -pub fn entry_from_context(ctx: &ConsentContext, now_ds: u64) -> KvConsentEntry { - KvConsentEntry { - fp: String::new(), - raw_tc_string: ctx.raw_tc_string.clone(), - raw_gpp_string: ctx.raw_gpp_string.clone(), - gpp_section_ids: ctx.gpp_section_ids.clone(), - raw_us_privacy: ctx.raw_us_privacy.clone(), - raw_ac_string: ctx.raw_ac_string.clone(), - gdpr_applies: ctx.gdpr_applies, - gpc: ctx.gpc, - jurisdiction: ctx.jurisdiction.to_string(), - stored_at_ds: now_ds, - } -} - -/// Converts a [`KvConsentEntry`] into [`crate::consent::types::RawConsentSignals`] -/// suitable for re-decoding via [`crate::consent::build_context_from_signals`]. -#[must_use] -pub fn signals_from_entry(entry: &KvConsentEntry) -> crate::consent::types::RawConsentSignals { - crate::consent::types::RawConsentSignals { - raw_tc_string: entry.raw_tc_string.clone(), - raw_gpp_string: entry.raw_gpp_string.clone(), - raw_gpp_sid: entry.gpp_section_ids.as_ref().map(|ids| { - ids.iter() - .map(ToString::to_string) - .collect::>() - .join(",") - }), - raw_us_privacy: entry.raw_us_privacy.clone(), - gpc: entry.gpc, - } -} - -/// Reconstructs a [`ConsentContext`] from a KV Store entry. -/// -/// Re-decodes the raw strings to populate structured fields (TCF, GPP, US -/// Privacy). The `source` is set to [`ConsentSource::KvStore`] and the -/// `jurisdiction` is parsed from the stored string representation. -#[must_use] -pub fn context_from_entry(entry: &KvConsentEntry) -> ConsentContext { - let signals = signals_from_entry(entry); - let mut ctx = crate::consent::build_context_from_signals(&signals); - - // Restore context fields that aren't derived from raw signals. - ctx.gdpr_applies = entry.gdpr_applies; - ctx.gpc = entry.gpc; - ctx.raw_ac_string = entry.raw_ac_string.clone(); - ctx.jurisdiction = parse_jurisdiction(&entry.jurisdiction); - ctx.source = ConsentSource::KvStore; - - ctx -} - -// --------------------------------------------------------------------------- -// Fingerprinting -// --------------------------------------------------------------------------- - -/// Computes a compact fingerprint of the consent signals for change detection. -/// -/// Returns the first 16 hex characters of a SHA-256 hash computed over all -/// raw consent strings and the GPC flag. This is sufficient for detecting -/// changes without storing full hashes. -#[must_use] -pub fn consent_fingerprint(ctx: &ConsentContext) -> String { - let mut hasher = Sha256::new(); - - // Feed each signal into the hash, separated by a sentinel byte to - // prevent ambiguity (e.g., None+Some("x") vs Some("x")+None). - hash_optional(&mut hasher, ctx.raw_tc_string.as_deref()); - hash_optional(&mut hasher, ctx.raw_gpp_string.as_deref()); - hash_optional(&mut hasher, ctx.raw_us_privacy.as_deref()); - hash_optional(&mut hasher, ctx.raw_ac_string.as_deref()); - hasher.update(if ctx.gpc { b"1" } else { b"0" }); - - // Include GPP section IDs so SID-only changes trigger a KV write. - if let Some(sids) = &ctx.gpp_section_ids { - let mut sorted = sids.clone(); - sorted.sort_unstable(); - for sid in &sorted { - hasher.update(sid.to_string().as_bytes()); - hasher.update(b"\xFF"); - } - } else { - hasher.update(b"\x00"); - } - - let result = hasher.finalize(); - hex::encode(&result[..8]) // 16 hex chars = 8 bytes = 64 bits -} - -/// Feeds an optional string into the hasher with sentinel bytes. -fn hash_optional(hasher: &mut Sha256, value: Option<&str>) { - match value { - Some(s) => { - hasher.update(b"\x01"); - hasher.update(s.as_bytes()); - } - None => hasher.update(b"\x00"), - } -} - -/// Parses a jurisdiction string back into a [`Jurisdiction`] enum. -fn parse_jurisdiction(s: &str) -> Jurisdiction { - match s { - "GDPR" => Jurisdiction::Gdpr, - "non-regulated" => Jurisdiction::NonRegulated, - "unknown" => Jurisdiction::Unknown, - s if s.starts_with("US-") => Jurisdiction::UsState(s[3..].to_owned()), - _ => Jurisdiction::Unknown, - } -} - -// --------------------------------------------------------------------------- -// KV Store operations -// --------------------------------------------------------------------------- - -/// Checks whether the stored consent fingerprint matches the current one. -/// -/// Returns `true` when the stored body's `fp` field equals `new_fp`, meaning -/// no write is needed. Returns `false` when the key is absent, the body -/// cannot be deserialized, or the fingerprint differs. -/// -/// Entries written before PR5 have an empty `fp` (via `#[serde(default)]`), -/// which never matches a computed fingerprint and triggers a self-healing -/// re-write. -fn fingerprint_unchanged(store: &dyn PlatformKvStore, key: &str, new_fp: &str) -> bool { - let bytes = match futures::executor::block_on(store.get_bytes(key)) { - Ok(Some(bytes)) => bytes, - _ => return false, - }; - - serde_json::from_slice::(&bytes) - .map(|entry| entry.fp == new_fp) - .unwrap_or(false) -} - -/// Loads consent data from the KV store for a given EC ID. -/// -/// Returns `Some(ConsentContext)` if a valid entry is found, [`None`] if the -/// key does not exist or deserialization fails. Errors are logged but never -/// propagated — KV failures must not break the request pipeline. -/// -/// # Arguments -/// -/// * `store` — KV store opened by the adapter. -/// * `ec_id` — Edge Cookie ID used as the KV key. -#[must_use] -pub fn load_consent_from_kv(store: &dyn PlatformKvStore, ec_id: &str) -> Option { - let bytes = match futures::executor::block_on(store.get_bytes(ec_id)) { - Ok(Some(bytes)) => bytes, - Ok(None) => { - log::debug!("Consent KV lookup miss"); - return None; - } - Err(e) => { - log::debug!("Consent KV lookup error: {e}"); - return None; - } - }; - - match serde_json::from_slice::(&bytes) { - Ok(entry) => { - log::info!( - "Loaded consent from KV store (stored_at_ds={})", - entry.stored_at_ds - ); - Some(context_from_entry(&entry)) - } - Err(e) => { - log::warn!("Failed to deserialize consent KV entry: {e}"); - None - } - } -} - -/// Saves consent data to the KV store, writing only when signals have changed. -/// -/// Compares the fingerprint of current consent signals against the fingerprint -/// embedded in the stored entry. If they match, the write is skipped. -/// The fingerprint is embedded in the body so no KV metadata is required. -/// -/// # Arguments -/// -/// * `store` — KV store opened by the adapter. -/// * `ec_id` — Edge Cookie ID used as the KV key. -/// * `ctx` — Current request's consent context. -/// * `max_age_days` — TTL for the entry, matching `max_consent_age_days`. -pub fn save_consent_to_kv( - store: &dyn PlatformKvStore, - ec_id: &str, - ctx: &ConsentContext, - max_age_days: u32, -) { - if ctx.is_empty() { - log::debug!("Skipping consent KV write: consent is empty"); - return; - } - - let fp = consent_fingerprint(ctx); - - if fingerprint_unchanged(store, ec_id, &fp) { - log::debug!("Consent unchanged; skipping write"); - return; - } - - let mut entry = entry_from_context(ctx, crate::consent::now_deciseconds()); - entry.fp = fp.clone(); - - let body = match serde_json::to_vec(&entry) { - Ok(body) => Bytes::from(body), - Err(e) => { - log::warn!("Failed to serialize consent entry: {e}"); - return; - } - }; - - let ttl = std::time::Duration::from_secs(u64::from(max_age_days) * 86_400); - - match futures::executor::block_on(store.put_bytes_with_ttl(ec_id, body, ttl)) { - Ok(()) => { - log::info!("Saved consent to KV store (ttl={max_age_days}d)"); - } - Err(e) => { - log::warn!("Failed to write consent to KV store: {e}"); - } - } -} - -/// Deletes a consent entry from the KV store for a given EC ID. -/// -/// Used when a user revokes consent — the existing EC cookie is being -/// expired, so the persisted consent data must also be removed. -/// -/// Errors are logged but never propagated — KV failures must not -/// break the request pipeline. -pub fn delete_consent_from_kv(store: &dyn PlatformKvStore, ec_id: &str) { - match futures::executor::block_on(store.delete(ec_id)) { - Ok(()) => { - log::info!("Deleted consent KV entry (consent revoked)"); - } - Err(e) => { - log::warn!("Failed to delete consent KV entry: {e}"); - } - } -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -fn make_test_context() -> ConsentContext { - ConsentContext { - raw_tc_string: Some("CPXxGfAPXxGfA".to_owned()), - raw_gpp_string: Some("DBACNYA~CPXxGfA".to_owned()), - gpp_section_ids: Some(vec![2, 6]), - raw_us_privacy: Some("1YNN".to_owned()), - raw_ac_string: None, - gdpr_applies: true, - tcf: None, - gpp: None, - us_privacy: None, - expired: false, - gpc: false, - jurisdiction: Jurisdiction::Gdpr, - source: ConsentSource::Cookie, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::consent::jurisdiction::Jurisdiction; - use crate::consent::types::{ConsentContext, ConsentSource}; - - #[test] - fn entry_roundtrip() { - let ctx = make_test_context(); - let entry = entry_from_context(&ctx, 1_000_000); - let json = serde_json::to_string(&entry).expect("should serialize"); - let restored: KvConsentEntry = serde_json::from_str(&json).expect("should deserialize"); - - assert_eq!(restored.raw_tc_string, ctx.raw_tc_string); - assert_eq!(restored.raw_gpp_string, ctx.raw_gpp_string); - assert_eq!(restored.gpp_section_ids, ctx.gpp_section_ids); - assert_eq!(restored.raw_us_privacy, ctx.raw_us_privacy); - assert_eq!(restored.gdpr_applies, ctx.gdpr_applies); - assert_eq!(restored.gpc, ctx.gpc); - assert_eq!(restored.jurisdiction, "GDPR"); - assert_eq!(restored.stored_at_ds, 1_000_000); - } - - #[test] - fn kv_consent_entry_roundtrip_preserves_fp() { - let ctx = make_test_context(); - let fp = consent_fingerprint(&ctx); - let mut entry = entry_from_context(&ctx, 1_000_000); - entry.fp = fp.clone(); - let json = serde_json::to_string(&entry).expect("should serialize"); - let restored: KvConsentEntry = serde_json::from_str(&json).expect("should deserialize"); - - assert_eq!( - restored.fp, fp, - "should preserve fingerprint through roundtrip" - ); - } - - #[test] - fn entry_fits_in_2000_bytes() { - let ctx = make_test_context(); - let mut entry = entry_from_context(&ctx, 1_000_000); - entry.fp = consent_fingerprint(&ctx); - let json = serde_json::to_string(&entry).expect("should serialize"); - assert!( - json.len() <= 2000, - "entry JSON must fit in 2000 bytes, was {} bytes", - json.len() - ); - } - - #[test] - fn context_roundtrip_via_entry() { - let original = make_test_context(); - let entry = entry_from_context(&original, 1_000_000); - let restored = context_from_entry(&entry); - - assert_eq!(restored.raw_tc_string, original.raw_tc_string); - assert_eq!(restored.raw_gpp_string, original.raw_gpp_string); - assert_eq!(restored.raw_us_privacy, original.raw_us_privacy); - assert_eq!(restored.gdpr_applies, original.gdpr_applies); - assert_eq!(restored.gpc, original.gpc); - assert_eq!(restored.jurisdiction, original.jurisdiction); - assert_eq!(restored.source, ConsentSource::KvStore); - } - - #[test] - fn fingerprint_deterministic() { - let ctx = make_test_context(); - let fp1 = consent_fingerprint(&ctx); - let fp2 = consent_fingerprint(&ctx); - assert_eq!(fp1, fp2, "fingerprint should be deterministic"); - assert_eq!(fp1.len(), 16, "fingerprint should be 16 hex chars"); - } - - #[test] - fn fingerprint_changes_with_different_signals() { - let ctx1 = make_test_context(); - let mut ctx2 = make_test_context(); - ctx2.raw_tc_string = Some("DIFFERENT_TC_STRING".to_owned()); - - assert_ne!( - consent_fingerprint(&ctx1), - consent_fingerprint(&ctx2), - "different TC strings should produce different fingerprints" - ); - } - - #[test] - fn fingerprint_changes_with_gpc() { - let mut ctx1 = make_test_context(); - ctx1.gpc = false; - let mut ctx2 = make_test_context(); - ctx2.gpc = true; - - assert_ne!( - consent_fingerprint(&ctx1), - consent_fingerprint(&ctx2), - "different GPC values should produce different fingerprints" - ); - } - - #[test] - fn fingerprint_distinguishes_none_from_empty() { - let mut ctx_none = make_test_context(); - ctx_none.raw_tc_string = None; - let mut ctx_empty = make_test_context(); - ctx_empty.raw_tc_string = Some(String::new()); - - assert_ne!( - consent_fingerprint(&ctx_none), - consent_fingerprint(&ctx_empty), - "None vs empty string should produce different fingerprints" - ); - } - - #[test] - fn signals_from_entry_roundtrip() { - let ctx = make_test_context(); - let entry = entry_from_context(&ctx, 1_000_000); - let signals = signals_from_entry(&entry); - - assert_eq!(signals.raw_tc_string, ctx.raw_tc_string); - assert_eq!(signals.raw_gpp_string, ctx.raw_gpp_string); - assert_eq!(signals.raw_us_privacy, ctx.raw_us_privacy); - assert_eq!(signals.gpc, ctx.gpc); - // gpp_sid is serialized as "2,6" from the section IDs - assert_eq!(signals.raw_gpp_sid, Some("2,6".to_owned())); - } - - #[test] - fn parse_jurisdiction_roundtrip() { - assert_eq!(parse_jurisdiction("GDPR"), Jurisdiction::Gdpr); - assert_eq!( - parse_jurisdiction("US-CA"), - Jurisdiction::UsState("CA".to_owned()) - ); - assert_eq!( - parse_jurisdiction("non-regulated"), - Jurisdiction::NonRegulated - ); - assert_eq!(parse_jurisdiction("unknown"), Jurisdiction::Unknown); - assert_eq!( - parse_jurisdiction("something-else"), - Jurisdiction::Unknown, - "unrecognized jurisdiction should default to Unknown" - ); - } - - #[test] - fn empty_entry_serializes_compact() { - let ctx = ConsentContext::default(); - let entry = entry_from_context(&ctx, 0); - let json = serde_json::to_string(&entry).expect("should serialize"); - // With skip_serializing_if = "Option::is_none", omitted fields keep it small. - assert!( - !json.contains("raw_tc_string"), - "None fields should be omitted from JSON" - ); - } - - #[test] - fn entry_preserves_ac_string() { - let mut ctx = make_test_context(); - ctx.raw_ac_string = Some("2~1234.5678~dv.".to_owned()); - let entry = entry_from_context(&ctx, 0); - let restored = context_from_entry(&entry); - - assert_eq!( - restored.raw_ac_string, - Some("2~1234.5678~dv.".to_owned()), - "AC string should survive roundtrip" - ); - } -} - -#[cfg(test)] -mod new_api_tests { - use super::*; - use edgezero_core::key_value_store::NoopKvStore; - - fn noop() -> NoopKvStore { - NoopKvStore - } - - #[test] - fn load_returns_none_when_key_absent() { - let result = load_consent_from_kv(&noop(), "some-ec-id"); - assert!(result.is_none(), "should return None when key is absent"); - } - - #[test] - fn save_does_not_panic_with_noop_store() { - let ctx = make_test_context(); - save_consent_to_kv(&noop(), "some-ec-id", &ctx, 30); - } - - #[test] - fn delete_does_not_panic_with_noop_store() { - delete_consent_from_kv(&noop(), "some-ec-id"); - } - - #[test] - fn kv_consent_entry_missing_fp_deserialises_as_empty() { - let json = r#"{"gdpr_applies":true,"gpc":false,"jurisdiction":"GDPR","stored_at_ds":0}"#; - let entry: KvConsentEntry = - serde_json::from_str(json).expect("should deserialize legacy entry"); - assert_eq!( - entry.fp, - String::new(), - "should default fp to empty string for legacy entries" - ); - } -} diff --git a/crates/trusted-server-core/src/storage/mod.rs b/crates/trusted-server-core/src/storage/mod.rs deleted file mode 100644 index 0c6998b6d..000000000 --- a/crates/trusted-server-core/src/storage/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod kv_store; diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index b975590c6..705586fdb 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -378,7 +378,13 @@ TRUSTED_SERVER__TESTER_COOKIE__ENABLED=true ## EC Configuration -Settings for generating privacy-preserving Edge Cookie identifiers. The `ec_store` KV store is the only KV-backed EC lifecycle store; it holds identity graph state, minimal consent metadata, source-domain keyed partner UIDs, and withdrawal tombstones. Consent configuration controls request-local interpretation and forwarding, not separate KV persistence. +Settings for generating privacy-preserving Edge Cookie identifiers. The `ec_store` KV store is the only KV-backed EC lifecycle store; it holds identity graph state, minimal consent metadata, source-domain keyed partner UIDs, and withdrawal tombstones. Live consent is interpreted from request cookies, headers, geolocation, and policy defaults, not separate KV persistence. + +### Migrating from `consent_store` + +The legacy `[consent].consent_store` setting has been removed. Trusted Server uses a strict configuration schema, so TOML and JSON/app-config that still contain `consent_store` fail during configuration loading. Remove the field before upgrading; it is not accepted as an ignored or deprecated option. + +Legacy consent-store records are not read or migrated into `ec.ec_store`. Their payload schema is not authoritative EC lifecycle state, so do not copy those records into the identity store. You may retain the old store unchanged for a defined rollback window, then unlink its platform resource binding and delete it. No browser-cookie or EC identity-store migration is required. ### `[ec]` diff --git a/docs/guide/fastly.md b/docs/guide/fastly.md index 20faf1995..8d6b6f4b6 100644 --- a/docs/guide/fastly.md +++ b/docs/guide/fastly.md @@ -128,7 +128,9 @@ Verify stores are linked to your active service version: fastly resource-link list --service-id --version ``` -If EC sync returns `kv_unavailable` or identify responses are degraded, first check that the identity store is present and linked to the active version. Legacy partner/consent KV bindings can be removed once no deployment-specific tooling depends on them. +If EC sync returns `kv_unavailable` or identify responses are degraded, first check that the identity store is present and linked to the active version. + +Before upgrading a deployment that used the legacy consent store, remove `[consent].consent_store` from TOML or JSON/app-config; strict configuration loading rejects the removed field. Remove its local Viceroy fixture and active Fastly resource link as well. The old records are not read or migrated into `ec.ec_store` and must not be copied there. You may retain the old store unchanged for a defined rollback window, then delete the store and its records. ## Next Steps diff --git a/docs/superpowers/plans/2026-07-13-issue-883-remove-legacy-consent-store.md b/docs/superpowers/plans/2026-07-13-issue-883-remove-legacy-consent-store.md new file mode 100644 index 000000000..824073c10 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-issue-883-remove-legacy-consent-store.md @@ -0,0 +1,306 @@ +# Issue #883 — Remove Legacy Consent Store Implementation Plan + +## Metadata + +- **Issue:** [#883 — Complete removal of the legacy consent KV persistence path](https://github.com/IABTechLab/trusted-server/issues/883) +- **Stack base:** Draft PR #902, branch `perf/group-batch-sync-by-ec-id` +- **Implementation branch:** `refactor/remove-legacy-consent-store` +- **Status:** Implemented and verified +- **Date:** 2026-07-13 + +## Goal + +Completely remove the inactive legacy consent KV persistence path without wiring +new persistence onto the request hot path. Live consent remains request-local; +the EC identity graph remains the only KV-backed EC lifecycle store. + +## Approved Behavioral and Migration Contract + +1. Live consent is interpreted on each request from cookies, headers, + geolocation, and publisher policy defaults. +2. `ec.ec_store` remains authoritative for EC identity graph state, minimal + consent metadata, source-domain partner IDs, and withdrawal tombstones. It + does not become a store for the deleted legacy consent payload schema. +3. `consent_store` is removed from `ConsentConfig`. Because the configuration + schema is strict, legacy TOML and JSON/app-config containing that field fail + during deserialization. There is no ignored field, alias, warning-only + compatibility path, or silent migration. +4. Operators must remove `consent_store` before upgrading. Legacy records are + not read or copied into `ec.ec_store`; an old binding/store may be retained + for a short rollback window and then unlinked and deleted. +5. Auction, page-bids, and publisher routes no longer open an otherwise-unused + consent KV store. They continue using the request's existing + `RuntimeServices` and `EcContext`. +6. Existing consent gating, consent forwarding, EC creation/withdrawal, + tombstone behavior, authentication, routing, and wire contracts remain + unchanged. +7. Generic `PlatformKvStore`, the generic `RuntimeServices` KV slot, adapter KV + implementations, and EC-specific KV primitives remain in place; they are + platform infrastructure outside this issue. + +## Current State + +- `EcContext` always calls `build_consent_context` with `ec_id: None` and + `kv_store: None`, so the fallback/write path and all consent storage helpers + are unreachable in production. +- Fastly's EdgeZero path still opens `settings.consent.consent_store` for + auction, page-bids, and publisher routes and returns `503` if it cannot be + opened, even though the returned store is never consumed by consent logic. +- `ConsentConfig` still accepts the misleading field, and local Fastly config + still provisions a placeholder store. +- The dead storage module, `ConsentSource::KvStore`, adapter comments, and tests + continue to advertise persisted consent continuity that runtime requests do + not have. + +## File Map + +### Modify + +- `crates/trusted-server-core/src/settings.rs` + - Add full-settings TOML and runtime JSON rejection coverage for the removed field. +- `crates/trusted-server-core/src/consent_config.rs` + - Remove the field and default. +- `crates/trusted-server-core/src/consent/mod.rs` + - Remove KV pipeline inputs, branches, re-export, tests, and stale docs. +- `crates/trusted-server-core/src/consent/types.rs` + - Remove `ConsentSource::KvStore` and update source documentation. +- `crates/trusted-server-core/src/ec/mod.rs` + - Remove obsolete `None` pipeline fields. +- `crates/trusted-server-core/src/integrations/prebid.rs` + - Preserve non-cookie consent forwarding coverage using `PolicyDefault`. +- `crates/trusted-server-core/src/lib.rs` + - Remove the deleted storage module export. +- `crates/trusted-server-core/src/migration_guards.rs` + - Remove deleted storage files from the source include list. +- `crates/trusted-server-adapter-fastly/src/app.rs` + - Remove the consent-route service resolver and obsolete tests; pass existing + services directly. +- `crates/trusted-server-adapter-fastly/src/platform.rs` + - Remove the now-unused named generic KV opener and imports. +- `crates/trusted-server-adapter-spin/src/platform.rs` + - Remove stale consent-specific claims from generic TTL documentation. +- `crates/trusted-server-adapter-axum/src/platform.rs` + - Remove stale consent-specific claims from generic unavailable-KV messaging. +- `fastly.toml` + - Remove the local `consent_store` fixture. +- `docs/guide/configuration.md` + - Document strict migration and authoritative sources. +- `docs/guide/fastly.md` + - Document field/binding removal and rollback cleanup. + +### Delete + +- `crates/trusted-server-core/src/storage/kv_store.rs` + - Dead consent persistence implementation and tests. +- `crates/trusted-server-core/src/storage/mod.rs` + - Empty after deleting the only child module. + +### Add + +- `docs/superpowers/plans/2026-07-13-issue-883-remove-legacy-consent-store.md` + - Record the reviewed removal and verification contract. + +No dependency, EC KV schema, request/response schema, or adapter routing change +is expected. + +## Implementation Tasks + +### Task 1 — Establish fail-fast migration tests + +- [x] Add `settings_rejects_removed_consent_store_toml` using + `crate_test_settings_str()` plus `[consent] consent_store = ...`. +- [x] Assert `Settings::from_toml` fails during deserialization and its diagnostic + identifies `consent_store`. +- [x] Add `settings_rejects_removed_consent_store_json` by serializing valid + settings, injecting the legacy nested field, and calling + `Settings::from_json_value`; assert the same field-specific diagnostic on + the runtime app-config deserialization path. +- [x] Run both tests before removing the field and confirm they fail because the + legacy field is still accepted. +- [x] Preserve `#[serde(deny_unknown_fields)]`; do not add compatibility aliases. + +### Task 2 — Remove active configuration and request-pipeline plumbing + +- [x] Remove `ConsentConfig::consent_store`, its documentation, serde attributes, + and default initialization. +- [x] Remove the consent storage re-export from `consent`. +- [x] Remove `ec_id` and `kv_store` from `ConsentPipelineInput` and every struct + literal. +- [x] Remove the empty-signal KV fallback and write-on-change branch. +- [x] Update module/entry-point documentation to describe request-local consent + as the only live source. +- [x] Delete the consent-only in-memory KV test double and three persistence/ + fallback tests while retaining all request-local gating tests. + +### Task 3 — Delete dead storage and source metadata + +- [x] Delete `storage/kv_store.rs` and `storage/mod.rs`. +- [x] Remove `pub mod storage` and the two migration-guard `include_str!` entries. +- [x] Remove `ConsentSource::KvStore` and update type-level documentation. +- [x] Adapt the Prebid cookies-only regression to `PolicyDefault`, preserving the + Cookie-versus-non-Cookie forwarding distinction. +- [x] Confirm no production EC identity graph or generic platform KV code changed. + +### Task 4 — Remove Fastly's no-value route dependency + +- [x] Delete `runtime_services_for_consent_route` and its `open_kv_store` call. +- [x] Pass the existing request `RuntimeServices` directly to auction, page-bids, + publisher handling, and publisher response buffering. +- [x] Remove comments claiming those routes require separate consent KV. +- [x] Delete `settings_with_missing_consent_store` and the two obsolete route + failure tests. +- [x] Remove test/production imports made unused by the deletion. +- [x] Delete the now-unused Fastly `open_kv_store` wrapper and its imports. +- [x] Keep `AppState::default_kv_store` and generic runtime KV infrastructure. + +### Task 5 — Remove fixtures and stale active commentary + +- [x] Remove `[[local_server.kv_stores.consent_store]]` from `fastly.toml`. +- [x] Rewrite Spin's generic TTL documentation without claims about consent + persistence/fallback. +- [x] Rewrite Axum's unavailable generic-KV documentation/log without claims + about consent routes. +- [x] Confirm no active example or manifest provisions the deleted setting. + +### Task 6 — Document migration and authoritative sources + +- [x] State that stale TOML and JSON/app-config fail fast and the field must be + removed before upgrade. +- [x] Tell Fastly operators to remove the resource binding/local fixture and to + retain legacy data only for rollback before deletion. +- [x] State that legacy consent records are not read or migrated and must not be + copied into `ec.ec_store`. +- [x] Reaffirm request-local live consent and the identity/tombstone role of + `ec.ec_store`. +- [x] Leave explicitly historical/superseded plans and specs unchanged. + +### Task 7 — Residual audit, review, and full verification + +- [x] Search active source/manifests for `consent_store`, + `ConsentSource::KvStore`, `storage::kv_store`, + `runtime_services_for_consent_route`, and `open_kv_store`. +- [x] Allow `consent_store` only in strict rejection tests, migration guidance, + the new plan, and explicitly historical records. +- [x] Run independent scope/correctness and migration/test-quality reviews. +- [x] Apply only issue-scoped fixes. +- [x] Mark this plan implemented only after every verification check passes. + +## Acceptance Mapping + +| Issue requirement | Planned evidence | +| ------------------------------------------- | ---------------------------------------------------------------- | +| Remove active `consent_store` configuration | Field/default deletion plus strict TOML and JSON tests | +| Remove pipeline KV fields and branches | Simplified input/callers and no persistence code | +| Remove dead storage implementation | Deleted storage files, export, and guard entries | +| Remove unused consent source | Deleted enum variant; policy-source Prebid regression retained | +| Remove Fastly route dependency | Resolver/opener deletion and direct service passing | +| Remove manifests/examples | Fastly local fixture deletion and residual audit | +| Preserve consent/tombstone behavior | Existing consent, EC, finalization, and adapter suites | +| Document authoritative sources | Updated configuration and Fastly guides | +| Document migration behavior | Explicit fail-fast, rollback, and no-data-migration instructions | + +## Migration Details + +Before deploying this version, operators with the old setting must: + +1. Remove `consent_store` from `[consent]` in TOML or from the equivalent JSON/ + app-config object. Startup/config loading intentionally fails while it is + present. +2. Remove the legacy Fastly resource link and local Viceroy fixture when no + rollback depends on them. +3. Do not copy old consent payloads into `ec.ec_store`; the schemas and + authority differ. +4. Optionally retain the old store unchanged during a defined rollback window, + then delete it and its records. This release neither reads nor transforms + those records. + +No browser cookie migration is required. Request-local signals continue to be +interpreted normally, and existing EC identity/tombstone entries are unchanged. + +## Focused Verification + +```bash +cargo test-fastly removed_consent_store +cargo test-fastly settings_rejects_removed_consent_store_json +cargo test-fastly missing_geo_keeps_unknown_jurisdiction +cargo test-fastly to_openrtb_includes_policy_default +cargo test-fastly dispatch +cargo fmt --all -- --check + +rg -n 'ConsentSource::KvStore|storage::kv_store|runtime_services_for_consent_route|open_kv_store' crates fastly.toml trusted-server.example.toml +rg -n 'consent_store' crates fastly.toml trusted-server.example.toml docs/guide +``` + +Expected residual `consent_store` matches are limited to rejection tests and +migration documentation. + +## Full Verification Contract + +```bash +cargo fmt --all -- --check + +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +./scripts/test-cli.sh + +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm + +cd crates/trusted-server-js/lib +npx vitest run +npm run format +cd ../../.. + +cd docs +npm run format +cd .. + +cargo doc --package trusted-server-core --no-deps --all-features --target wasm32-wasip1 +cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 +git diff --check +``` + +## Risks and Mitigations + +- **Intentional breaking config change:** field-specific TOML and JSON tests plus + prominent migration guidance prevent silent drift. +- **Fastly service-threading regression:** pass the exact existing `services` + reference to both publisher handling and response buffering; run all route and + adapter tests. +- **Consent forwarding regression:** retain the non-cookie Prebid test with + `PolicyDefault` and all request-local consent tests. +- **Accidental EC KV removal:** do not alter `EcKvStore`, `KvIdentityGraph`, + `ec.ec_store`, or withdrawal code. +- **Overbroad generic KV cleanup:** keep `PlatformKvStore`, RuntimeServices' KV + slot, and adapter implementations. +- **Misleading residual history:** active code/docs must be clean; explicitly + superseded historical records remain as history and are allowlisted in audits. +- **Out-of-repository API users:** the deleted module/variant are public, but the + core crate is private and `publish = false`; repository callers are audited. + +## Non-Goals + +- No persisted live-consent continuity or new consent data model. +- No migration of legacy consent payloads into the EC identity graph. +- No change to consent interpretation, expiry, conflict resolution, gating, or + forwarding policy. +- No change to EC identity creation, partner IDs, withdrawal, or tombstone TTL. +- No removal of generic platform KV infrastructure. +- No authentication, routing, response, or adapter platform redesign. + +## Review and Verification Record + +- Independent plan review: approved (no blockers; runtime JSON-path and doc-command corrections applied) +- Test-first strict-schema failures: confirmed before field removal +- Focused tests: passed (strict schema, consent, Prebid source, and Fastly dispatch) +- Independent code review: approved (no findings) +- Residual audit: passed; only rejection tests, migration docs, plan, and explicit history remain +- Full cross-adapter verification: passed, including host CLI tests and all-features core docs +- Release WASM build: passed diff --git a/fastly.toml b/fastly.toml index 56002bc5a..88d192223 100644 --- a/fastly.toml +++ b/fastly.toml @@ -40,10 +40,6 @@ build = """ [[local_server.kv_stores.ec_identity_store]] key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01" data = '{"v":1,"created":1700000000,"last_seen":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US"}}' - - [[local_server.kv_stores.consent_store]] - key = "placeholder" - data = "placeholder" [local_server.secret_stores] [[local_server.secret_stores.signing_keys]] key = "ts-2025-10-A"