diff --git a/.github/actions/setup-integration-test-env/action.yml b/.github/actions/setup-integration-test-env/action.yml index 87c59d2fd..a491eb6e1 100644 --- a/.github/actions/setup-integration-test-env/action.yml +++ b/.github/actions/setup-integration-test-env/action.yml @@ -80,7 +80,7 @@ runs: env: TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:${{ inputs.origin-port }} TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret - TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY: integration-test-secret-key + TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 186569da5..1dd8f0323 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,6 +51,14 @@ jobs: - name: Run tests run: cargo test --workspace + - name: Verify Fastly WASM release build + env: + TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:8080 + TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret + TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret + TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" + run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 + test-typescript: name: vitest runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index ec76ee46e..b5e2b6f0a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -366,7 +366,7 @@ both runtime behavior and build/tooling changes. | `crates/trusted-server-core/src/tsjs.rs` | Script tag generation with module IDs | | `crates/trusted-server-core/src/html_processor.rs` | Injects `"#; - let params = OwnedProcessResponseParams { - content_encoding: String::new(), - origin_host: "origin.example.com".to_string(), - origin_url: "https://origin.example.com".to_string(), - request_host: "proxy.example.com".to_string(), - request_scheme: "https".to_string(), - content_type: "text/html".to_string(), - }; - - let mut output = Vec::new(); - stream_publisher_body( - Body::from(html.to_vec()), - &mut output, - ¶ms, - &settings, - ®istry, - ) - .expect("should process RSC push"); - - let processed = String::from_utf8(output).expect("valid UTF-8"); - assert!( - !processed.contains("__ts_rsc_payload_"), - "placeholder must be substituted before reaching output. Got: {processed}" - ); - assert!( - processed.contains("proxy.example.com/page"), - "origin URL must be rewritten in the substituted payload. Got: {processed}" - ); - assert!( - !processed.contains("origin.example.com"), - "origin host must not leak. Got: {processed}" - ); - } } diff --git a/crates/trusted-server-core/src/request_signing/endpoints.rs b/crates/trusted-server-core/src/request_signing/endpoints.rs index 3b14317e5..9b4b957e8 100644 --- a/crates/trusted-server-core/src/request_signing/endpoints.rs +++ b/crates/trusted-server-core/src/request_signing/endpoints.rs @@ -473,7 +473,7 @@ mod tests { #[test] fn test_handle_rotate_key_with_empty_body() { let settings = crate::test_support::tests::create_test_settings(); - let req = Request::new(Method::POST, "https://test.com/admin/keys/rotate"); + let req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/rotate"); let result = handle_rotate_key(&settings, req); match result { @@ -500,7 +500,7 @@ mod tests { }; let body_json = serde_json::to_string(&req_body).expect("should serialize rotate request"); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/rotate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/rotate"); req.set_body(body_json); let result = handle_rotate_key(&settings, req); @@ -522,7 +522,7 @@ mod tests { #[test] fn test_handle_rotate_key_invalid_json() { let settings = crate::test_support::tests::create_test_settings(); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/rotate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/rotate"); req.set_body("invalid json"); let result = handle_rotate_key(&settings, req); @@ -540,7 +540,7 @@ mod tests { let body_json = serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/deactivate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/deactivate"); req.set_body(body_json); let result = handle_deactivate_key(&settings, req); @@ -570,7 +570,7 @@ mod tests { let body_json = serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/deactivate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/deactivate"); req.set_body(body_json); let result = handle_deactivate_key(&settings, req); @@ -592,7 +592,7 @@ mod tests { #[test] fn test_handle_deactivate_key_invalid_json() { let settings = crate::test_support::tests::create_test_settings(); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/deactivate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/deactivate"); req.set_body("invalid json"); let result = handle_deactivate_key(&settings, req); diff --git a/crates/trusted-server-core/src/rsc_flight.rs b/crates/trusted-server-core/src/rsc_flight.rs index 6bd173667..309e95056 100644 --- a/crates/trusted-server-core/src/rsc_flight.rs +++ b/crates/trusted-server-core/src/rsc_flight.rs @@ -1,7 +1,3 @@ -//! RSC flight data processor. -//! -//! See [`crate::platform`] module doc for platform notes. - use std::io; use crate::host_rewrite::rewrite_bare_host_at_boundaries; diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 785492620..c2f8b52ce 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -5,6 +5,7 @@ use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize}; use serde_json::Value as JsonValue; use std::collections::HashMap; use std::ops::{Deref, DerefMut}; +use std::str::FromStr; use std::sync::OnceLock; use url::Url; use validator::{Validate, ValidationError}; @@ -19,7 +20,10 @@ pub const ENVIRONMENT_VARIABLE_SEPARATOR: &str = "__"; #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] pub struct Publisher { + #[validate(custom(function = validate_publisher_domain))] pub domain: String, + /// Domain for non-EC cookies. EC cookies use a separate computed domain + /// (see [`ec_cookie_domain`](Self::ec_cookie_domain)). #[validate(custom(function = validate_cookie_domain))] pub cookie_domain: String, #[validate(custom(function = validate_no_trailing_slash))] @@ -34,6 +38,17 @@ impl Publisher { /// Known placeholder values that must not be used in production. pub const PROXY_SECRET_PLACEHOLDERS: &[&str] = &["change-me-proxy-secret", "proxy-secret"]; + /// Returns the EC cookie domain, computed as `.{domain}`. + /// + /// Per spec §5.2, EC cookies derive their domain from + /// `publisher.domain` — **not** from `publisher.cookie_domain`. + /// This ensures the EC cookie is always scoped to the publisher's + /// apex domain regardless of how `cookie_domain` is configured. + #[must_use] + pub fn ec_cookie_domain(&self) -> String { + format!(".{}", self.domain) + } + /// Returns `true` if `proxy_secret` matches a known placeholder value /// (case-insensitive). #[must_use] @@ -203,35 +218,211 @@ impl DerefMut for IntegrationSettings { } } -/// Edge Cookie configuration. -#[allow(unused)] +/// A partner (SSP, DSP, identity vendor) configured in `[[ec.partners]]`. +/// +/// Partners are defined statically in `trusted-server.toml` rather than +/// registered via API. At startup, each partner's `api_token` is hashed +/// (SHA-256) for O(1) auth lookups; the plaintext is never stored at runtime. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +pub struct EcPartner { + /// Unique partner identifier. Must match `^[a-z0-9_-]{1,32}$` and + /// not collide with reserved IDs (`ec`, `ts`, `eids`, etc.). + #[validate(custom(function = EcPartner::validate_id))] + pub id: String, + /// Human-readable partner name. + pub name: String, + /// `OpenRTB` `source.domain` for EID entries (e.g. `"liveramp.com"`). + pub source_domain: String, + /// `OpenRTB` `atype` value (typically 3). + #[serde( + default = "EcPartner::default_openrtb_atype", + deserialize_with = "from_value_or_str" + )] + pub openrtb_atype: u8, + /// Whether this partner's UIDs appear in auction `user.eids`. + #[serde(default, deserialize_with = "from_value_or_str")] + pub bidstream_enabled: bool, + /// Plaintext API token. Hashed at startup for auth lookups. + /// Used by batch sync (inbound) and identify (inbound). + pub api_token: Redacted, + /// Max batch sync API requests per partner per minute. + #[serde( + default = "EcPartner::default_batch_rate_limit", + deserialize_with = "from_value_or_str" + )] + pub batch_rate_limit: u32, + /// Whether server-to-server pull sync is enabled for this partner. + #[serde(default, deserialize_with = "from_value_or_str")] + pub pull_sync_enabled: bool, + /// URL to call for pull sync. Required when `pull_sync_enabled`. + #[serde(default)] + pub pull_sync_url: Option, + /// Allowlist of domains TS may call for this partner's pull sync. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub pull_sync_allowed_domains: Vec, + /// Legacy pull-sync refresh interval retained for config compatibility. + /// + /// EC identity entries no longer store per-partner sync timestamps, so + /// this value is not used by the current fill-missing-only pull sync + /// behavior. + #[serde( + default = "EcPartner::default_pull_sync_ttl_sec", + deserialize_with = "from_value_or_str" + )] + pub pull_sync_ttl_sec: u64, + /// Max pull sync calls per EC hash per partner per hour. + #[serde( + default = "EcPartner::default_pull_sync_rate_limit", + deserialize_with = "from_value_or_str" + )] + pub pull_sync_rate_limit: u32, + /// Outbound bearer token for pull sync requests. + #[serde(default)] + pub ts_pull_token: Option>, +} + +impl EcPartner { + const RESERVED_IDS: &[&str] = &[ + "ec", + "eids", + "ec-consent", + "eids-truncated", + "synthetic", + "ts", + "version", + "env", + ]; + + /// Validates a partner ID for safe use in dynamic headers and cookies. + /// + /// # Errors + /// + /// Returns a validation error when `id` does not match the configured + /// lowercase identifier policy or collides with a reserved name. + pub fn validate_id(id: &str) -> Result<(), ValidationError> { + if id.is_empty() || id.len() > 32 { + return Err(ValidationError::new("invalid_partner_id_length")); + } + if Self::RESERVED_IDS.contains(&id) { + return Err(ValidationError::new("reserved_partner_id")); + } + if !id.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' || byte == b'-' + }) { + return Err(ValidationError::new("invalid_partner_id")); + } + Ok(()) + } + + #[must_use] + pub const fn default_openrtb_atype() -> u8 { + 3 + } + + #[must_use] + pub const fn default_batch_rate_limit() -> u32 { + 60 + } + + #[must_use] + pub const fn default_pull_sync_ttl_sec() -> u64 { + 86400 + } + + #[must_use] + pub const fn default_pull_sync_rate_limit() -> u32 { + 10 + } +} + +/// Edge Cookie (EC) configuration. +/// +/// Mapped from the `[ec]` TOML section. Controls EC identity generation, +/// KV store names, and partner registry. #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -pub struct EdgeCookie { - #[validate(custom(function = EdgeCookie::validate_secret_key))] - pub secret_key: Redacted, +pub struct Ec { + /// Publisher passphrase used as HMAC key for EC generation. + #[validate(custom(function = Ec::validate_passphrase))] + pub passphrase: Redacted, + + /// Fastly KV store name for the EC identity graph. + #[serde(default)] + pub ec_store: Option, + + /// Maximum number of concurrent pull-sync requests. + #[serde(default = "Ec::default_pull_sync_concurrency")] + pub pull_sync_concurrency: usize, + + /// Entries with `cluster_size` at or below this value are treated as + /// individual users for identity resolution. B2B publishers should + /// raise this to 50+ since readers are frequently on office networks. + #[serde(default = "Ec::default_cluster_trust_threshold")] + pub cluster_trust_threshold: u32, + + /// Legacy cluster re-check interval retained for config compatibility. + /// + /// EC identity entries no longer store cluster-check timestamps, so this + /// value is not used. `/_ts/api/v1/identify` computes cluster size only + /// when an entry does not already have a stored `cluster_size`. + #[serde(default = "Ec::default_cluster_recheck_secs")] + pub cluster_recheck_secs: u64, + + /// Partners (SSPs, DSPs, identity vendors) for EC identity sync. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + #[validate(nested)] + pub partners: Vec, } -impl EdgeCookie { +impl Ec { /// Known placeholder values that must not be used in production. - pub const SECRET_KEY_PLACEHOLDERS: &[&str] = &["secret-key", "secret_key", "trusted-server"]; + pub const PASSPHRASE_PLACEHOLDERS: &[&str] = &["secret-key", "secret_key", "trusted-server"]; - /// Returns `true` if `secret_key` matches a known placeholder value + /// Default maximum concurrent pull-sync requests. + #[must_use] + pub const fn default_pull_sync_concurrency() -> usize { + 3 + } + + /// Default cluster trust threshold. + #[must_use] + pub const fn default_cluster_trust_threshold() -> u32 { + 10 + } + + /// Default cluster re-check interval (1 hour). + #[must_use] + pub const fn default_cluster_recheck_secs() -> u64 { + 3600 + } + + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] - pub fn is_placeholder_secret_key(secret_key: &str) -> bool { - Self::SECRET_KEY_PLACEHOLDERS + pub fn is_placeholder_passphrase(passphrase: &str) -> bool { + Self::PASSPHRASE_PLACEHOLDERS .iter() - .any(|p| p.eq_ignore_ascii_case(secret_key)) + .any(|p| p.eq_ignore_ascii_case(passphrase)) } - /// Validates that the secret key is not empty. + /// Minimum passphrase length for HMAC key strength. + /// + /// This lower bound is only meant to reject obviously bad values; operators + /// are still expected to use a high-entropy random passphrase per the EC + /// setup and key-rotation documentation. + const MIN_PASSPHRASE_LENGTH: usize = 8; + + /// Validates that the passphrase is not empty and meets minimum length. /// /// # Errors /// - /// Returns a validation error if the secret key is empty. - pub fn validate_secret_key(secret_key: &Redacted) -> Result<(), ValidationError> { - if secret_key.expose().is_empty() { - return Err(ValidationError::new("empty_secret_key")); + /// Returns a validation error if the passphrase is empty or shorter + /// than [`Self::MIN_PASSPHRASE_LENGTH`] characters. + pub fn validate_passphrase(passphrase: &Redacted) -> Result<(), ValidationError> { + if passphrase.expose().is_empty() { + return Err(ValidationError::new("empty_passphrase")); + } + if passphrase.expose().len() < Self::MIN_PASSPHRASE_LENGTH { + return Err(ValidationError::new("short_passphrase")); } Ok(()) } @@ -405,7 +596,7 @@ pub struct Settings { pub publisher: Publisher, #[serde(default)] #[validate(nested)] - pub edge_cookie: EdgeCookie, + pub ec: Ec, #[serde(default)] pub integrations: IntegrationSettings, #[serde(default, deserialize_with = "vec_from_seq_or_map")] @@ -425,7 +616,6 @@ pub struct Settings { pub proxy: Proxy, } -#[allow(unused)] impl Settings { /// Creates a new [`Settings`] instance from a pre-built TOML string. /// @@ -444,6 +634,13 @@ impl Settings { settings.proxy.normalize(); settings.consent.validate(); settings.prepare_runtime()?; + + settings.validate().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Configuration validation failed: {err}"), + }) + })?; + settings.validate_admin_coverage()?; Ok(settings) @@ -507,6 +704,31 @@ impl Settings { Ok(()) } + /// Reject settings that still contain known placeholder secrets. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::InsecureDefault`] when one or more secret + /// fields still contain a placeholder value. + pub fn reject_placeholder_secrets(&self) -> Result<(), Report> { + let mut insecure_fields: Vec<&str> = Vec::new(); + + if Ec::is_placeholder_passphrase(self.ec.passphrase.expose()) { + insecure_fields.push("ec.passphrase"); + } + if Publisher::is_placeholder_proxy_secret(self.publisher.proxy_secret.expose()) { + insecure_fields.push("publisher.proxy_secret"); + } + + if insecure_fields.is_empty() { + Ok(()) + } else { + Err(Report::new(TrustedServerError::InsecureDefault { + field: insecure_fields.join(", "), + })) + } + } + /// Resolve the first handler whose regex matches the request path. /// /// # Errors @@ -532,7 +754,8 @@ impl Settings { /// endpoints are always protected by authentication. /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new /// admin routes to `crates/trusted-server-adapter-fastly/src/main.rs`. - pub(crate) const ADMIN_ENDPOINTS: &[&str] = &["/admin/keys/rotate", "/admin/keys/deactivate"]; + pub(crate) const ADMIN_ENDPOINTS: &[&str] = + &["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"]; /// Returns admin endpoint paths that no configured handler covers. /// @@ -576,7 +799,7 @@ impl Settings { Err(Report::new(TrustedServerError::Configuration { message: format!( "No handler covers admin endpoint(s): {}. \ - Add a [[handlers]] entry with a path regex matching /admin/ \ + Add a [[handlers]] entry with a path regex matching /_ts/admin/ \ to protect admin access.", uncovered.join(", ") ), @@ -599,6 +822,33 @@ impl Settings { } } +fn validate_publisher_domain(value: &str) -> Result<(), ValidationError> { + if value.trim() != value || value.is_empty() || value.len() > 253 { + return Err(ValidationError::new("invalid_publisher_domain")); + } + if value.starts_with('.') || value.ends_with('.') || value.contains(['/', ':']) { + return Err(ValidationError::new("invalid_publisher_domain")); + } + + for label in value.split('.') { + if label.is_empty() || label.len() > 63 { + return Err(ValidationError::new("invalid_publisher_domain")); + } + let bytes = label.as_bytes(); + if bytes.first() == Some(&b'-') || bytes.last() == Some(&b'-') { + return Err(ValidationError::new("invalid_publisher_domain")); + } + if !bytes + .iter() + .all(|byte| byte.is_ascii_alphanumeric() || *byte == b'-') + { + return Err(ValidationError::new("invalid_publisher_domain")); + } + } + + Ok(()) +} + fn validate_cookie_domain(value: &str) -> Result<(), ValidationError> { // `=` is excluded: it only has special meaning in the name=value pair, // not within the Domain attribute value. @@ -636,6 +886,19 @@ fn validate_path(value: &str) -> Result<(), ValidationError> { validation_error }) } +fn from_value_or_str<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: DeserializeOwned + FromStr, + T::Err: std::fmt::Display, +{ + let value = JsonValue::deserialize(deserializer)?; + match value { + JsonValue::String(value) => T::from_str(&value).map_err(serde::de::Error::custom), + other => serde_json::from_value(other).map_err(serde::de::Error::custom), + } +} + // Helper: allow Vec fields to deserialize from either a JSON array or a map of numeric indices. // This lets env vars like TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__0=smartadserver work, which the config env source // represents as an object {"0": "value"} rather than a sequence. Also supports string inputs that are @@ -802,11 +1065,16 @@ mod tests { ); assert_eq!(settings.publisher.domain, "test-publisher.com"); assert_eq!(settings.publisher.cookie_domain, ".test-publisher.com"); + assert_eq!( + settings.publisher.ec_cookie_domain(), + ".test-publisher.com", + "EC cookie domain should be computed as .{{domain}}" + ); assert_eq!( settings.publisher.origin_url, "https://origin.test-publisher.com" ); - assert_eq!(settings.edge_cookie.secret_key.expose(), "test-secret-key"); + assert_eq!(settings.ec.passphrase.expose(), "test-secret-key"); settings.validate().expect("Failed to validate settings"); } @@ -818,14 +1086,72 @@ mod tests { r#"origin_url = "https://origin.test-publisher.com/""#, ); - let settings = Settings::from_toml(&toml_str).expect("should parse TOML"); - let result = settings.validate(); + let result = Settings::from_toml(&toml_str); assert!( result.is_err(), "origin_url ending with '/' should fail validation" ); } + #[test] + fn validate_rejects_invalid_publisher_domains() { + for domain in [ + "", + ".example.com", + "example.com.", + "https://example.com", + "bad_domain.com", + ] { + let toml_str = crate_test_settings_str().replace( + r#"domain = "test-publisher.com""#, + &format!(r#"domain = "{domain}""#), + ); + + let result = Settings::from_toml(&toml_str); + assert!(result.is_err(), "should reject invalid domain {domain:?}"); + } + } + + #[test] + fn validate_accepts_localhost_publisher_domain() { + let toml_str = crate_test_settings_str().replace( + r#"domain = "test-publisher.com""#, + r#"domain = "localhost""#, + ); + + let settings = Settings::from_toml(&toml_str).expect("should accept localhost domain"); + assert_eq!(settings.publisher.ec_cookie_domain(), ".localhost"); + } + + #[test] + fn validate_rejects_invalid_ec_partner_ids() { + for partner_id in [ + "Upper", + "bad id", + "ec", + "", + "abcdefghijklmnopqrstuvwxyzabcdefg", + ] { + let toml_str = format!( + r#"{} + [[ec.partners]] + id = "{}" + name = "Invalid Partner" + source_domain = "invalid.example.com" + api_token = "invalid-token" + "#, + crate_test_settings_str(), + partner_id, + ); + + let result = Settings::from_toml(&toml_str); + assert!( + result.is_err(), + "should reject invalid partner ID {partner_id:?}" + ); + } + } + #[test] fn prepare_runtime_rejects_invalid_handler_regex() { let toml_str = crate_test_settings_str().replace(r#"path = "^/secure""#, r#"path = "(""#); @@ -853,32 +1179,32 @@ mod tests { } #[test] - fn is_placeholder_secret_key_rejects_all_known_placeholders() { - for placeholder in EdgeCookie::SECRET_KEY_PLACEHOLDERS { + fn is_placeholder_passphrase_rejects_all_known_placeholders() { + for placeholder in Ec::PASSPHRASE_PLACEHOLDERS { assert!( - EdgeCookie::is_placeholder_secret_key(placeholder), - "should detect placeholder secret_key '{placeholder}'" + Ec::is_placeholder_passphrase(placeholder), + "should detect placeholder passphrase '{placeholder}'" ); } } #[test] - fn is_placeholder_secret_key_is_case_insensitive() { + fn is_placeholder_passphrase_is_case_insensitive() { assert!( - EdgeCookie::is_placeholder_secret_key("SECRET-KEY"), - "should detect case-insensitive placeholder secret_key" + Ec::is_placeholder_passphrase("SECRET-KEY"), + "should detect case-insensitive placeholder passphrase" ); assert!( - EdgeCookie::is_placeholder_secret_key("Trusted-Server"), - "should detect mixed-case placeholder secret_key" + Ec::is_placeholder_passphrase("Trusted-Server"), + "should detect mixed-case placeholder passphrase" ); } #[test] - fn is_placeholder_secret_key_accepts_non_placeholder() { + fn is_placeholder_passphrase_accepts_non_placeholder() { assert!( - !EdgeCookie::is_placeholder_secret_key("test-secret-key"), - "should accept non-placeholder secret_key" + !Ec::is_placeholder_passphrase("test-secret-key"), + "should accept non-placeholder passphrase" ); } @@ -1092,7 +1418,7 @@ mod tests { (path_key_0, Some("^/env-handler")), (username_key_0, Some("env-user")), (password_key_0, Some("env-pass")), - (path_key_1, Some("^/admin")), + (path_key_1, Some("^/_ts/admin")), (username_key_1, Some("admin")), (password_key_1, Some("admin-pass")), ], @@ -1108,6 +1434,156 @@ mod tests { ); } + #[test] + fn test_ec_partners_override_with_indexed_env() { + let toml_str = crate_test_settings_str(); + + let origin_key = format!( + "{}{}PUBLISHER{}ORIGIN_URL", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_id_key = format!( + "{}{}EC{}PARTNERS{}0{}ID", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_name_key = format!( + "{}{}EC{}PARTNERS{}0{}NAME", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_source_domain_key = format!( + "{}{}EC{}PARTNERS{}0{}SOURCE_DOMAIN", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_openrtb_atype_key = format!( + "{}{}EC{}PARTNERS{}0{}OPENRTB_ATYPE", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_bidstream_enabled_key = format!( + "{}{}EC{}PARTNERS{}0{}BIDSTREAM_ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_api_token_key = format!( + "{}{}EC{}PARTNERS{}0{}API_TOKEN", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_id_key = format!( + "{}{}EC{}PARTNERS{}1{}ID", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_name_key = format!( + "{}{}EC{}PARTNERS{}1{}NAME", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_source_domain_key = format!( + "{}{}EC{}PARTNERS{}1{}SOURCE_DOMAIN", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_openrtb_atype_key = format!( + "{}{}EC{}PARTNERS{}1{}OPENRTB_ATYPE", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_bidstream_enabled_key = format!( + "{}{}EC{}PARTNERS{}1{}BIDSTREAM_ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_api_token_key = format!( + "{}{}EC{}PARTNERS{}1{}API_TOKEN", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + temp_env::with_vars( + [ + (origin_key, Some("https://origin.test-publisher.com")), + (partner_0_id_key, Some("envpartner0")), + (partner_0_name_key, Some("Env Partner 0")), + (partner_0_source_domain_key, Some("envpartner0.example.com")), + (partner_0_openrtb_atype_key, Some("1")), + (partner_0_bidstream_enabled_key, Some("true")), + (partner_0_api_token_key, Some("env-token-0")), + (partner_1_id_key, Some("envpartner1")), + (partner_1_name_key, Some("Env Partner 1")), + (partner_1_source_domain_key, Some("envpartner1.example.com")), + (partner_1_openrtb_atype_key, Some("3")), + (partner_1_bidstream_enabled_key, Some("false")), + (partner_1_api_token_key, Some("env-token-1")), + ], + || { + let settings = Settings::from_toml_and_env(&toml_str) + .expect("Settings should load indexed EC partners from env"); + + assert_eq!(settings.ec.partners.len(), 2); + assert_eq!(settings.ec.partners[0].id, "envpartner0"); + assert_eq!(settings.ec.partners[0].name, "Env Partner 0"); + assert_eq!( + settings.ec.partners[0].source_domain, + "envpartner0.example.com" + ); + assert_eq!(settings.ec.partners[0].openrtb_atype, 1); + assert!(settings.ec.partners[0].bidstream_enabled); + assert_eq!(settings.ec.partners[0].api_token.expose(), "env-token-0"); + assert_eq!(settings.ec.partners[1].id, "envpartner1"); + assert_eq!(settings.ec.partners[1].name, "Env Partner 1"); + assert_eq!( + settings.ec.partners[1].source_domain, + "envpartner1.example.com" + ); + assert_eq!(settings.ec.partners[1].openrtb_atype, 3); + assert!(!settings.ec.partners[1].bidstream_enabled); + assert_eq!(settings.ec.partners[1].api_token.expose(), "env-token-1"); + }, + ); + } + #[test] fn test_invalid_handler_override_fails_during_runtime_preparation() { let toml_str = crate_test_settings_str(); @@ -1815,8 +2291,8 @@ mod tests { origin_url = "https://origin.test-publisher.com" proxy_secret = "unit-test-proxy-secret" - [edge_cookie] - secret_key = "test-secret-key" + [ec] + passphrase = "test-secret-key" [request_signing] config_store_id = "test-config-store-id" @@ -1836,8 +2312,8 @@ mod tests { .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/admin/keys/rotate", "/admin/keys/deactivate"], - "should report both admin endpoints as uncovered" + vec!["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate",], + "should report all admin endpoints as uncovered" ); } @@ -1849,7 +2325,7 @@ mod tests { .expect("should check admin coverage"); assert!( uncovered.is_empty(), - "should report no uncovered admin endpoints when handler covers /admin" + "should report no uncovered admin endpoints when handler covers /_ts/admin" ); } @@ -1858,7 +2334,7 @@ mod tests { let toml_str = settings_str_without_admin_handler() + r#" [[handlers]] - path = "^/admin/keys/rotate$" + path = "^/_ts/admin/keys/rotate$" username = "admin" password = "secret" "#; @@ -1870,8 +2346,8 @@ mod tests { .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/admin/keys/deactivate"], - "should detect that only deactivate is uncovered" + vec!["/_ts/admin/keys/deactivate"], + "should detect endpoints not covered by the rotate-only handler" ); } @@ -1941,9 +2417,9 @@ mod tests { .lines() .filter_map(|line| { let trimmed = line.trim(); - // Match arms look like: (Method::POST, "/admin/...") => ... - if trimmed.starts_with('(') && trimmed.contains("\"/admin/") { - let start = trimmed.find("\"/admin/")?; + // Match arms look like: (Method::POST, "/_ts/admin/...") => ... + if trimmed.starts_with('(') && trimmed.contains("\"/_ts/admin/") { + let start = trimmed.find("\"/_ts/admin/")?; let rest = &trimmed[start + 1..]; let end = rest.find('"')?; Some(&rest[..end]) diff --git a/crates/trusted-server-core/src/settings_data.rs b/crates/trusted-server-core/src/settings_data.rs index f69fc7ba2..f8772dfa2 100644 --- a/crates/trusted-server-core/src/settings_data.rs +++ b/crates/trusted-server-core/src/settings_data.rs @@ -3,7 +3,7 @@ use error_stack::{Report, ResultExt}; use validator::Validate; use crate::error::TrustedServerError; -use crate::settings::{EdgeCookie, Publisher, Settings}; +use crate::settings::Settings; pub use crate::auction_config_types::AuctionConfig; @@ -40,37 +40,117 @@ pub fn get_settings() -> Result> { ); } - if EdgeCookie::is_placeholder_secret_key(settings.edge_cookie.secret_key.expose()) { - log::warn!( - "INSECURE: edge_cookie.secret_key is set to a default placeholder — \ - HMAC-SHA256 signatures can be forged. \ - Override via TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY at build time" + settings.reject_placeholder_secrets()?; + + Ok(settings) +} + +#[cfg(test)] +mod tests { + use crate::error::TrustedServerError; + use crate::settings::Settings; + use crate::test_support::tests::crate_test_settings_str; + + /// Builds a TOML string with the given secret values swapped in. + /// + /// # Panics + /// + /// Panics if the replacement patterns no longer match the test TOML, + /// which would cause the substitution to silently no-op. + fn toml_with_secrets(passphrase: &str, proxy_secret: &str) -> String { + let original = crate_test_settings_str(); + let after_passphrase = original.replace( + r#"passphrase = "test-secret-key""#, + &format!(r#"passphrase = "{passphrase}""#), + ); + assert_ne!( + after_passphrase, original, + "should have replaced passphrase value" + ); + let result = after_passphrase.replace( + r#"proxy_secret = "unit-test-proxy-secret""#, + &format!(r#"proxy_secret = "{proxy_secret}""#), ); + assert_ne!( + result, after_passphrase, + "should have replaced proxy_secret value" + ); + result } - if Publisher::is_placeholder_proxy_secret(settings.publisher.proxy_secret.expose()) { - log::warn!( - "INSECURE: publisher.proxy_secret is set to a default placeholder — \ - XChaCha20-Poly1305 encrypted URLs can be decrypted by anyone. \ - Override via TRUSTED_SERVER__PUBLISHER__PROXY_SECRET at build time" + #[test] + fn rejects_placeholder_passphrase() { + let toml = toml_with_secrets("secret-key", "real-proxy-secret"); + let settings = Settings::from_toml(&toml).expect("should parse TOML"); + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject placeholder secret_key"); + let root = err.current_context(); + assert!( + matches!(root, TrustedServerError::InsecureDefault { field } if field.contains("ec.passphrase")), + "error should mention ec.passphrase, got: {root}" ); } - Ok(settings) -} + #[test] + fn rejects_placeholder_proxy_secret() { + let toml = toml_with_secrets("real-secret-key", "change-me-proxy-secret"); + let settings = Settings::from_toml(&toml).expect("should parse TOML"); + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject placeholder proxy_secret"); + let root = err.current_context(); + assert!( + matches!(root, TrustedServerError::InsecureDefault { field } if field.contains("publisher.proxy_secret")), + "error should mention publisher.proxy_secret, got: {root}" + ); + } -#[cfg(test)] -mod tests { - use super::*; + #[test] + fn rejects_both_placeholders_in_single_error() { + let toml = toml_with_secrets("secret_key", "change-me-proxy-secret"); + let settings = Settings::from_toml(&toml).expect("should parse TOML"); + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject both placeholder secrets"); + let root = err.current_context(); + match root { + TrustedServerError::InsecureDefault { field } => { + assert!( + field.contains("ec.passphrase"), + "error should mention ec.passphrase, got: {field}" + ); + assert!( + field.contains("publisher.proxy_secret"), + "error should mention publisher.proxy_secret, got: {field}" + ); + } + other => panic!("expected InsecureDefault, got: {other}"), + } + } + + #[test] + fn accepts_non_placeholder_secrets() { + let toml = toml_with_secrets("production-secret-key", "production-proxy-secret"); + let settings = Settings::from_toml(&toml).expect("should parse TOML"); + settings + .reject_placeholder_secrets() + .expect("non-placeholder secrets should pass validation"); + } + /// Smoke-test the full `get_settings()` pipeline (embedded bytes → UTF-8 → + /// parse → validate → placeholder check). The build-time TOML ships with + /// placeholder secrets, so the expected outcome is an [`InsecureDefault`] + /// error — but reaching that error proves every earlier stage succeeded. #[test] - fn get_settings_loads_embedded_toml_successfully() { - // The embedded TOML contains placeholder secrets (e.g. "trusted-server", - // "change-me-proxy-secret"). This is expected — production builds override - // them via TRUSTED_SERVER__* env vars at build time. - let settings = get_settings().expect("should load settings from embedded TOML"); - assert!(!settings.publisher.domain.is_empty()); - assert!(!settings.publisher.cookie_domain.is_empty()); - assert!(!settings.publisher.origin_url.is_empty()); + fn get_settings_rejects_embedded_placeholder_secrets() { + let err = super::get_settings().expect_err("should reject embedded placeholder secrets"); + assert!( + matches!( + err.current_context(), + TrustedServerError::InsecureDefault { .. } + ), + "should fail with InsecureDefault, got: {err}" + ); } } diff --git a/crates/trusted-server-core/src/storage/kv_store.rs b/crates/trusted-server-core/src/storage/kv_store.rs deleted file mode 100644 index c118005a4..000000000 --- a/crates/trusted-server-core/src/storage/kv_store.rs +++ /dev/null @@ -1,570 +0,0 @@ -//! KV Store consent persistence. -//! -//! Stores and retrieves consent data from a KV Store, keyed by EC ID. This -//! provides consent continuity for returning users whose browsers may not -//! have consent cookies on every request. -//! -//! # Storage layout -//! -//! Each entry uses a single JSON body ([`KvConsentEntry`]) containing the raw -//! consent strings, context flags, and a fingerprint for write-on-change -//! detection. -//! -//! # Change detection -//! -//! Writes only occur when consent signals have actually changed. -//! [`consent_fingerprint`] hashes the raw strings into a compact fingerprint -//! stored in the body's `fp` field. On the next request, the existing -//! fingerprint is compared before writing. - -use bytes::Bytes; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; - -use crate::consent::jurisdiction::Jurisdiction; -use crate::consent::types::{ConsentContext, ConsentSource}; -use crate::platform::PlatformKvStore; - -// --------------------------------------------------------------------------- -// KV body (JSON, stored as value) -// --------------------------------------------------------------------------- - -/// Consent data stored in the KV Store body. -/// -/// Contains the raw consent strings needed to reconstruct a [`ConsentContext`]. -/// Decoded data (TCF, GPP, US Privacy) is not stored — it is re-decoded on -/// read to avoid stale decoded state. -/// -/// The `fp` field holds the consent fingerprint for write-on-change detection. -/// Entries written before PR5 lack this field; `#[serde(default)]` treats them -/// as having an empty fingerprint, which always triggers a self-healing -/// re-write. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct KvConsentEntry { - /// Fingerprint of consent signals for write-on-change detection. - /// - /// Written by [`save_consent_to_kv`]. Entries written before PR5 lack - /// this field; `#[serde(default)]` treats them as having an empty - /// fingerprint, which always triggers a self-healing re-write. - #[serde(default)] - pub fp: String, - /// Raw TC String from `euconsent-v2` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_tc_string: Option, - /// Raw GPP string from `__gpp` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_gpp_string: Option, - /// GPP section IDs (decoded or from `__gpp_sid` cookie). - #[serde(skip_serializing_if = "Option::is_none")] - pub gpp_section_ids: Option>, - /// Raw US Privacy string from `us_privacy` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_us_privacy: Option, - /// Raw Google Additional Consent (AC) string. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_ac_string: Option, - - /// Whether GDPR applies to this request. - pub gdpr_applies: bool, - /// Global Privacy Control signal. - pub gpc: bool, - /// Serialized jurisdiction (e.g. `"GDPR"`, `"US-CA"`, `"unknown"`). - pub jurisdiction: String, - - /// When this entry was stored (deciseconds since Unix epoch). - pub stored_at_ds: u64, -} - -// --------------------------------------------------------------------------- -// Conversions -// --------------------------------------------------------------------------- - -/// Builds a [`KvConsentEntry`] from a [`ConsentContext`]. -/// -/// Captures only the raw strings and contextual flags. Decoded data is -/// intentionally omitted — it will be re-decoded on read. The `fp` field is -/// initialized to an empty string and must be set by the caller before writing. -#[must_use] -pub fn entry_from_context(ctx: &ConsentContext, now_ds: u64) -> KvConsentEntry { - KvConsentEntry { - fp: String::new(), - raw_tc_string: ctx.raw_tc_string.clone(), - raw_gpp_string: ctx.raw_gpp_string.clone(), - gpp_section_ids: ctx.gpp_section_ids.clone(), - raw_us_privacy: ctx.raw_us_privacy.clone(), - raw_ac_string: ctx.raw_ac_string.clone(), - gdpr_applies: ctx.gdpr_applies, - gpc: ctx.gpc, - jurisdiction: ctx.jurisdiction.to_string(), - stored_at_ds: now_ds, - } -} - -/// Converts a [`KvConsentEntry`] into [`crate::consent::types::RawConsentSignals`] -/// suitable for re-decoding via [`crate::consent::build_context_from_signals`]. -#[must_use] -pub fn signals_from_entry(entry: &KvConsentEntry) -> crate::consent::types::RawConsentSignals { - crate::consent::types::RawConsentSignals { - raw_tc_string: entry.raw_tc_string.clone(), - raw_gpp_string: entry.raw_gpp_string.clone(), - raw_gpp_sid: entry.gpp_section_ids.as_ref().map(|ids| { - ids.iter() - .map(ToString::to_string) - .collect::>() - .join(",") - }), - raw_us_privacy: entry.raw_us_privacy.clone(), - gpc: entry.gpc, - } -} - -/// Reconstructs a [`ConsentContext`] from a KV Store entry. -/// -/// Re-decodes the raw strings to populate structured fields (TCF, GPP, US -/// Privacy). The `source` is set to [`ConsentSource::KvStore`] and the -/// `jurisdiction` is parsed from the stored string representation. -#[must_use] -pub fn context_from_entry(entry: &KvConsentEntry) -> ConsentContext { - let signals = signals_from_entry(entry); - let mut ctx = crate::consent::build_context_from_signals(&signals); - - // Restore context fields that aren't derived from raw signals. - ctx.gdpr_applies = entry.gdpr_applies; - ctx.gpc = entry.gpc; - ctx.raw_ac_string = entry.raw_ac_string.clone(); - ctx.jurisdiction = parse_jurisdiction(&entry.jurisdiction); - ctx.source = ConsentSource::KvStore; - - ctx -} - -// --------------------------------------------------------------------------- -// Fingerprinting -// --------------------------------------------------------------------------- - -/// Computes a compact fingerprint of the consent signals for change detection. -/// -/// Returns the first 16 hex characters of a SHA-256 hash computed over all -/// raw consent strings and the GPC flag. This is sufficient for detecting -/// changes without storing full hashes. -#[must_use] -pub fn consent_fingerprint(ctx: &ConsentContext) -> String { - let mut hasher = Sha256::new(); - - // Feed each signal into the hash, separated by a sentinel byte to - // prevent ambiguity (e.g., None+Some("x") vs Some("x")+None). - hash_optional(&mut hasher, ctx.raw_tc_string.as_deref()); - hash_optional(&mut hasher, ctx.raw_gpp_string.as_deref()); - hash_optional(&mut hasher, ctx.raw_us_privacy.as_deref()); - hash_optional(&mut hasher, ctx.raw_ac_string.as_deref()); - hasher.update(if ctx.gpc { b"1" } else { b"0" }); - - // Include GPP section IDs so SID-only changes trigger a KV write. - if let Some(sids) = &ctx.gpp_section_ids { - let mut sorted = sids.clone(); - sorted.sort_unstable(); - for sid in &sorted { - hasher.update(sid.to_string().as_bytes()); - hasher.update(b"\xFF"); - } - } else { - hasher.update(b"\x00"); - } - - let result = hasher.finalize(); - hex::encode(&result[..8]) // 16 hex chars = 8 bytes = 64 bits -} - -/// Feeds an optional string into the hasher with sentinel bytes. -fn hash_optional(hasher: &mut Sha256, value: Option<&str>) { - match value { - Some(s) => { - hasher.update(b"\x01"); - hasher.update(s.as_bytes()); - } - None => hasher.update(b"\x00"), - } -} - -/// Parses a jurisdiction string back into a [`Jurisdiction`] enum. -fn parse_jurisdiction(s: &str) -> Jurisdiction { - match s { - "GDPR" => Jurisdiction::Gdpr, - "non-regulated" => Jurisdiction::NonRegulated, - "unknown" => Jurisdiction::Unknown, - s if s.starts_with("US-") => Jurisdiction::UsState(s[3..].to_owned()), - _ => Jurisdiction::Unknown, - } -} - -// --------------------------------------------------------------------------- -// KV Store operations -// --------------------------------------------------------------------------- - -/// Checks whether the stored consent fingerprint matches the current one. -/// -/// Returns `true` when the stored body's `fp` field equals `new_fp`, meaning -/// no write is needed. Returns `false` when the key is absent, the body -/// cannot be deserialized, or the fingerprint differs. -/// -/// Entries written before PR5 have an empty `fp` (via `#[serde(default)]`), -/// which never matches a computed fingerprint and triggers a self-healing -/// re-write. -fn fingerprint_unchanged(store: &dyn PlatformKvStore, key: &str, new_fp: &str) -> bool { - let bytes = match futures::executor::block_on(store.get_bytes(key)) { - Ok(Some(bytes)) => bytes, - _ => return false, - }; - - serde_json::from_slice::(&bytes) - .map(|entry| entry.fp == new_fp) - .unwrap_or(false) -} - -/// Loads consent data from the KV store for a given EC ID. -/// -/// Returns `Some(ConsentContext)` if a valid entry is found, [`None`] if the -/// key does not exist or deserialization fails. Errors are logged but never -/// propagated — KV failures must not break the request pipeline. -/// -/// # Arguments -/// -/// * `store` — KV store opened by the adapter. -/// * `ec_id` — Edge Cookie ID used as the KV key. -#[must_use] -pub fn load_consent_from_kv(store: &dyn PlatformKvStore, ec_id: &str) -> Option { - let bytes = match futures::executor::block_on(store.get_bytes(ec_id)) { - Ok(Some(bytes)) => bytes, - Ok(None) => { - log::debug!("Consent KV lookup miss for '{ec_id}'"); - return None; - } - Err(e) => { - log::debug!("Consent KV lookup error for '{ec_id}': {e}"); - return None; - } - }; - - match serde_json::from_slice::(&bytes) { - Ok(entry) => { - log::info!( - "Loaded consent from KV store for '{ec_id}' (stored_at_ds={})", - entry.stored_at_ds - ); - Some(context_from_entry(&entry)) - } - Err(e) => { - log::warn!("Failed to deserialize consent KV entry for '{ec_id}': {e}"); - None - } - } -} - -/// Saves consent data to the KV store, writing only when signals have changed. -/// -/// Compares the fingerprint of current consent signals against the fingerprint -/// embedded in the stored entry. If they match, the write is skipped. -/// The fingerprint is embedded in the body so no KV metadata is required. -/// -/// # Arguments -/// -/// * `store` — KV store opened by the adapter. -/// * `ec_id` — Edge Cookie ID used as the KV key. -/// * `ctx` — Current request's consent context. -/// * `max_age_days` — TTL for the entry, matching `max_consent_age_days`. -pub fn save_consent_to_kv( - store: &dyn PlatformKvStore, - ec_id: &str, - ctx: &ConsentContext, - max_age_days: u32, -) { - if ctx.is_empty() { - log::debug!("Skipping consent KV write: consent is empty"); - return; - } - - let fp = consent_fingerprint(ctx); - - if fingerprint_unchanged(store, ec_id, &fp) { - log::debug!("Consent unchanged for '{ec_id}' (fp={fp}), skipping write"); - return; - } - - let mut entry = entry_from_context(ctx, crate::consent::now_deciseconds()); - entry.fp = fp.clone(); - - let body = match serde_json::to_vec(&entry) { - Ok(body) => Bytes::from(body), - Err(e) => { - log::warn!("Failed to serialize consent entry for '{ec_id}': {e}"); - return; - } - }; - - let ttl = std::time::Duration::from_secs(u64::from(max_age_days) * 86_400); - - match futures::executor::block_on(store.put_bytes_with_ttl(ec_id, body, ttl)) { - Ok(()) => { - log::info!("Saved consent to KV store for '{ec_id}' (fp={fp}, ttl={max_age_days}d)"); - } - Err(e) => { - log::warn!("Failed to write consent to KV store for '{ec_id}': {e}"); - } - } -} - -/// Deletes a consent entry from the KV store for a given EC ID. -/// -/// Used when a user revokes consent — the existing EC cookie is being -/// expired, so the persisted consent data must also be removed. -/// -/// Errors are logged but never propagated — KV failures must not -/// break the request pipeline. -pub fn delete_consent_from_kv(store: &dyn PlatformKvStore, ec_id: &str) { - match futures::executor::block_on(store.delete(ec_id)) { - Ok(()) => { - log::info!("Deleted consent KV entry for '{ec_id}' (consent revoked)"); - } - Err(e) => { - log::warn!("Failed to delete consent KV entry for '{ec_id}': {e}"); - } - } -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -fn make_test_context() -> ConsentContext { - ConsentContext { - raw_tc_string: Some("CPXxGfAPXxGfA".to_owned()), - raw_gpp_string: Some("DBACNYA~CPXxGfA".to_owned()), - gpp_section_ids: Some(vec![2, 6]), - raw_us_privacy: Some("1YNN".to_owned()), - raw_ac_string: None, - gdpr_applies: true, - tcf: None, - gpp: None, - us_privacy: None, - expired: false, - gpc: false, - jurisdiction: Jurisdiction::Gdpr, - source: ConsentSource::Cookie, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::consent::jurisdiction::Jurisdiction; - use crate::consent::types::{ConsentContext, ConsentSource}; - - #[test] - fn entry_roundtrip() { - let ctx = make_test_context(); - let entry = entry_from_context(&ctx, 1_000_000); - let json = serde_json::to_string(&entry).expect("should serialize"); - let restored: KvConsentEntry = serde_json::from_str(&json).expect("should deserialize"); - - assert_eq!(restored.raw_tc_string, ctx.raw_tc_string); - assert_eq!(restored.raw_gpp_string, ctx.raw_gpp_string); - assert_eq!(restored.gpp_section_ids, ctx.gpp_section_ids); - assert_eq!(restored.raw_us_privacy, ctx.raw_us_privacy); - assert_eq!(restored.gdpr_applies, ctx.gdpr_applies); - assert_eq!(restored.gpc, ctx.gpc); - assert_eq!(restored.jurisdiction, "GDPR"); - assert_eq!(restored.stored_at_ds, 1_000_000); - } - - #[test] - fn kv_consent_entry_roundtrip_preserves_fp() { - let ctx = make_test_context(); - let fp = consent_fingerprint(&ctx); - let mut entry = entry_from_context(&ctx, 1_000_000); - entry.fp = fp.clone(); - let json = serde_json::to_string(&entry).expect("should serialize"); - let restored: KvConsentEntry = serde_json::from_str(&json).expect("should deserialize"); - - assert_eq!( - restored.fp, fp, - "should preserve fingerprint through roundtrip" - ); - } - - #[test] - fn entry_fits_in_2000_bytes() { - let ctx = make_test_context(); - let mut entry = entry_from_context(&ctx, 1_000_000); - entry.fp = consent_fingerprint(&ctx); - let json = serde_json::to_string(&entry).expect("should serialize"); - assert!( - json.len() <= 2000, - "entry JSON must fit in 2000 bytes, was {} bytes", - json.len() - ); - } - - #[test] - fn context_roundtrip_via_entry() { - let original = make_test_context(); - let entry = entry_from_context(&original, 1_000_000); - let restored = context_from_entry(&entry); - - assert_eq!(restored.raw_tc_string, original.raw_tc_string); - assert_eq!(restored.raw_gpp_string, original.raw_gpp_string); - assert_eq!(restored.raw_us_privacy, original.raw_us_privacy); - assert_eq!(restored.gdpr_applies, original.gdpr_applies); - assert_eq!(restored.gpc, original.gpc); - assert_eq!(restored.jurisdiction, original.jurisdiction); - assert_eq!(restored.source, ConsentSource::KvStore); - } - - #[test] - fn fingerprint_deterministic() { - let ctx = make_test_context(); - let fp1 = consent_fingerprint(&ctx); - let fp2 = consent_fingerprint(&ctx); - assert_eq!(fp1, fp2, "fingerprint should be deterministic"); - assert_eq!(fp1.len(), 16, "fingerprint should be 16 hex chars"); - } - - #[test] - fn fingerprint_changes_with_different_signals() { - let ctx1 = make_test_context(); - let mut ctx2 = make_test_context(); - ctx2.raw_tc_string = Some("DIFFERENT_TC_STRING".to_owned()); - - assert_ne!( - consent_fingerprint(&ctx1), - consent_fingerprint(&ctx2), - "different TC strings should produce different fingerprints" - ); - } - - #[test] - fn fingerprint_changes_with_gpc() { - let mut ctx1 = make_test_context(); - ctx1.gpc = false; - let mut ctx2 = make_test_context(); - ctx2.gpc = true; - - assert_ne!( - consent_fingerprint(&ctx1), - consent_fingerprint(&ctx2), - "different GPC values should produce different fingerprints" - ); - } - - #[test] - fn fingerprint_distinguishes_none_from_empty() { - let mut ctx_none = make_test_context(); - ctx_none.raw_tc_string = None; - let mut ctx_empty = make_test_context(); - ctx_empty.raw_tc_string = Some(String::new()); - - assert_ne!( - consent_fingerprint(&ctx_none), - consent_fingerprint(&ctx_empty), - "None vs empty string should produce different fingerprints" - ); - } - - #[test] - fn signals_from_entry_roundtrip() { - let ctx = make_test_context(); - let entry = entry_from_context(&ctx, 1_000_000); - let signals = signals_from_entry(&entry); - - assert_eq!(signals.raw_tc_string, ctx.raw_tc_string); - assert_eq!(signals.raw_gpp_string, ctx.raw_gpp_string); - assert_eq!(signals.raw_us_privacy, ctx.raw_us_privacy); - assert_eq!(signals.gpc, ctx.gpc); - // gpp_sid is serialized as "2,6" from the section IDs - assert_eq!(signals.raw_gpp_sid, Some("2,6".to_owned())); - } - - #[test] - fn parse_jurisdiction_roundtrip() { - assert_eq!(parse_jurisdiction("GDPR"), Jurisdiction::Gdpr); - assert_eq!( - parse_jurisdiction("US-CA"), - Jurisdiction::UsState("CA".to_owned()) - ); - assert_eq!( - parse_jurisdiction("non-regulated"), - Jurisdiction::NonRegulated - ); - assert_eq!(parse_jurisdiction("unknown"), Jurisdiction::Unknown); - assert_eq!( - parse_jurisdiction("something-else"), - Jurisdiction::Unknown, - "unrecognized jurisdiction should default to Unknown" - ); - } - - #[test] - fn empty_entry_serializes_compact() { - let ctx = ConsentContext::default(); - let entry = entry_from_context(&ctx, 0); - let json = serde_json::to_string(&entry).expect("should serialize"); - // With skip_serializing_if = "Option::is_none", omitted fields keep it small. - assert!( - !json.contains("raw_tc_string"), - "None fields should be omitted from JSON" - ); - } - - #[test] - fn entry_preserves_ac_string() { - let mut ctx = make_test_context(); - ctx.raw_ac_string = Some("2~1234.5678~dv.".to_owned()); - let entry = entry_from_context(&ctx, 0); - let restored = context_from_entry(&entry); - - assert_eq!( - restored.raw_ac_string, - Some("2~1234.5678~dv.".to_owned()), - "AC string should survive roundtrip" - ); - } -} - -#[cfg(test)] -mod new_api_tests { - use super::*; - use edgezero_core::key_value_store::NoopKvStore; - - fn noop() -> NoopKvStore { - NoopKvStore - } - - #[test] - fn load_returns_none_when_key_absent() { - let result = load_consent_from_kv(&noop(), "some-ec-id"); - assert!(result.is_none(), "should return None when key is absent"); - } - - #[test] - fn save_does_not_panic_with_noop_store() { - let ctx = make_test_context(); - save_consent_to_kv(&noop(), "some-ec-id", &ctx, 30); - } - - #[test] - fn delete_does_not_panic_with_noop_store() { - delete_consent_from_kv(&noop(), "some-ec-id"); - } - - #[test] - fn kv_consent_entry_missing_fp_deserialises_as_empty() { - let json = r#"{"gdpr_applies":true,"gpc":false,"jurisdiction":"GDPR","stored_at_ds":0}"#; - let entry: KvConsentEntry = - serde_json::from_str(json).expect("should deserialize legacy entry"); - assert_eq!( - entry.fp, - String::new(), - "should default fp to empty string for legacy entries" - ); - } -} diff --git a/crates/trusted-server-core/src/storage/mod.rs b/crates/trusted-server-core/src/storage/mod.rs index ed5ff1ff5..0d8ed1452 100644 --- a/crates/trusted-server-core/src/storage/mod.rs +++ b/crates/trusted-server-core/src/storage/mod.rs @@ -9,7 +9,6 @@ pub(crate) mod api_client; pub(crate) mod config_store; -pub mod kv_store; pub(crate) mod secret_store; pub use api_client::FastlyApiClient; diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index ec5f8ddfa..ccb3b25ba 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -5,19 +5,6 @@ //! - Pluggable content processors (text replacement, HTML rewriting, etc.) //! - Memory-efficient streaming //! - UTF-8 boundary handling -//! -//! # Platform notes -//! -//! This module is **platform-agnostic** (verified 2026-03-31; see -//! `docs/superpowers/plans/2026-03-31-pr8-content-rewriting-verification.md`). It has zero -//! `fastly` imports. [`StreamingPipeline::process`] is generic over -//! `R: Read + W: Write` — any reader or writer works, including -//! `fastly::Body` (which implements `std::io::Read`) or standard -//! `std::io::Cursor<&[u8]>`. -//! -//! Future adapters (Cloudflare Workers, Axum, Spin) do not need to implement any compression or -//! streaming interface. See `crate::platform` module doc for the -//! authoritative note. use std::cell::RefCell; use std::io::{self, Read, Write}; diff --git a/crates/trusted-server-core/src/streaming_replacer.rs b/crates/trusted-server-core/src/streaming_replacer.rs index 1e7291e57..faf8f9a20 100644 --- a/crates/trusted-server-core/src/streaming_replacer.rs +++ b/crates/trusted-server-core/src/streaming_replacer.rs @@ -2,8 +2,6 @@ //! //! This module provides functionality for replacing patterns in content //! in streaming fashion, handling content that may be split across multiple chunks. -//! -//! See [`crate::platform`] module doc for platform notes. // Note: std::io::{Read, Write} were previously used by stream_process function // which has been removed in favor of StreamingPipeline diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 8fdfaa85d..77b65b12f 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -15,7 +15,7 @@ pub mod tests { password = "pass" [[handlers]] - path = "^/admin" + path = "^/_ts/admin" username = "admin" password = "admin-pass" @@ -34,8 +34,8 @@ pub mod tests { enabled = false rewrite_attributes = ["href", "link", "url"] - [edge_cookie] - secret_key = "test-secret-key" + [ec] + passphrase = "test-secret-key" [request_signing] config_store_id = "test-config-store-id" secret_store_id = "test-secret-store-id" diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index d66a820c0..36ed84c5b 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -78,6 +78,7 @@ export default withMermaid( text: 'Core Concepts', items: [ { text: 'Edge Cookies', link: '/guide/edge-cookies' }, + { text: 'EC Setup Guide', link: '/guide/ec-setup-guide' }, { text: 'GDPR Compliance', link: '/guide/gdpr-compliance' }, { text: 'Ad Serving', link: '/guide/ad-serving' }, { diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 880efc835..a44f202b7 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -5,6 +5,7 @@ Quick reference for all Trusted Server HTTP endpoints. ## Endpoint Categories - [First-Party Endpoints](#first-party-endpoints) - Core ad serving and proxying +- [Edge Cookie Endpoints](#edge-cookie-endpoints) - Identity sync and enrichment - [Request Signing](#request-signing-endpoints) - Cryptographic signing and key management - [TSJS Library](#tsjs-library-endpoint) - JavaScript library serving - [Integration Endpoints](#integration-endpoints) - Third-party service proxying @@ -47,6 +48,72 @@ curl "https://edge.example.com/first-party/ad?slot=header-banner&w=728&h=90" --- +## Edge Cookie Endpoints + +Partners are configured statically in `[[ec.partners]]` and loaded into an in-memory registry at startup. There is no runtime partner-registration endpoint and the legacy browser pixel sync endpoint has been removed; browser-resolved IDs are ingested through Prebid EID cookies. + +--- + +### GET /\_ts/api/v1/identify + +Returns EC identity plus the authenticated partner's UID and EID for the current user. + +**Auth:** Bearer token (`Authorization: Bearer `) + +**Request:** + +- Uses `ts-ec` cookie and consent signals + +**Response (example):** + +```json +{ + "ec": "954d...e0c3.nZ1GxL", + "consent": "ok", + "degraded": false, + "partner_id": "mocktioneer", + "uid": "mock-user-123", + "eid": { + "source": "formally-vital-lion.edgecompute.app", + "uids": [{ "id": "mock-user-123", "atype": 3 }] + } +} +``` + +--- + +### POST /\_ts/api/v1/batch-sync + +Server-to-server batch sync endpoint for writing EC ID to partner UID mappings. Mapping timestamps are retained in the request schema for compatibility, but they no longer order writes because EC identity entries do not store per-partner sync timestamps. Valid mappings use idempotent last-write-wins semantics. + +**Auth:** Bearer token (`Authorization: Bearer `) + +**Request Body:** + +```json +{ + "mappings": [ + { + "ec_id": "954d8e7398dd993f78e3875ca1ef7841249781240e913157c1f2d6a6c960e0c3.nZ1GxL", + "partner_uid": "mock-user-123", + "timestamp": 1775147300 + } + ] +} +``` + +**Response:** + +```json +{ + "accepted": 1, + "rejected": 0, + "errors": [] +} +``` + +--- + ### POST /third-party/ad Client-side auction endpoint for TSJS library. @@ -329,7 +396,7 @@ curl -X POST https://edge.example.com/verify-signature \ --- -### POST /admin/keys/rotate +### POST /\_ts/admin/keys/rotate Generates and activates a new signing key. @@ -359,7 +426,7 @@ If omitted, auto-generates date-based ID (e.g., `ts-2025-01-15-A`). **Example:** ```bash -curl -X POST https://edge.example.com/admin/keys/rotate \ +curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ -u admin:password \ -H "Content-Type: application/json" ``` @@ -374,7 +441,7 @@ See [Key Rotation Guide](./key-rotation.md) for workflow details. --- -### POST /admin/keys/deactivate +### POST /\_ts/admin/keys/deactivate Deactivates or deletes a signing key. @@ -407,7 +474,7 @@ Deactivates or deletes a signing key. **Example:** ```bash -curl -X POST https://edge.example.com/admin/keys/deactivate \ +curl -X POST https://edge.example.com/_ts/admin/keys/deactivate \ -u admin:password \ -H "Content-Type: application/json" \ -d '{"kid":"ts-2025-01-14-A","delete":true}' @@ -588,7 +655,7 @@ Endpoints under protected paths require HTTP Basic Authentication: ```toml [[handlers]] -path = "^/admin" +path = "^/_ts/admin" username = "admin" password = "secure-password" ``` @@ -596,13 +663,13 @@ password = "secure-password" **Usage:** ```bash -curl -u admin:secure-password https://edge.example.com/admin/keys/rotate +curl -u admin:secure-password https://edge.example.com/_ts/admin/keys/rotate ``` **Protected Endpoints:** -- `/admin/keys/rotate` -- `/admin/keys/deactivate` +- `/_ts/admin/keys/rotate` +- `/_ts/admin/keys/deactivate` - Any paths matching configured `handlers` patterns --- diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 029163bbf..3795495df 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -23,8 +23,8 @@ cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" proxy_secret = "your-secure-secret-here" -[edge_cookie] -secret_key = "your-hmac-secret" +[ec] +passphrase = "your-hmac-secret" ``` ### Environment Variable Overrides @@ -37,7 +37,7 @@ at runtime. # Format: TRUSTED_SERVER__SECTION__FIELD export TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com -export TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=your-secret +export TRUSTED_SERVER__EC__PASSPHRASE=your-passphrase ``` ### Generate Secure Secrets @@ -60,7 +60,7 @@ openssl rand -base64 32 | Section | Purpose | | ------------------- | -------------------------------------------- | | `[publisher]` | Domain, origin, proxy settings | -| `[edge_cookie]` | Edge Cookie (EC) ID generation | +| `[ec]` | Edge Cookie (EC) ID generation | | `[proxy]` | Proxy SSRF allowlist | | `[request_signing]` | Ed25519 request signing | | `[auction]` | Auction orchestration | @@ -75,8 +75,8 @@ cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" proxy_secret = "change-me-to-secure-value" -[edge_cookie] -secret_key = "your-hmac-secret-key" +[ec] +passphrase = "your-hmac-secret-key" [request_signing] enabled = true @@ -153,19 +153,22 @@ Core publisher settings for domain, origin, and proxy configuration. ### `[publisher]` -| Field | Type | Required | Description | -| --------------- | ------ | -------- | ------------------------------------------------------- | -| `domain` | String | Yes | Publisher's domain name | -| `cookie_domain` | String | Yes | Domain for setting cookies (typically with leading dot) | -| `origin_url` | String | Yes | Full URL of publisher origin server | -| `proxy_secret` | String | Yes | Secret key for encrypting/signing proxy URLs | +| Field | Type | Required | Description | +| --------------- | ------ | -------- | ------------------------------------------------------ | +| `domain` | String | Yes | Publisher's apex domain name | +| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | +| `origin_url` | String | Yes | Full URL of publisher origin server | +| `proxy_secret` | String | Yes | Secret key for encrypting/signing proxy URLs | + +> **Note:** EC cookies (`ts-ec`) derive their domain automatically as `.{domain}` and +> do not use `cookie_domain`. The `cookie_domain` field is used by other cookie helpers. **Example**: ```toml [publisher] domain = "publisher.com" -cookie_domain = ".publisher.com" # Includes subdomains +cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" proxy_secret = "change-me-to-secure-random-value" ``` @@ -199,12 +202,12 @@ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=your-secret-here #### `cookie_domain` -**Purpose**: Domain scope for EC cookies. +**Purpose**: Domain scope for non-EC cookies. **Usage**: -- Set on `ts-ec` cookie -- Controls cookie sharing across subdomains +- Used by non-EC cookie helpers for domain scoping +- EC cookies (`ts-ec`) use a separate computed domain derived from `domain` **Format**: Domain with optional leading dot @@ -263,32 +266,46 @@ Changing `proxy_secret` invalidates all existing signed URLs. Plan rotations car ## EC Configuration -Settings for generating privacy-preserving Edge Cookie identifiers. +Settings for generating privacy-preserving Edge Cookie identifiers. The `ec_store` KV store is the only KV-backed EC lifecycle store; it holds identity graph state, minimal consent metadata, partner IDs, and withdrawal tombstones. Consent configuration controls request-local interpretation and forwarding, not separate KV persistence. -### `[edge_cookie]` +### `[ec]` -| Field | Type | Required | Description | -| ------------ | ------ | -------- | ----------------------------- | -| `secret_key` | String | Yes | HMAC secret for ID generation | +| Field | Type | Required | Description | +| ------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | +| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | +| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | +| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | +| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | +| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | +| `partners` | Array | No | Static partner registry entries | **Example**: ```toml -[edge_cookie] -secret_key = "your-secure-hmac-secret" +[ec] +passphrase = "your-secure-hmac-secret" +ec_store = "ec_identity_store" + +[[ec.partners]] +id = "mocktioneer" +name = "Mocktioneer SSP" +source_domain = "mocktioneer.example" +api_token = "partner-api-token" +bidstream_enabled = true ``` **Environment Override**: ```bash -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=your-secret +TRUSTED_SERVER__EC__PASSPHRASE=your-secret +TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ``` ### Field Details -#### `secret_key` +#### `passphrase` -**Purpose**: HMAC secret for EC ID base generation. +**Purpose**: Publisher passphrase used as HMAC key for EC ID generation. **Security**: @@ -436,7 +453,7 @@ Path-based HTTP Basic Authentication. ```toml # Single handler [[handlers]] -path = "^/admin" +path = "^/_ts/admin" username = "admin" password = "secure-password" @@ -456,7 +473,7 @@ password = "api-pass" ```bash # Handler 0 -TRUSTED_SERVER__HANDLERS__0__PATH="^/admin" +TRUSTED_SERVER__HANDLERS__0__PATH="^/_ts/admin" TRUSTED_SERVER__HANDLERS__0__USERNAME="admin" TRUSTED_SERVER__HANDLERS__0__PASSWORD="secure-password" @@ -474,10 +491,10 @@ TRUSTED_SERVER__HANDLERS__1__PASSWORD="api-pass" ```toml # Exact path -path = "^/admin$" # Only /admin +path = "^/_ts/admin$" # Only /_ts/admin # Prefix match -path = "^/admin" # /admin, /admin/users, /admin/settings +path = "^/_ts/admin" # /_ts/admin, /_ts/admin/users, /_ts/admin/settings # Multiple paths path = "^/(admin|secure|private)" @@ -886,8 +903,8 @@ Configuration is validated at startup: **EC Validation**: -- `secret_key` ≥ 1 character -- `secret_key` ≠ known placeholders (`"secret-key"`, `"secret_key"`, `"trusted-server"` — case-insensitive) +- `passphrase` ≥ 1 character +- `passphrase` ≠ known placeholders (`"secret-key"`, `"secret_key"`, `"trusted-server"` — case-insensitive) **Handler Validation**: @@ -944,7 +961,7 @@ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret_staging) ```bash # All secrets from environment TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret) -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=$(cat /run/secrets/ec_secret) +TRUSTED_SERVER__EC__PASSPHRASE=$(cat /run/secrets/ec_secret) TRUSTED_SERVER__HANDLERS__0__PASSWORD=$(cat /run/secrets/admin_password) ``` @@ -998,7 +1015,7 @@ trusted-server.dev.toml # Development overrides **"Configuration field '...' is set to a known placeholder value"**: -- `edge_cookie.secret_key` cannot be `"secret-key"`, `"secret_key"`, or `"trusted-server"` (case-insensitive) +- `ec.passphrase` cannot be `"secret-key"`, `"secret_key"`, or `"trusted-server"` (case-insensitive) - `publisher.proxy_secret` cannot be `"change-me-proxy-secret"` (case-insensitive) - Must be non-empty - Change to a secure random value (see generation commands above) @@ -1006,7 +1023,7 @@ trusted-server.dev.toml # Development overrides **"Invalid regex"**: - Handler `path` must be valid regex -- Test pattern: `echo "^/admin" | grep -E "^/admin"` +- Test pattern: `echo "^/_ts/admin" | grep -E "^/_ts/admin"` - Escape special characters: `\.`, `\$`, etc. **"Integration configuration could not be parsed"**: diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md new file mode 100644 index 000000000..a1bfa08bc --- /dev/null +++ b/docs/guide/ec-setup-guide.md @@ -0,0 +1,211 @@ +# Edge Cookie Setup Guide + +End-to-end setup and verification guide for Edge Cookie (EC) identity flows. + +This guide covers: + +1. Fastly store setup +2. Partner configuration +3. Server-to-server batch sync (`/_ts/api/v1/batch-sync`) +4. Identity verification (`/_ts/api/v1/identify`) +5. Auction bidstream verification (`/auction`) + +## Prerequisites + +- Trusted Server deployed and reachable (example: `https://getpurpose.ai`) +- Access to update `trusted-server.toml` / deployment configuration +- Fastly CLI authenticated (for store verification) +- A valid TCF consent string (`euconsent-v2`) for consent-required requests + +## 1) Required Configuration + +Set EC configuration in `trusted-server.toml`: + +```toml +[ec] +passphrase = "your-secure-hmac-secret" +ec_store = "ec_identity_store" + +[[ec.partners]] +id = "mocktioneer" +name = "Mocktioneer SSP" +source_domain = "formally-vital-lion.edgecompute.app" +api_token = "test-batch-sync-key-2026" +bidstream_enabled = true +``` + +Required behavior assumptions: + +- `ec_store` is linked to the active Fastly service version +- `ec_store` is the only KV-backed EC lifecycle store; it contains identity graph state, minimal consent metadata, partner IDs, and withdrawal tombstones +- Live consent is interpreted from request cookies, headers, geolocation, and policy defaults rather than a separate consent KV store +- Partners are configured statically in `[[ec.partners]]` and loaded into an in-memory registry at startup +- Partner has `bidstream_enabled = true` if you want `user.ext.eids` in bidstream + +## 2) Configure Demo Variables + +```bash +TS_BASE_URL="https://getpurpose.ai" +MOCK_SSP_URL="https://formally-vital-lion.edgecompute.app" + +PARTNER_ID="mocktioneer" +PARTNER_NAME="Mocktioneer SSP" +PARTNER_API_KEY="test-batch-sync-key-2026" + +# Optional: use a real browser EC if already present +EC_ID="<64hex.6chars>" + +TCF_CONSENT="" +PARTNER_UID="mock-user-$(date +%s)" +``` + +## 3) Configure Partner + +Partners are configured in `trusted-server.toml` and loaded at startup: + +```toml +[[ec.partners]] +id = "mocktioneer" +name = "Mocktioneer SSP" +source_domain = "formally-vital-lion.edgecompute.app" +api_token = "test-batch-sync-key-2026" +bidstream_enabled = true +``` + +Deploy/restart after changing partner configuration. + +## 4) Acquire or Reuse EC Cookie + +If you already have an EC from browser traffic, reuse it. + +Otherwise, attempt generation with consent: + +```bash +curl -si "${TS_BASE_URL}/" \ + -H "Cookie: euconsent-v2=${TCF_CONSENT}" +``` + +Look for: + +- `Set-Cookie: ts-ec=<64hex.6chars>` + +## 5) Batch Sync (S2S) + +Endpoint: `POST /_ts/api/v1/batch-sync` + +Important: request field is `ec_id` (full `{64hex}.{6alnum}` value). The `timestamp` field remains required for API compatibility, but it no longer orders writes because EC identity entries do not store per-partner sync timestamps. Valid mappings are idempotent last-write-wins: unchanged UIDs are accepted without a write, and different UIDs replace the stored value. + +```bash +BATCH_UID="${PARTNER_UID}-batch" +NOW_TS="$(date +%s)" + +curl -X POST "${TS_BASE_URL}/_ts/api/v1/batch-sync" \ + -H "Authorization: Bearer ${PARTNER_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{ + \"mappings\": [{ + \"ec_id\": \"${EC_ID}\", + \"partner_uid\": \"${BATCH_UID}\", + \"timestamp\": ${NOW_TS} + }] + }" | python3 -m json.tool +``` + +Expected: + +```json +{ + "accepted": 1, + "rejected": 0, + "errors": [] +} +``` + +## 6) Verify Identity + +Endpoint: `GET /_ts/api/v1/identify` + +```bash +curl -s "${TS_BASE_URL}/_ts/api/v1/identify" \ + -H "Authorization: Bearer ${PARTNER_API_KEY}" \ + -H "Cookie: ts-ec=${EC_ID}; euconsent-v2=${TCF_CONSENT}" | python3 -m json.tool +``` + +Expected shape: + +```json +{ + "ec": "", + "consent": "ok", + "degraded": false, + "partner_id": "mocktioneer", + "uid": "mock-user-123", + "eid": { + "source": "formally-vital-lion.edgecompute.app", + "uids": [{ "id": "mock-user-123", "atype": 3 }] + }, + "cluster_size": 12 +} +``` + +## 7) Verify Auction Bidstream Enrichment + +Endpoint: `POST /auction` + +```bash +curl -si -X POST "${TS_BASE_URL}/auction" \ + -H "Cookie: ts-ec=${EC_ID}; euconsent-v2=${TCF_CONSENT}" \ + -H "Content-Type: application/json" \ + -d '{"adUnits":[{"code":"test","mediaTypes":{"banner":{"sizes":[[300,250]]}}}]}' +``` + +Check response headers: + +- `x-ts-ec` +- `x-ts-ec-consent` +- `x-ts-eids` + For returning users, ordinary page views should include `x-ts-ec` but should not refresh `Set-Cookie: ts-ec=...`. A `Set-Cookie` header is expected when the EC is newly generated. + +Decode `x-ts-eids`: + +```bash +echo "" | base64 -d | python3 -m json.tool +``` + +Expected decoded payload contains: + +- `source = formally-vital-lion.edgecompute.app` +- `uids[0].id = ` + +## 8) Fastly KV Operational Checks + +List stores: + +```bash +fastly kv-store list +``` + +Check service resource links for active version: + +```bash +fastly resource-link list --service-id --version +``` + +Inspect EC identity entry: + +```bash +fastly kv-store-entry get --store-id --key "${EC_ID}" +``` + +If batch sync returns `ineligible`, check whether the KV entry is missing or has `consent.ok = false` from a withdrawal tombstone. + +## 9) Troubleshooting Quick Map + +| Symptom | Likely Cause | Check | +| ----------------------------------------------------- | ------------------------------ | --------------------------------------------------------------------------- | +| `invalid_token` on batch sync | Wrong partner API key | Re-register partner with known API key | +| `missing field ec_id` | Wrong request schema | Use `ec_id` field | +| `/_ts/api/v1/identify` returns `{"consent":"denied"}` | No consent for current request | Send consent cookie | +| No `uid` in `/_ts/api/v1/identify` | No successful sync yet | Run batch sync or ensure Prebid EID ingestion has populated the partner UID | + +See also: [Edge Cookies](/guide/edge-cookies), [Configuration](/guide/configuration), [API Reference](/guide/api-reference) diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index d0e31e2e0..7f3f1dad1 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -8,6 +8,8 @@ Edge Cookies (EC) are privacy-safe identifiers generated on a first site visit u Trusted Server surfaces the current EC ID via response headers and a first-party cookie. For the exact header and cookie names, see the [API Reference](/guide/api-reference). +For full operational onboarding (partner configuration, batch sync, identify, and auction verification), use the [EC Setup Guide](/guide/ec-setup-guide). + ## How They Work ### HMAC-Based Generation @@ -18,9 +20,144 @@ EC IDs use HMAC (Hash-based Message Authentication Code) to generate a determini **IP normalization**: IPv6 addresses are normalized to a /64 prefix before hashing. +### Request Lifecycle + +Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction) — read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. + +```mermaid +sequenceDiagram + participant B as Browser + participant TS as Trusted Server + participant KV as KV Store + + B->>TS: Request (ts-ec cookie + consent signals) + Note over TS: Phase 1: Pre-routing
Read EC from cookie/header
Build consent context
Extract device signals + + alt First Visit (no EC cookie) + Note over TS: Phase 2: Routing (organic only)
generate_if_needed() + TS->>TS: HMAC-SHA256(IP) + random suffix + TS->>KV: Create entry (consent, geo, device) + Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies + TS-->>B: Response + Set-Cookie: ts-ec=... + else Return Visit (EC cookie present) + Note over TS: Phase 2: Routing
EC exists — skip generation + Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies + TS-->>B: Response + x-ts-ec header
(no cookie refresh) + end + + Note over TS,KV: Phase 4: Post-send (background)
Dispatch pull-sync to partners +``` + +### Response Finalization + +After routing completes, the server evaluates consent state and cookie presence to decide what to do with the EC cookie on the response. + +```mermaid +flowchart TD + Start[ec_finalize_response] --> ConsentCheck{Consent
allows EC?} + + ConsentCheck -- "No" --> ExplicitWithdrawal{Explicit
withdrawal?} + ExplicitWithdrawal -- "Yes" --> CookiePresent{Cookie was
present?} + CookiePresent -- "Yes" --> Withdraw["Expire ts-ec cookie
Write withdrawal tombstone in ec_identity_store (24h TTL)
Strip all x-ts-* headers"] + CookiePresent -- "No" --> HeaderOnly["Strip all x-ts-* headers only
(no cookie expiry or KV tombstone)"] + ExplicitWithdrawal -- "No" --> HeaderOnly + + ConsentCheck -- "Yes" --> WasPresent{EC was present
in request?} + WasPresent -- "Yes, not generated" --> Returning["Ingest Prebid EID cookies
Set x-ts-ec header only
(no cookie or KV TTL refresh)"] + WasPresent -- "No, just generated" --> NewEc["Ingest Prebid EID cookies
Set ts-ec cookie + x-ts-ec header"] +``` + +When consent cannot be verified for the current request — for example, unknown jurisdiction or missing/undecodable consent signals in a regulated region — Trusted Server fails closed for EC use by stripping EC headers, but it does **not** treat that as authoritative revocation of an already-issued EC. + +## Consent Model + +EC creation is gated by jurisdiction. The server detects jurisdiction from geolocation data attached to the request and applies the appropriate consent framework. Live consent comes from request-local signals (`euconsent-v2`, `__gpp`, `__gpp_sid`, `us_privacy`, `Sec-GPC`) plus geolocation and policy defaults; there is no separate consent KV fallback. + +```mermaid +flowchart TD + Start[Detect Jurisdiction] --> J{Jurisdiction?} + + J -- "GDPR
(EU/UK)" --> TCF{TCF string
present?} + TCF -- "Yes" --> P1{Purpose 1
granted?} + P1 -- "Yes" --> Allow([Allow EC]) + P1 -- "No" --> Deny([Deny EC]) + TCF -- "No" --> Deny + + J -- "US State" --> GPC{GPC header
set?} + GPC -- "Yes" --> Deny + GPC -- "No" --> USTCF{TCF from CMP
e.g. Didomi?} + USTCF -- "Yes" --> USP1{Purpose 1
granted?} + USP1 -- "Yes" --> Allow + USP1 -- "No" --> Deny + USTCF -- "No" --> USP{US Privacy
string?} + USP -- "Yes" --> OptOut{Opt-out
sale?} + OptOut -- "No" --> Allow + OptOut -- "Yes" --> Deny + USP -- "No" --> Deny + + J -- "Non-regulated" --> Allow + J -- "Unknown
(no geo data)" --> Deny +``` + +- **GDPR**: Opt-in required. TCF Purpose 1 (store/access device) must be explicitly consented. +- **US State**: Opt-out model with three-tier fallback — GPC always blocks, then TCF if a CMP uses it, then US Privacy string, then fail-closed. +- **Non-regulated**: EC always allowed. +- **Unknown**: Fail-closed when jurisdiction cannot be determined. + +The `ec_identity_store` KV store is the only EC lifecycle store. It holds identity graph state, partner IDs, a minimal consent snapshot used for EC entry metadata, and withdrawal tombstones. Consent interpretation for each request remains based on the live request signals listed above. + +## Partner Sync Channels + +Partner identities flow into the KV identity graph through three channels. Each writes to the same `ids` map in the KV entry via idempotent upsert logic: unchanged UIDs are accepted without a KV write, while different UIDs replace the stored value. + +```mermaid +flowchart LR + subgraph Browser-initiated + Prebid["Prebid EID Cookies
ts-eids + sharedId
Passive cookie ingestion"] + end + + subgraph Server-initiated + Batch["Batch Sync (S2S)
POST /_ts/api/v1/batch-sync
Partner POST + Bearer auth"] + Pull["Pull Sync (Background)
TS calls partner URL
Post-send on organic routes"] + end + + Prebid --> KV[(KV Identity Graph
ids map)] + Batch --> KV + Pull --> KV +``` + +### Prebid EID Cookie Flow + +The `ts-eids` cookie bridges client-side Prebid user ID modules with the server-side identity graph. + +```mermaid +sequenceDiagram + participant Prebid as Prebid.js + participant TSJS as TSJS Prebid Module + participant B as Browser Cookie Jar + participant TS as Trusted Server + participant KV as KV Store + + Prebid->>Prebid: Auction completes + Prebid->>TSJS: bidsBackHandler fires + TSJS->>Prebid: getUserIdsAsEids() + Prebid-->>TSJS: [{source, uids: [{id, atype}]}] + TSJS->>TSJS: Base64 encode full OpenRTB-style EID array
[{source, uids:[{id, atype, ext?}]}] + TSJS->>B: document.cookie = "ts-eids=..." + + Note over B,TS: Next page request + B->>TS: Request with ts-eids cookie + TS->>TS: Base64 decode → parse OpenRTB-style EIDs
match source domains to partners + TS->>KV: upsert_partner_id() per match
(skips write when UID unchanged) +``` + +Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shape in `ts-eids`. The server remains backward-compatible with earlier flattened `{source, id, atype}` cookies during rollout, but new cookies use the structured `uids[]` form. + +The `sharedId` cookie follows a similar path but is written directly by Prebid's SharedID module rather than by TSJS. The server reads it separately and maps it via the `sharedid.org` source domain. + ## Configuration -Configure EC secrets in `trusted-server.toml`. See the full [Configuration Reference](/guide/configuration) for the `[edge_cookie]` section and environment variable overrides. +Configure EC settings in `trusted-server.toml`. See the full [Configuration Reference](/guide/configuration) for the `[ec]` section and environment variable overrides. ## Privacy Considerations @@ -35,8 +172,18 @@ Configure EC secrets in `trusted-server.toml`. See the full [Configuration Refer 2. Rotate secret keys periodically 3. Monitor ID collision rates +## Runtime Behavior Notes + +- Returning requests with consent and an existing `ts-ec` receive an `x-ts-ec` response header only; ordinary page views do not refresh the EC cookie or KV TTL. +- Newly generated ECs receive both `Set-Cookie: ts-ec=...` and `x-ts-ec`. +- When consent is blocked but not explicitly withdrawn, Trusted Server strips EC response headers for that request but leaves any existing `ts-ec` cookie intact; cookie expiry and tombstones happen only on explicit withdrawal. +- `/_ts/api/v1/identify` is read-oriented and returns identity enrichment for the authenticated partner. It computes `cluster_size` only when the EC entry does not already store one. +- `/_ts/api/v1/batch-sync` writes mappings into the EC identity graph. Mapping timestamps are retained for API compatibility but no longer order writes; valid mappings use idempotent last-write-wins semantics. +- Pull sync fills missing partner UIDs only. Existing partner UIDs are not periodically refreshed because EC entries no longer store per-partner sync timestamps. + ## Next Steps +- Follow the [EC Setup Guide](/guide/ec-setup-guide) - Learn about [GDPR Compliance](/guide/gdpr-compliance) - Configure [Ad Serving](/guide/ad-serving) - Learn about [Collective Sync](/guide/collective-sync) for cross-publisher data sharing details and diagrams diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index 99f611aac..b4dd91b83 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -69,7 +69,7 @@ proxy_secret = "change-me-to-random-string" - `publisher.domain` - `publisher.origin_url` - `publisher.proxy_secret` -- `edge_cookie.secret_key` +- `ec.passphrase` --- @@ -141,17 +141,17 @@ Failed to generate EC ID: HMAC error **Solution:** -1. Ensure `secret_key` is set in `trusted-server.toml`: +1. Ensure `passphrase` is set in `trusted-server.toml`: ```toml -[edge_cookie] -secret_key = "your-secure-hmac-secret" +[ec] +passphrase = "your-secure-hmac-secret" ``` 2. Or set via environment variable: ```bash -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=your-secure-hmac-secret +TRUSTED_SERVER__EC__PASSPHRASE=your-secure-hmac-passphrase ``` --- @@ -245,7 +245,9 @@ curl -w "%{time_total}\n" https://upstream-service.example.com Warning: Cookie not set due to domain mismatch ``` -**Cause:** `publisher.cookie_domain` doesn't match request domain +**Cause:** `publisher.cookie_domain` doesn't match request domain. +Note: EC cookies (`ts-ec`) use a computed domain from `publisher.domain`, +not `cookie_domain`. **Solution:** @@ -402,7 +404,7 @@ Signing key not found: ts-2025-01-A 3. Run key rotation to generate new key: ```bash -curl -X POST https://edge.example.com/admin/keys/rotate \ +curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ -u admin:password ``` @@ -425,7 +427,7 @@ curl -X POST https://edge.example.com/admin/keys/rotate \ 1. Initialize keys using rotation endpoint: ```bash -curl -X POST https://edge.example.com/admin/keys/rotate \ +curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ -u admin:password ``` diff --git a/docs/guide/fastly.md b/docs/guide/fastly.md index 2a1edd79b..1b5ad3555 100644 --- a/docs/guide/fastly.md +++ b/docs/guide/fastly.md @@ -94,8 +94,45 @@ fastly secret-store create --name signing_keys Note the store IDs - you'll need them for your `trusted-server.toml` configuration. +## Create EC KV Store + +Edge Cookie flows require one KV store: + +- Identity graph store (`ec_store`) - EC identity graph, partner IDs, minimal consent metadata, and withdrawal tombstones + +Partners are configured statically in `[[ec.partners]]` and loaded into an in-memory registry at startup. There is no separate consent KV store. Consent is interpreted from live request cookies, headers, geolocation, and policy defaults. + +Create it: + +```bash +fastly kv-store create --name ec_identity_store +``` + +Configure in `trusted-server.toml`: + +```toml +[ec] +passphrase = "your-hmac-secret" +ec_store = "ec_identity_store" +``` + +Verify stores exist: + +```bash +fastly kv-store list +``` + +Verify stores are linked to your active service version: + +```bash +fastly resource-link list --service-id --version +``` + +If EC sync returns `kv_unavailable` or identify responses are degraded, first check that the identity store is present and linked to the active version. Legacy partner/consent KV bindings can be removed once no deployment-specific tooling depends on them. + ## Next Steps - Return to [Getting Started](/guide/getting-started) to continue setup - See [Configuration](/guide/configuration) for detailed configuration options +- See [EC Setup Guide](/guide/ec-setup-guide) for end-to-end EC verification - See [Request Signing](/guide/request-signing) for setting up cryptographic signing diff --git a/docs/guide/first-party-proxy.md b/docs/guide/first-party-proxy.md index b978e35f7..a2567c51a 100644 --- a/docs/guide/first-party-proxy.md +++ b/docs/guide/first-party-proxy.md @@ -420,9 +420,9 @@ Configure proxy behavior in `trusted-server.toml`: ```toml [publisher] domain = "publisher.com" +cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" proxy_secret = "your-secure-random-secret" -cookie_domain = ".publisher.com" # For ts-ec cookies ``` ### Proxy Allowlist diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index ce4b328d4..a4ed418cc 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -72,5 +72,6 @@ fastly compute publish ## Next Steps - Learn about [Edge Cookies](/guide/edge-cookies) +- Follow the [EC Setup Guide](/guide/ec-setup-guide) - Understand [GDPR Compliance](/guide/gdpr-compliance) - Configure [Ad Serving](/guide/ad-serving) diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index fb8c99daa..79576b8bd 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -328,6 +328,16 @@ When the integration is enabled, the `IntegrationAttributeRewriter` removes any The NPM integration lives in `crates/js/lib/src/integrations/prebid/index.ts`. Tests typically assert that publisher references disappear and the deferred `tsjs-prebid.min.js` tag is present. +**5. Hybrid EID forwarding** + +For Prebid-routed auctions, Trusted Server now forwards identity using a hybrid model: + +- TSJS reads current-request EIDs from `pbjs.getUserIdsAsEids()` and includes them in the `/auction` payload. +- The edge resolves additional EIDs from the EC/KV identity graph. +- The auction handler merges and deduplicates both sets. +- The Prebid provider forwards the merged result to Prebid Server as `user.ext.eids`. +- The `ts-eids` cookie is still ingested after the response so future requests can benefit from those IDs even without fresh browser-side resolution. + Reusing these patterns makes it straightforward to convert additional legacy flows (for example, Next.js rewrites) into first-class integrations. ## Future Improvements diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index e9ba6fcb6..f7b7a910e 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -223,6 +223,60 @@ The build script (`build-all.mjs`) validates that each adapter exists in `prebid Adding a new client-side bidder requires both a config change (`client_side_bidders`) **and** a rebuild with the adapter included in `TSJS_PREBID_ADAPTERS`. Without the adapter in the bundle, the bidder is silently dropped from both server-side and client-side auctions. ::: +## Identity Forwarding + +Trusted Server uses a **hybrid EID forwarding model** for Prebid-routed auctions: + +1. **Current-request EIDs from Prebid.js** are read from `pbjs.getUserIdsAsEids()` in the browser and sent in the `/auction` request body. +2. **Server-side EIDs from the EC/KV identity graph** are resolved on the edge from the current EC ID. +3. Trusted Server **merges and deduplicates** both sets before calling Prebid Server. +4. The merged result is forwarded downstream as `user.ext.eids` in the OpenRTB request. +5. The `ts-eids` cookie is still ingested after the response so later requests can reuse the IDs even when the current auction does not provide them again. + +This means Prebid auctions get same-request transparency for browser-resolved IDs without giving up the durability of the server-managed EC identity graph. + +### Identity flow + +```mermaid +sequenceDiagram + participant B as Browser / Prebid.js + participant T as Trusted Server /auction + participant K as EC + KV identity graph + participant P as Prebid Server + + B->>B: User ID modules resolve EIDs + B->>T: POST /auction\n(adUnits + current-request eids) + T->>K: Resolve EC-backed partner IDs + K-->>T: KV-derived EIDs + T->>T: Merge + dedupe client + KV EIDs + T->>T: Apply consent gating + T->>P: OpenRTB request\nuser.ext.eids = merged set + P-->>T: OpenRTB bid response + T-->>B: Auction response + T->>K: Ingest ts-eids cookie for future requests +``` + +### Merge and deduplication rules + +- Client-request EIDs and KV-resolved EIDs are merged by `source` +- UIDs are deduplicated by `source + id` +- If the same UID appears in both places, it is sent only once downstream +- Distinct UIDs under the same source are preserved +- Consent gating is applied to the **merged** set before forwarding + +### What reaches Prebid Server + +The downstream Prebid Server request includes: + +- `user.id` when EC forwarding is allowed +- `user.ext.eids` containing the merged, deduplicated EID set +- forwarded browser cookies (subject to consent-forwarding mode) + +In practice, this gives operators both: + +- **same-request identity transparency** for Prebid User ID Module output, and +- **future-request continuity** through cookie ingestion and KV-backed partner resolution. + ## Endpoints ### GET /first-party/ad @@ -290,6 +344,7 @@ The `to_openrtb()` method in `PrebidAuctionProvider` builds OpenRTB requests: - Sets `tagid` from the slot ID - Adds site metadata with publisher domain, page URL, `site.ref` from the Referer header, and `site.publisher` from the domain - Injects EC ID in the user object +- Merges current-request browser EIDs with KV-resolved EIDs and forwards the deduplicated result as `user.ext.eids` - Forwards user consent string and sets the GDPR flag based on geo and consent presence - Translates the `Sec-GPC` header to a US Privacy string (`us_privacy`) - Extracts `DNT` and `Accept-Language` headers into device fields diff --git a/docs/guide/key-rotation.md b/docs/guide/key-rotation.md index d4467bc8d..e7aaef508 100644 --- a/docs/guide/key-rotation.md +++ b/docs/guide/key-rotation.md @@ -240,14 +240,14 @@ You should see a JWKS response with your public keys. ### Using the Rotation Endpoint -**Endpoint**: `POST /admin/keys/rotate` +**Endpoint**: `POST /_ts/admin/keys/rotate` #### Automatic Key ID (Recommended) Let Trusted Server generate a date-based key ID: ```bash -curl -X POST https://your-domain/admin/keys/rotate \ +curl -X POST https://your-domain/_ts/admin/keys/rotate \ -H "Content-Type: application/json" \ -d '{}' ``` @@ -276,7 +276,7 @@ curl -X POST https://your-domain/admin/keys/rotate \ Specify a custom key identifier: ```bash -curl -X POST https://your-domain/admin/keys/rotate \ +curl -X POST https://your-domain/_ts/admin/keys/rotate \ -H "Content-Type: application/json" \ -d '{"kid": "production-2024-q1"}' ``` @@ -356,14 +356,14 @@ Deactivate old keys after: ### Deactivation Endpoint -**Endpoint**: `POST /admin/keys/deactivate` +**Endpoint**: `POST /_ts/admin/keys/deactivate` #### Deactivate (Keep in Storage) Remove from active rotation but keep in storage: ```bash -curl -X POST https://your-domain/admin/keys/deactivate \ +curl -X POST https://your-domain/_ts/admin/keys/deactivate \ -H "Content-Type: application/json" \ -d '{ "kid": "ts-2024-01-15", @@ -388,7 +388,7 @@ curl -X POST https://your-domain/admin/keys/deactivate \ Remove from storage completely: ```bash -curl -X POST https://your-domain/admin/keys/deactivate \ +curl -X POST https://your-domain/_ts/admin/keys/deactivate \ -H "Content-Type: application/json" \ -d '{ "kid": "ts-2024-01-15", @@ -476,14 +476,14 @@ Regular rotation on a fixed schedule: ```bash #!/bin/bash # Rotate signing keys -curl -X POST https://your-domain/admin/keys/rotate +curl -X POST https://your-domain/_ts/admin/keys/rotate # Wait 30 days grace period sleep $((30 * 24 * 60 * 60)) # Deactivate old key OLD_KEY=$(date -d '90 days ago' +ts-%Y-%m-%d) -curl -X POST https://your-domain/admin/keys/deactivate \ +curl -X POST https://your-domain/_ts/admin/keys/deactivate \ -d "{\"kid\": \"$OLD_KEY\", \"delete\": true}" ``` @@ -647,13 +647,13 @@ If a key is compromised: 1. **Immediate**: Rotate to new key ```bash -curl -X POST /admin/keys/rotate +curl -X POST /_ts/admin/keys/rotate ``` 2. **Urgent**: Deactivate compromised key ```bash -curl -X POST /admin/keys/deactivate \ +curl -X POST /_ts/admin/keys/deactivate \ -d '{"kid": "compromised-key", "delete": false}' ``` @@ -664,7 +664,7 @@ curl -X POST /admin/keys/deactivate \ 5. **Cleanup**: Delete compromised key after investigation ```bash -curl -X POST /admin/keys/deactivate \ +curl -X POST /_ts/admin/keys/deactivate \ -d '{"kid": "compromised-key", "delete": true}' ``` diff --git a/docs/guide/onboarding.md b/docs/guide/onboarding.md index 83203d355..5d4d5b32a 100644 --- a/docs/guide/onboarding.md +++ b/docs/guide/onboarding.md @@ -40,7 +40,7 @@ Welcome to the Trusted Server project! This guide keeps internal onboarding note | `crates/trusted-server-adapter-fastly/src/main.rs` | Request routing entry point | | `crates/trusted-server-core/src/publisher.rs` | Publisher origin handling | | `crates/trusted-server-core/src/proxy.rs` | First-party proxy implementation | -| `crates/trusted-server-core/src/edge_cookie.rs` | EC ID generation | +| `crates/trusted-server-core/src/ec/` | EC identity subsystem | | `crates/trusted-server-core/src/integrations/registry.rs` | Integration module pattern | | `trusted-server.toml` | Application configuration | @@ -146,7 +146,7 @@ Use this checklist to track your onboarding progress: - [ ] Read through `main.rs` to understand request routing - [ ] Trace a request through `publisher.rs` and `proxy.rs` -- [ ] Understand EC ID generation in `edge_cookie.rs` +- [ ] Understand the EC identity subsystem in `ec/` - [ ] Review an existing integration (e.g., `prebid.rs`) ### Documentation & Contribution diff --git a/docs/guide/testing.md b/docs/guide/testing.md index cfa7ea51c..fb12ac775 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -21,7 +21,7 @@ cargo test Tests are organized alongside source code in `#[cfg(test)]` modules: ```rust -// crates/trusted-server-core/src/edge_cookie.rs +// crates/trusted-server-core/src/ec/generation.rs #[cfg(test)] mod tests { use super::*; @@ -88,7 +88,7 @@ curl http://localhost:7676/.well-known/trusted-server.json ### EC ID Tests -From `crates/trusted-server-core/src/edge_cookie.rs`: +From `crates/trusted-server-core/src/ec/mod.rs`: ```rust #[test] diff --git a/docs/superpowers/specs/2026-03-11-production-readiness-report-design.md b/docs/superpowers/specs/2026-03-11-production-readiness-report-design.md index 5203e5ff7..e9bf5d162 100644 --- a/docs/superpowers/specs/2026-03-11-production-readiness-report-design.md +++ b/docs/superpowers/specs/2026-03-11-production-readiness-report-design.md @@ -51,10 +51,10 @@ match config. ### C-2: Admin endpoints unprotected unless handler regex covers them -`/admin/keys/rotate` and `/admin/keys/deactivate` are always routed. The +`/_ts/admin/keys/rotate` and `/_ts/admin/keys/deactivate` are always routed. The `enforce_basic_auth` gate only triggers for paths that match a configured `handlers[].path` regex. The default config (`^/secure`) does not cover -`/admin/*`. An operator who doesn't add an explicit admin handler has +`/_ts/admin/*`. An operator who doesn't add an explicit admin handler has **publicly-accessible key rotation/deletion endpoints**. **Refs:** @@ -64,7 +64,7 @@ match config. - `settings.rs:381` -- `handlers` parsing - `trusted-server.toml:1` -- default handler only covers `^/secure` -**Recommendation:** Either hard-require auth for `/admin/*` paths regardless of +**Recommendation:** Either hard-require auth for `/_ts/admin/*` paths regardless of handler config, or validate at startup that an admin handler exists. --- diff --git a/docs/superpowers/specs/2026-03-19-edgezero-migration-design.md b/docs/superpowers/specs/2026-03-19-edgezero-migration-design.md index 0ae8c906b..c48a947dd 100644 --- a/docs/superpowers/specs/2026-03-19-edgezero-migration-design.md +++ b/docs/superpowers/specs/2026-03-19-edgezero-migration-design.md @@ -46,7 +46,7 @@ These decisions are finalized and reflected in this plan: 2. **Migrate all integrations** including GPT and Google Tag Manager as first-class scope. 3. **Admin key routes must be supported on all adapters** — - `/admin/keys/rotate` and `/admin/keys/deactivate` are required on Fastly, + `/_ts/admin/keys/rotate` and `/_ts/admin/keys/deactivate` are required on Fastly, Axum, and Cloudflare (no disabled-route mode). 4. **Temporary Fastly compatibility adapter is required** — `compat.rs` lives in trusted-server during migration (created in PR 11, deleted in PR 15), @@ -1357,7 +1357,7 @@ Changes: - Local development without Viceroy - Mock stores for local KV/config/secret - Implement required admin key routes - (`/admin/keys/rotate`, `/admin/keys/deactivate`) — core signing logic + (`/_ts/admin/keys/rotate`, `/_ts/admin/keys/deactivate`) — core signing logic composes the Axum store primitives (local config/secret providers) - Add `.env.dev` or local config file for Axum-specific **non-secret** settings only (listen address, mock store paths, log level). @@ -1387,7 +1387,7 @@ Changes: - Construct `RuntimeServices` with Cloudflare-backed trait implementations - Wrangler configuration - Implement required admin key routes - (`/admin/keys/rotate`, `/admin/keys/deactivate`) — core signing logic + (`/_ts/admin/keys/rotate`, `/_ts/admin/keys/deactivate`) — core signing logic composes the Cloudflare store primitives (Workers API bindings) - Add `wrangler.toml` with bindings for KV, secrets, and config - Add integration tests: route smoke tests, admin key route tests, @@ -1421,7 +1421,7 @@ Changes: - Route parity validation for all routes currently in `crates/trusted-server-adapter-fastly/src/main.rs` (`/static/tsjs=*`, `/.well-known/trusted-server.json`, - `/verify-signature`, `/admin/keys/rotate`, `/admin/keys/deactivate`, + `/verify-signature`, `/_ts/admin/keys/rotate`, `/_ts/admin/keys/deactivate`, `/auction`, `/first-party/*`, integration routes, and publisher fallback) - Cross-adapter behavior parity tests (Fastly vs Axum vs Cloudflare) for: response status/body, required headers, cookie behavior, and request-signing diff --git a/docs/superpowers/specs/2026-03-24-ssc-prd-design.md b/docs/superpowers/specs/2026-03-24-ssc-prd-design.md index 7f88ef15a..7ed0a6ccd 100644 --- a/docs/superpowers/specs/2026-03-24-ssc-prd-design.md +++ b/docs/superpowers/specs/2026-03-24-ssc-prd-design.md @@ -68,8 +68,8 @@ Today, regular cookies don't suffice for publisher and partner needs. Additional - Implement real-time consent withdrawal: delete cookie and KV entry when consent is revoked - Build a server-side identity graph in Fastly KV Store that accumulates resolved partner IDs over time - Provide three KV write paths: real-time pixel sync redirects, S2S batch push from partners, and TS-initiated S2S pull from partner resolution endpoints -- Expose two bidstream integration modes: header decoration (`/identify`) and full auction orchestration (`/auction`) -- Expose a publisher-authenticated `/admin/partners/register` endpoint for partner provisioning without direct KV access +- Expose two bidstream integration modes: header decoration (`/_ts/api/v1/identify`) and full auction orchestration (`/auction`) +- Expose a publisher-authenticated `/_ts/admin/v1/partners/register` endpoint for partner provisioning without direct KV access ### Non-Goals @@ -116,9 +116,9 @@ TS Lite is a runtime configuration of the existing Trusted Server binary. It is | `GET /first-party/proxy-rebuild` | Enabled | Disabled | | HTML injection pipeline | Enabled | Disabled | | GTM integration | Enabled | Disabled | -| `GET /sync` | Disabled | **Enabled** | -| `GET /identify` | Disabled | **Enabled** | -| `POST /api/v1/sync` | Disabled | **Enabled** | +| `GET /_ts/api/v1/sync` | Disabled | **Enabled** | +| `GET /_ts/api/v1/identify` | Disabled | **Enabled** | +| `POST /_ts/api/v1/batch-sync` | Disabled | **Enabled** | | `GET /.well-known/trusted-server.json` | Enabled | Enabled | When a disabled route is requested, TS returns `404` with the header `X-ts-error: feature-disabled`. @@ -337,19 +337,19 @@ The existing `counter_store` and `opid_store` settings (currently defined but un The EC cookie is deterministic (derived from IP + publisher salt) and lives in the browser. It does not depend on KV Store availability. KV Store holds identity enrichment only — resolved partner UIDs accumulated over time. The degraded behavior policy follows from this: **EC always works; enrichment degrades gracefully.** -| Operation | KV unavailable or error | Rationale | -| --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| EC cookie creation | Set the cookie. Skip the KV entry creation silently. Log the failure at `warn` level. | The cookie is the identity anchor — it does not require KV. The KV entry will be created on the next request once KV recovers. | -| EC cookie refresh (existing user) | Refresh the cookie. Skip the KV `last_seen` update silently. Log at `warn`. | Same as above — the cookie continues working. Stale `last_seen` is acceptable. | -| `/sync` KV write | Redirect to `return` with `ts_synced=0&ts_reason=write_failed`. | The browser redirect must not be blocked by KV availability. This case is already specified in Section 9.4. | -| `/identify` KV read | Return `200` with `ec` hash (from cookie) and `degraded: true`. Set `uids: {}` and `eids: []`. | The EC hash is still valid and useful for attribution and analytics. Empty uids signal that enrichment is unavailable, not that the user has no synced partners. `degraded: true` lets callers distinguish transient KV failure from a genuinely unenriched user. | -| S2S batch write (`/api/v1/sync`) | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | The request was valid; the failure is infrastructure. Partners should retry the batch. | -| S2S pull sync write (async) | Discard the resolved uid. Log at `warn`. Retry will occur on the next qualifying request per the `pull_sync_ttl_sec` window. | Async path — no user-facing impact. | -| Consent withdrawal KV delete | Expire the cookie immediately. Log the KV delete failure at `error` level. Retry the KV delete on the next request for this user. | Cookie deletion is the primary enforcement mechanism. KV delete failure must not block or delay the cookie expiry. | +| Operation | KV unavailable or error | Rationale | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| EC cookie creation | Set the cookie. Skip the KV entry creation silently. Log the failure at `warn` level. | The cookie is the identity anchor — it does not require KV. The KV entry will be created on the next request once KV recovers. | +| EC cookie refresh (existing user) | Refresh the cookie. Skip the KV `last_seen` update silently. Log at `warn`. | Same as above — the cookie continues working. Stale `last_seen` is acceptable. | +| `/_ts/api/v1/sync` KV write | Redirect to `return` with `ts_synced=0&ts_reason=write_failed`. | The browser redirect must not be blocked by KV availability. This case is already specified in Section 9.4. | +| `/_ts/api/v1/identify` KV read | Return `200` with `ec` hash (from cookie) and `degraded: true`. Set `uids: {}` and `eids: []`. | The EC hash is still valid and useful for attribution and analytics. Empty uids signal that enrichment is unavailable, not that the user has no synced partners. `degraded: true` lets callers distinguish transient KV failure from a genuinely unenriched user. | +| S2S batch write (`/_ts/api/v1/sync`) | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | The request was valid; the failure is infrastructure. Partners should retry the batch. | +| S2S pull sync write (async) | Discard the resolved uid. Log at `warn`. Retry will occur on the next qualifying request per the `pull_sync_ttl_sec` window. | Async path — no user-facing impact. | +| Consent withdrawal KV delete | Expire the cookie immediately. Log the KV delete failure at `error` level. Retry the KV delete on the next request for this user. | Cookie deletion is the primary enforcement mechanism. KV delete failure must not block or delay the cookie expiry. | -**`degraded: true` in `/identify` responses** +**`degraded: true` in `/_ts/api/v1/identify` responses** -When a KV read fails, the `/identify` response includes `"degraded": true` in the JSON body alongside an empty `uids` and `eids`. The `ec` field is still populated from the cookie. Callers should proceed with identity-only targeting (EC hash) and omit partner UID parameters from downstream requests. +When a KV read fails, the `/_ts/api/v1/identify` response includes `"degraded": true` in the JSON body alongside an empty `uids` and `eids`. The `ec` field is still populated from the cookie. Callers should proceed with identity-only targeting (EC hash) and omit partner UID parameters from downstream requests. ```json { @@ -461,7 +461,7 @@ This is the primary real-time write path for building the identity graph from ex ### 9.2 Endpoint ``` -GET /sync +GET /_ts/api/v1/sync ``` ### 9.3 Parameters @@ -505,7 +505,7 @@ Partners should treat `ts_synced=0` as a signal that the mapping was not stored. **Acceptance criteria:** -- [ ] `GET /sync?partner=ssp_x&uid=abc&return=https://sync.ssp.com/ack` returns a redirect to the `return` URL within 50ms (excluding KV write time) +- [ ] `GET /_ts/api/v1/sync?partner=ssp_x&uid=abc&return=https://sync.ssp.com/ack` returns a redirect to the `return` URL within 50ms (excluding KV write time) - [ ] KV entry for the EC hash contains `ids.ssp_x.uid = "abc"` after a successful sync; response redirects to `return` with `ts_synced=1` - [ ] If no `ts-ec` cookie is present, redirects to `return` with `ts_synced=0&ts_reason=no_ec`; no KV write performed - [ ] If consent is absent or invalid, redirects to `return` with `ts_synced=0&ts_reason=no_consent`; no KV write performed @@ -524,17 +524,17 @@ The S2S batch sync API allows partners to push ID mappings to Trusted Server in ### 10.2 Endpoint ``` -POST /api/v1/sync +POST /_ts/api/v1/batch-sync ``` ### 10.3 Authentication -Partners authenticate with a rotatable API key. Key rotation must not require redeploying the binary. Partner provisioning is handled via the `/admin/partners/register` endpoint (see Section 15, Open Questions). +Partners authenticate with a rotatable API key. Key rotation must not require redeploying the binary. Partner provisioning is handled via the `/_ts/admin/v1/partners/register` endpoint (see Section 15, Open Questions). ### 10.4 Request ``` -POST /api/v1/sync +POST /_ts/api/v1/batch-sync Content-Type: application/json Authorization: Bearer @@ -593,7 +593,7 @@ Before writing a mapping, Trusted Server checks the KV metadata for the given EC **Acceptance criteria:** -- [ ] `POST /api/v1/sync` with a valid Bearer token and a batch of up to 1000 mappings returns a response within 5 seconds +- [ ] `POST /_ts/api/v1/batch-sync` with a valid Bearer token and a batch of up to 1000 mappings returns a response within 5 seconds - [ ] Accepted mappings are written to the corresponding KV identity graph entries within 1 second - [ ] Mappings for unknown `ec_hash` values are rejected with `ec_hash_not_found` - [ ] Mappings for users with withdrawn consent are rejected with `consent_withdrawn` @@ -721,22 +721,22 @@ The following fields are added to the partner record schema (Section 13.3): Trusted Server exposes two modes for injecting EC identity into the bidstream. Publishers choose the mode that fits their existing ad stack. -### 12.2 Mode A: Identity resolution (`/identify`) +### 12.2 Mode A: Identity resolution (`/_ts/api/v1/identify`) -Trusted Server exposes `/identify` as a standalone identity resolution endpoint for callers that need EC identity and resolved partner UIDs outside of TS's own auction orchestration. TS builds the OpenRTB request in Mode B — `/identify` is not part of that path. It serves three distinct use cases: +Trusted Server exposes `/_ts/api/v1/identify` as a standalone identity resolution endpoint for callers that need EC identity and resolved partner UIDs outside of TS's own auction orchestration. TS builds the OpenRTB request in Mode B — `/_ts/api/v1/identify` is not part of that path. It serves three distinct use cases: **Use case 1 — Attribution and analytics** Any server-side or browser-side system that needs to tag an event, impression, or conversion with the user's EC hash. Examples: analytics pipelines, attribution platforms, reporting dashboards. **Use case 2 — Publisher ad server outbid context** -After TS's auction completes and winners are delivered to the publisher's ad server endpoint, the publisher's ad server may need EC identity and resolved partner UIDs to evaluate whether to accept the programmatic winner or outbid with a direct-sold placement. For this use case, TS includes the EC identity in the winner notification payload directly (see Section 12.3) — a separate `/identify` call is only needed if the publisher's ad server receives the winner through a path that does not carry TS headers. +After TS's auction completes and winners are delivered to the publisher's ad server endpoint, the publisher's ad server may need EC identity and resolved partner UIDs to evaluate whether to accept the programmatic winner or outbid with a direct-sold placement. For this use case, TS includes the EC identity in the winner notification payload directly (see Section 12.3) — a separate `/_ts/api/v1/identify` call is only needed if the publisher's ad server receives the winner through a path that does not carry TS headers. **Use case 3 — Client-side wrappers for non-TS SSPs** -Some SSPs run client-side header bidding wrappers (e.g., Amazon TAM, certain Index Exchange configurations) that do not participate in TS's server-side auction orchestration. A Prebid.js module or custom wrapper script calls `/identify` from the browser to obtain the EC hash and resolved partner UIDs, then injects those values into bid requests sent to those SSPs. This ensures non-TS demand sources bid with the same identity enrichment as TS-orchestrated bids, enabling a fair comparison at winner selection. +Some SSPs run client-side header bidding wrappers (e.g., Amazon TAM, certain Index Exchange configurations) that do not participate in TS's server-side auction orchestration. A Prebid.js module or custom wrapper script calls `/_ts/api/v1/identify` from the browser to obtain the EC hash and resolved partner UIDs, then injects those values into bid requests sent to those SSPs. This ensures non-TS demand sources bid with the same identity enrichment as TS-orchestrated bids, enabling a fair comparison at winner selection. -> **Prerequisite for use case 3:** For a non-TS SSP to receive a useful UID from `/identify`, that SSP must already be a registered partner in `partner_store` and must have a resolved uid in the KV identity graph for this user (via pixel sync, S2S batch, or S2S pull). Without a prior sync, `/identify` returns no uid for that partner. +> **Prerequisite for use case 3:** For a non-TS SSP to receive a useful UID from `/_ts/api/v1/identify`, that SSP must already be a registered partner in `partner_store` and must have a resolved uid in the KV identity graph for this user (via pixel sync, S2S batch, or S2S pull). Without a prior sync, `/_ts/api/v1/identify` returns no uid for that partner. -**Endpoint:** `GET /identify` +**Endpoint:** `GET /_ts/api/v1/identify` **When to call:** Once per auction event — not per-pageview. For use case 3, call before sending bid requests to non-TS SSPs. @@ -744,7 +744,7 @@ Some SSPs run client-side header bidding wrappers (e.g., Amazon TAM, certain Ind **Pattern 1 — Browser-direct (recommended for use cases 1 and 3)** -A script on the publisher's page calls `/identify` via `fetch()`. Because `ec.publisher.com` is same-site with the publisher's domain, the browser sends the `ts-ec` cookie and consent cookies automatically. No forwarding required. +A script on the publisher's page calls `/_ts/api/v1/identify` via `fetch()`. Because `ec.publisher.com` is same-site with the publisher's domain, the browser sends the `ts-ec` cookie and consent cookies automatically. No forwarding required. ```js const identity = await fetch('https://ec.publisher.com/identify').then((r) => @@ -773,7 +773,7 @@ A server-side caller must forward the following from the original browser reques #### Cookie and consent handling -`/identify` follows the EC retrieval priority from Section 6.4. It does not generate a new EC — if no EC is present, the response body contains `consent: denied` and empty identity fields. Consent is evaluated per Section 7.1. `/identify` never sets or modifies cookies. +`/_ts/api/v1/identify` follows the EC retrieval priority from Section 6.4. It does not generate a new EC — if no EC is present, the response body contains `consent: denied` and empty identity fields. Consent is evaluated per Section 7.1. `/_ts/api/v1/identify` never sets or modifies cookies. #### Response @@ -850,7 +850,7 @@ Trusted Server owns the full auction path in Mode B. TS builds the OpenRTB reque **EC context in winner notification to publisher's ad server:** -When TS delivers auction winners to the publisher's ad server endpoint, the response includes EC identity so the publisher's ad server has full context for its outbid decision without needing to call `/identify` separately: +When TS delivers auction winners to the publisher's ad server endpoint, the response includes EC identity so the publisher's ad server has full context for its outbid decision without needing to call `/_ts/api/v1/identify` separately: | Header | Value | | ----------------- | ------------------------------------------------------------ | @@ -895,21 +895,21 @@ Each partner registered in `partner_store` declares: ### 12.6 User stories -**As a publisher using Mode A for analytics/attribution**, I want to call `/identify` from a browser script so that I can tag events and impressions with the user's EC hash and resolved partner UIDs using URL parameters. +**As a publisher using Mode A for analytics/attribution**, I want to call `/_ts/api/v1/identify` from a browser script so that I can tag events and impressions with the user's EC hash and resolved partner UIDs using URL parameters. **Acceptance criteria:** -- [ ] `GET /identify` returns `200` with a valid JSON body within 30ms when EC is present and consent is valid +- [ ] `GET /_ts/api/v1/identify` returns `200` with a valid JSON body within 30ms when EC is present and consent is valid - [ ] `uids` object contains one key per partner with `bidstream_enabled: true` and a resolved UID; partners with no resolved UID are omitted - [ ] If consent is denied, response is `403 Forbidden` with body `{"consent": "denied"}` - [ ] If no EC is present, response is `204 No Content` with no body - [ ] Response headers `X-ts-ec`, `X-ts-eids`, `X-ts-`, and `X-ts-ec-consent` are present on `200` responses as supplementary signals -**As a publisher using a client-side wrapper for non-TS SSPs**, I want to call `/identify` from my Prebid.js configuration so that SSPs outside TS's auction receive the same identity enrichment as TS-orchestrated bids, enabling a fair winner comparison. +**As a publisher using a client-side wrapper for non-TS SSPs**, I want to call `/_ts/api/v1/identify` from my Prebid.js configuration so that SSPs outside TS's auction receive the same identity enrichment as TS-orchestrated bids, enabling a fair winner comparison. **Acceptance criteria:** -- [ ] `GET /identify` called from the browser returns resolved UIDs for all registered partners with a KV entry for this user +- [ ] `GET /_ts/api/v1/identify` called from the browser returns resolved UIDs for all registered partners with a KV entry for this user - [ ] A partner with no KV entry for this user is omitted from `uids` — no empty or null entries - [ ] Response is available within 30ms so it does not block Prebid.js auction timeout @@ -933,7 +933,7 @@ The following capabilities must be configurable without redeploying the binary: - **Publisher passphrase** — the HMAC key used for EC hash generation; same value across all of the publisher's domains; shared with trusted partners to form an identity-federated consortium - **Identity graph store** — the KV store backing the EC hash → identity graph - **Partner registry store** — the KV store backing partner configuration and API key validation -- **Partner records** — each partner's allowed sync domains, bidstream settings, pull sync configuration, and API credentials; managed via `/admin/partners/register` without redeployment +- **Partner records** — each partner's allowed sync domains, bidstream settings, pull sync configuration, and API credentials; managed via `/_ts/admin/v1/partners/register` without redeployment The exact configuration format (TOML keys, KV schema, JSON field names) is an engineering decision and will be documented in the technical design doc. @@ -945,11 +945,11 @@ The following documentation changes are required alongside the EC feature: - **Rename SyntheticID → Edge Cookie** across the entire `docs/` GitHub Pages site. The underlying concept is the same but the product name changes. - **New integration guides**, one per customer type: - - Publisher (full TS): enabling EC in `trusted-server.toml`, partner onboarding via `/admin/partners/register` + - Publisher (full TS): enabling EC in `trusted-server.toml`, partner onboarding via `/_ts/admin/v1/partners/register` - SSP: pixel sync integration guide, sync pixel URL format, callback handling, optional pull resolution endpoint - DSP: S2S batch API reference, authentication, conflict resolution behavior, optional pull resolution endpoint - Identity Provider: registering as a partner, `source_domain` and `openrtb_atype` configuration, sync patterns -- **API reference** for the four new endpoints: `GET /sync`, `GET /identify`, `POST /api/v1/sync`, and the partner-side pull resolution contract +- **API reference** for the four new endpoints: `GET /_ts/api/v1/sync`, `GET /_ts/api/v1/identify`, `POST /_ts/api/v1/batch-sync`, and the partner-side pull resolution contract - **Pull sync integration guide**: partner requirements for exposing a resolution endpoint, authentication, expected response shape, rate limit behavior - **Consent enforcement guide**: how TCF and GPP signals are read, precedence rules, what happens on withdrawal @@ -957,10 +957,10 @@ The following documentation changes are required alongside the EC feature: ## 15. Open Questions -| # | Question | Owner | Status | -| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | --------------------------------------------------------------------------- | -| 1 | Partner provisioning: TS will expose a `/admin/partners/register` endpoint authenticated at the publisher level (bearer token issued per publisher Fastly service), so publishers can onboard SSP/DSP partners without touching KV directly. Engineering to define the exact auth mechanism. | Engineering | **Resolved** — `/admin/partners/register` endpoint, publisher-authenticated | -| 2 | Should TS Lite expose a `GET /health` endpoint so partners can programmatically verify their service is running and their partner config is active in KV? | Product | **N/A** — TS Lite deferred (see Section 5) | +| # | Question | Owner | Status | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | --------------------------------------------------------------------------------------- | +| 1 | Partner provisioning: TS will expose a `/_ts/admin/v1/partners/register` endpoint authenticated at the publisher level, so publishers can onboard SSP/DSP partners without touching KV directly. | Engineering | **Resolved** — `/_ts/admin/v1/partners/register` endpoint protected by admin basic auth | +| 2 | Should TS Lite expose a `GET /health` endpoint so partners can programmatically verify their service is running and their partner config is active in KV? | Product | **N/A** — TS Lite deferred (see Section 5) | --- @@ -970,7 +970,7 @@ The following documentation changes are required alongside the EC feature: | ------------------------------- | --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | | EC match rate (returning users) | >90% within 30 days | Fastly real-time logs: ratio of requests with existing `ts-ec` cookie vs. new EC generations | | Consent enforcement accuracy | 0 ECs created for opted-out EU/UK users | Log audit: verify no `ts-ec` `Set-Cookie` in responses where consent signal is absent | -| KV sync latency (pixel sync) | p99 <75ms end-to-end | Fastly log timing on `/sync` endpoint | +| KV sync latency (pixel sync) | p99 <75ms end-to-end | Fastly log timing on `/_ts/api/v1/sync` endpoint | | S2S batch API throughput | >500 mappings/sec sustained | Load test prior to partner onboarding | | S2S pull sync resolution rate | >30% of pull calls return a non-null uid within 60 days of first partner go-live | Fastly log: pull call outcomes per partner | | Identity graph fill rate | >50% of EC hashes with at least 1 resolved partner ID within 60 days of partner go-live | KV scan sample | diff --git a/docs/superpowers/specs/2026-03-24-ssc-technical-spec-design.md b/docs/superpowers/specs/2026-03-24-ssc-technical-spec-design.md index ad47c65d3..f4e7a2177 100644 --- a/docs/superpowers/specs/2026-03-24-ssc-technical-spec-design.md +++ b/docs/superpowers/specs/2026-03-24-ssc-technical-spec-design.md @@ -3,7 +3,13 @@ **Status:** Draft **Author:** Engineering **PRD reference:** `docs/internal/ssc-prd.md` -**Last updated:** 2026-03-18 +**Last updated:** 2026-04-14 + +> **Supersession note (issue #666):** Sections in this historical design spec +> that describe a separate `consent_store` or consent KV fallback are obsolete. +> Current runtime behavior interprets live consent from request cookies, headers, +> geolocation, and policy defaults. `ec_identity_store` is the only KV-backed EC +> lifecycle store and holds identity graph state plus withdrawal tombstones. --- @@ -16,12 +22,13 @@ 5. [Cookie and Header Handling](#5-cookie-and-header-handling) 6. [Consent Enforcement](#6-consent-enforcement) 7. [KV Store Identity Graph](#7-kv-store-identity-graph) -8. [Pixel Sync Endpoint (`GET /sync`)](#8-pixel-sync-endpoint-get-sync) -9. [S2S Batch Sync API (`POST /api/v1/sync`)](#9-s2s-batch-sync-api-post-apiv1sync) + 7A. [Device Signals and Bot Gate](#7a-device-signals-and-bot-gate) +8. [Prebid EID Cookie Ingestion](#8-prebid-eid-cookie-ingestion) +9. [S2S Batch Sync API (`POST /_ts/api/v1/batch-sync`)](#9-s2s-batch-sync-api-post-apiv1sync) 10. [S2S Pull Sync (TS-Initiated)](#10-s2s-pull-sync-ts-initiated) -11. [Identity Resolution Endpoint (`GET /identify`)](#11-identity-resolution-endpoint-get-identify) +11. [Identity Resolution Endpoint (`GET /_ts/api/v1/identify`)](#11-identity-resolution-endpoint-get-identify) 12. [Bidstream Decoration (`/auction` Mode B)](#12-bidstream-decoration-auction-mode-b) -13. [Partner Registry and Admin Endpoint](#13-partner-registry-and-admin-endpoint) +13. [Partner Registry (Config-Based)](#13-partner-registry-config-based) 14. [Configuration](#14-configuration) 15. [Constants and Header Names](#15-constants-and-header-names) 16. [Error Handling](#16-error-handling) @@ -39,12 +46,12 @@ EC is the full replacement for SyntheticID. The PRD explicitly states backward c **Prerequisites (must be merged before this epic begins):** -- **SyntheticID → Edge Cookie rename** — [PR #479](https://github.com/IABTechLab/trusted-server/pull/479) renames SyntheticID to Edge Cookie (EC) across all code paths: `synthetic.rs` → `edge_cookie.rs`, `COOKIE_SYNTHETIC_ID` → `COOKIE_EC_ID`, `X-Synthetic-*` → `X-ts-ec`/`X-ts-ec-fresh` headers, `settings.synthetic` → `settings.edge_cookie`, and simplifies EC generation to IP-only HMAC-SHA256 (removing Handlebars templating). It also renames `ConsentPipelineInput.synthetic_id` to `ec_id`, updates consent KV helper parameters/docs, and handles consent-store key migration (old SyntheticID keys orphaned, TTL expiry cleans them up). **This PR must be merged before implementation of this spec begins.** The spec assumes a codebase where SyntheticID no longer exists. Verify before starting: +- **SyntheticID removal** — [PR #479](https://github.com/IABTechLab/trusted-server/pull/479) removes SyntheticID from all active code paths: `get_or_generate_synthetic_id()`, `COOKIE_SYNTHETIC_ID`, `X-Synthetic-*` headers, `synthetic.rs` module, `settings.synthetic` config, and all SyntheticID generation/cookie code from `publisher.rs`, `endpoints.rs`, and `registry.rs`. **This PR must be merged before implementation of this spec begins.** The spec assumes a codebase where SyntheticID no longer exists. Verify before starting: - `grep -r 'synthetic_id' crates/` returns no hits outside test fixtures - `grep -r 'X-Synthetic' crates/` returns no hits - `trusted-server.toml` has no `[synthetic]` section - - `ConsentPipelineInput` uses `ec_id`, not `synthetic_id` -- **Consent implementation** — The consent pipeline (`build_consent_context()`, `ConsentContext`, `allows_ec_creation()`, TCF/GPP/US-Privacy decoding) is implemented and available as a stable interface before this epic. PR `#380` merged to `main`. EC calls `allows_ec_creation()` directly — no new gating functions are introduced. Note: EC changes the _phase order_ relative to the old SyntheticID flow — consent is evaluated before EC generation, so first-visit consent KV persistence is deferred to the second request (see §6.1.1 for full analysis). + - `ConsentPipelineInput` uses `identity_key`, not `synthetic_id` +- **Consent implementation** — The consent pipeline (`build_consent_context()`, `ConsentContext`, `allows_ec_creation()`, TCF/GPP/US-Privacy decoding) is implemented and available as a stable interface before this epic. PR `#380` merged to `main`. EC calls `allows_ec_creation()` directly — no new gating functions are introduced. Consent is evaluated from live request cookies, headers, geolocation, and policy defaults before EC generation. **Deferred from this spec (not in scope):** @@ -66,8 +73,21 @@ Browser Request │ extract GeoInfo → enforce auth → route_request │ └──────────┬──────────────────────────────────────┘ │ -Two-phase model (matches existing codebase pattern): - +Phase 0 — bot gate (pure in-memory, no KV I/O): + ┌─────────────────────────────────────────────────┐ + │ derive_device_signals(req) │ + │ - UA → is_mobile, platform_class │ + │ - req.get_tls_ja4() → ja4_class (Section 1) │ + │ - req.get_client_h2_fingerprint() → h2_fp_hash │ + │ - (ja4_class, h2_fp_hash) → known_browser │ + │ │ + │ !looks_like_browser()? │ + │ → suppress KV graph (None), skip ec_finalize, │ + │ skip pull sync. Request still proxied to │ + │ origin — bot receives valid HTML but leaves │ + │ no trace in the identity graph. │ + └──────┬────────────────────────────────────────────┘ + │ Phase 1 — pre-routing (like `GeoInfo::from_request()`): ┌─────────────────────────────────────────┐ │ EcContext::read_from_request() │ @@ -75,6 +95,9 @@ Phase 1 — pre-routing (like `GeoInfo::from_request()`): │ - build_consent_context() → ConsentContext │ │ - allows_ec_creation(consent) │ │ No generation. No cookie writes. │ + │ │ + │ ec_context.set_device_signals(signals) │ + │ (passed through to KvEntry on creation) │ └──────┬──────────────────────────────────┘ │ Phase 2 — inside organic handlers only: @@ -84,20 +107,20 @@ Phase 2 — inside organic handlers only: handle_publisher_request() integration_registry.handle_proxy() calls ec_context.generate_if_needed() calls ec_context.generate_if_needed() -EC route handlers (GET /sync, GET /identify, POST /auction, -POST /api/v1/sync, POST /admin/*) NEVER call generate_if_needed(). -`/identify`, `/auction`, `POST /api/v1/sync`, and `POST /admin/*` -use `EcContext` in read-only form. `GET /sync` is the one exception: -it never bootstraps an EC, but it may replace `ec_context.consent` -with a locally-decoded fallback consent context for that request only -when the optional `consent` query param is the sole available signal. +EC route handlers (GET /_ts/api/v1/identify, POST /auction, +POST /_ts/api/v1/batch-sync) NEVER call generate_if_needed(). +`/_ts/api/v1/identify`, `/auction`, and `POST /_ts/api/v1/batch-sync` +use `EcContext` in read-only form. /auction reads EC identity but never bootstraps it — the publisher page-load path generates the EC before any auction request arrives. ec_finalize_response() — after every handler: - - consent withdrawn + cookie present? → clear_ec_on_response() + tombstone - - returning-user mismatch? → set_ec_on_response() [reconcile cookie to header EC] - - ec_generated == true? → set_ec_on_response() [new cookie only] + - !allows_ec_creation(&consent)? → strip EC response headers + - explicit withdrawal + cookie present? → also expire the cookie and write tombstones + - returning user with consent? → set x-ts-ec header only (no cookie/KV TTL refresh) + - ec_generated == true? → set EC cookie + x-ts-ec header + - Prebid EID ingestion: reads `ts-eids` cookie, matches source domains + via PartnerRegistry, writes changed partner UIDs to KV (same UID = no write) ``` EC state flows through an `EcContext` struct created once per request and passed through handlers. @@ -112,23 +135,28 @@ New files in `crates/trusted-server-core/src/`: crates/trusted-server-core/src/ ec/ mod.rs — EcContext, pub re-exports - identity.rs — EC generation (HMAC-SHA256, IP normalization) - cookie.rs — create_ec_cookie(), delete_ec_cookie(), set_ec_on_response() - finalize.rs — ec_finalize_response() (cookie write/delete, last_seen, tombstone) - kv.rs — KvIdentityGraph, read/write/delete identity entries - partner.rs — PartnerRecord, PartnerStore, load_partner() - sync_pixel.rs — handle_sync() handler - sync_batch.rs — handle_batch_sync() handler + generation.rs — EC generation (HMAC-SHA256, IP normalization) + cookies.rs — set_ec_cookie(), expire_ec_cookie() + consent.rs — EC consent gating helpers + device.rs — DeviceSignals derivation, UA/JA4/H2 parsing, known browser allowlist + eids.rs — OpenRTB EID construction helpers + finalize.rs — ec_finalize_response() (cookie write/delete, tombstone, EID ingestion) + kv.rs — KvIdentityGraph, read/write/delete identity entries, cluster evaluation + kv_types.rs — KvEntry, KvGeo, KvConsent, KvPubProperties, KvNetwork, KvDevice, KvMetadata + partner.rs — Partner validation helpers (ID format, API key hashing) + registry.rs — PartnerRegistry (in-memory, config-based, O(1) indexes) + rate_limiter.rs — RateLimiter trait and Fastly ERL implementation + prebid_eids.rs — ingest_prebid_eids() — ts-eids cookie parsing and KV sync + batch_sync.rs — handle_batch_sync() handler pull_sync.rs — PullSyncDispatcher, dispatch_background() identify.rs — handle_identify() handler - admin.rs — handle_register_partner() handler ``` Existing files modified: | File | Change | | -------------------------------------------------- | ----------------------------------------------------- | -| `crates/trusted-server-core/src/settings.rs` | Add `EdgeCookie` settings struct | +| `crates/trusted-server-core/src/settings.rs` | Add `Ec` and `EcPartner` settings structs | | `crates/trusted-server-core/src/constants.rs` | Add EC header/cookie name constants | | `crates/trusted-server-core/src/error.rs` | Add `EdgeCookie` error variant | | `crates/trusted-server-core/src/auction/` | Inject EC into `user.id`, `user.eids`, `user.consent` | @@ -138,7 +166,7 @@ Existing files modified: ## 4. EC Identity Generation -### 4.1 Module: `ec/identity.rs` +### 4.1 Module: `ec/generation.rs` The EC generation mirrors the SyntheticID approach (`synthetic.rs`) but strips volatile inputs. @@ -161,9 +189,12 @@ pub fn generate_ec(passphrase: &str, ip: IpAddr) -> Result String; -/// Extracts the stable 64-character hex prefix from a full EC value. +/// Extracts the stable 64-character hex prefix from a full EC ID. /// -/// The prefix is used as the KV store key. The `.suffix` is discarded. +/// This is primarily used for logging and debugging. Both the EC identity +/// EC identity KV operations use the **full EC ID** (including the +/// `.suffix`) as the key, not just this prefix. The suffix provides uniqueness +/// for users behind the same NAT/proxy infrastructure. /// /// Returns `None` if the value is not in `{64-hex}.{6-alnum}` format. pub fn ec_hash(ec_value: &str) -> Option<&str>; @@ -179,7 +210,7 @@ pub fn ec_hash(ec_value: &str) -> Option<&str>; **Output format:** `{64-char lowercase hex}.{6-char random alphanumeric}` -The random suffix is generated with `fastly::rand` (same approach as SyntheticID). Once set in a cookie the full value is preserved; only the hash prefix is used as the KV key. +The random suffix is generated with `fastly::rand` (same approach as SyntheticID). Once set in a cookie, the full value (hash + suffix) is preserved and used as the KV store key for the EC identity graph. The suffix provides uniqueness for users behind the same NAT/proxy who share the same IP-derived hash. **IPv6 /64 prefix:** Split on `:`, take first 4 groups, join with `:`. Example: `2001:db8:85a3:0000:0000:8a2e:0370:7334` → `2001:db8:85a3:0`. @@ -203,14 +234,16 @@ When both header and cookie are present, the **header wins** as `ec_value` (used - `ec_value` = header value (authoritative for handler reads) - `cookie_ec_value` = cookie value (tracked separately for withdrawal) -On consent **withdrawal** (`!allows_ec_creation && cookie_was_present`): +On **explicit consent withdrawal** (`has_explicit_ec_withdrawal(&consent) && cookie_was_present`): - Delete the browser cookie (always, based on `cookie_was_present`) - Tombstone the **cookie-derived** hash: `kv.write_withdrawal_tombstone(ec_hash(cookie_ec_value))` - If the header-derived hash differs, also tombstone it: `kv.write_withdrawal_tombstone(ec_hash(ec_value))` - This matches the existing SyntheticID behavior where revocation targets the cookie value (`publisher.rs:515`), not the header value. -On **non-withdrawal** paths (last_seen, handler reads): use `ec_value` (header-derived) as the active identity. When `cookie_ec_value` is set (mismatch), `ec_finalize_response()` overwrites the browser cookie with the header-derived `ec_value` via `set_ec_on_response()`. This reconciles the browser identity to match the publisher-forwarded identity and prevents persistent oscillation between two ECs on subsequent requests. +If `allows_ec_creation(&consent)` is `false` but there is **no explicit withdrawal signal** (for example, unknown jurisdiction or missing/undecodable consent in a regulated regime), the response strips EC-related headers only. It does **not** delete the cookie or write tombstones. + +On **non-withdrawal** paths (handler reads and response headers): use `ec_value` (header-derived) as the active identity. Returning-user responses set `x-ts-ec` for the active identity but do not refresh or repair the browser cookie. Cookie writes are reserved for newly generated ECs; cookie deletion is reserved for explicit consent withdrawal. **Validation:** Both the header and cookie values are validated independently via `ec_hash()` (`{64-hex}.{6-alnum}` format). If the header is present but malformed, it is discarded and the cookie value is used instead (if valid). A malformed header must not suppress a valid cookie — bad forwarding infrastructure should not break returning-user identity. `cookie_was_present` is set based on the raw cookie existing, regardless of validity — an invalid cookie value is still a cookie that needs to be cleared on withdrawal. @@ -220,18 +253,16 @@ Generation (step 3 above becoming a new EC) happens only inside organic handlers ```rust /// Per-request Edge Cookie state. Constructed pre-routing once per request. -/// Organic handlers call `generate_if_needed()` to mint new ECs. `/sync` is the -/// one EC route that may replace `consent` with a locally-decoded fallback for -/// the remainder of that request only. +/// Organic handlers call `generate_if_needed()` to mint new ECs. pub struct EcContext { - /// Full EC value (`hash.suffix`), if present on request or generated this request. + /// Full EC ID (`{64-hex}.{6-alnum}`), if present on request or generated this request. pub ec_value: Option, /// Whether the `ts-ec` **cookie** was present on the inbound request. /// This is the only field that gates consent-withdrawal cookie deletion — /// the PRD's delete branch is conditioned on the cookie, not on X-ts-ec header. pub cookie_was_present: bool, /// The cookie's EC value, if different from `ec_value` (header won priority). - /// Used only for withdrawal: tombstone targets the cookie-derived hash to match + /// Used only for withdrawal: tombstone targets the cookie-derived EC ID to match /// existing SyntheticID revocation behavior (`publisher.rs:515`). /// `None` when cookie absent or cookie == header value. pub cookie_ec_value: Option, @@ -249,6 +280,11 @@ pub struct EcContext { /// Stored here so pull sync can use it after `req` has been consumed by routing. /// `None` only if Fastly's `get_client_ip_addr()` returns `None`. pub client_ip: Option, + /// Device signals derived from TLS/H2/UA in the adapter layer. + /// Set via `set_device_signals()` after `read_from_request()` returns. + /// Converted to `KvDevice` and stored on new entries in `generate_if_needed()`. + /// `None` when the adapter does not provide signals (e.g., test environments). + pub device_signals: Option, } impl EcContext { @@ -256,16 +292,9 @@ impl EcContext { /// Does not write to the **EC identity KV store**. Called pre-routing, like /// `GeoInfo::from_request()` in the current `main.rs`. /// - /// Calls `build_consent_context()` with the EC hash (when present) passed - /// via `ConsentPipelineInput.ec_id` (renamed from `synthetic_id` - /// in PR #479). - /// - /// When an EC hash is available (returning user), this enables the consent - /// pipeline's KV fallback (read) and KV persistence (write to the - /// **consent** KV store). On a first visit (no EC cookie), `ec_hash` is - /// `None` and no consent KV interaction occurs; consent is evaluated purely - /// from request cookies/headers. This means consent is not persisted to - /// consent KV until the user's second request. See §6.1.1. + /// Calls `build_consent_context()` with request-local cookies, headers, + /// settings, and geo data. There is no separate consent KV fallback; live + /// consent is interpreted from the current request. pub fn read_from_request( req: &Request, settings: &Settings, @@ -292,19 +321,30 @@ impl EcContext { kv: &KvIdentityGraph, ); + /// Sets device signals derived from the adapter layer (TLS/H2/UA). + /// Must be called before `generate_if_needed()` so new entries include `KvDevice`. + pub fn set_device_signals(&mut self, signals: DeviceSignals); + + /// Returns the device signals, if set. + pub fn device_signals(&self) -> Option<&DeviceSignals>; + /// Returns the stable 64-char hex prefix, or `None` if no EC. + /// + /// Note: This extracts only the prefix for display/logging purposes. All KV + /// operations use the full EC ID (via `ec_value()`), not just this hash. pub fn ec_hash(&self) -> Option<&str>; } ``` -**`ec_finalize_response()` behavior** (signature: `ec_finalize_response(settings, geo, ec_context, kv, response)`): +**`ec_finalize_response()` behavior** (signature: `ec_finalize_response(settings, ec_context, kv, registry, eids_cookie, response)`): -1. If `!allows_ec_creation(&consent) && cookie_was_present`: call `clear_ec_on_response()` (deletes cookie **and** strips any handler-built `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, and `X-ts-` response headers) and write withdrawal tombstones for each valid known EC hash (cookie-derived and, when different, header-derived). This runs on **every route** — consent withdrawal is always real-time enforced. Keyed on `cookie_was_present`, not `ec_was_present`, because only a cookie-held EC can be deleted by the browser. When the cookie is malformed and there is no valid header-derived hash, no tombstone is written. -2. If `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)`: call `kv.update_last_seen()` (debounced). If `cookie_ec_value.is_some()`, also call `set_ec_on_response()` to reconcile the browser cookie to the authoritative header-derived EC. -3. If `ec_generated == true`: call `set_ec_on_response()` — sets `Set-Cookie` and `X-ts-ec`. KV create already happened inside `generate_if_needed()`; `ec_finalize_response()` does NOT write KV beyond tombstones and `last_seen`. -4. Handler-built response headers (`X-ts-ec`, `X-ts-eids` set directly by `/identify`) are preserved on non-withdrawal paths only. +1. If `!allows_ec_creation(&consent)`: call `clear_ec_headers_on_response()` to strip any handler-built `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, and `X-ts-` response headers. This runs on **every route**, including fail-closed cases where consent cannot be verified. +2. If `has_explicit_ec_withdrawal(&consent) && cookie_was_present`: additionally expire the cookie and write withdrawal tombstones for each valid known EC ID (cookie-derived and, when different, header-derived). Keyed on `cookie_was_present`, not `ec_was_present`, because only a cookie-held EC can be deleted by the browser. When the cookie is malformed and there is no valid header-derived EC ID, no tombstone is written. +3. If `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)`: ingest Prebid EIDs from the `ts-eids` cookie if present (see section 8) and set the `x-ts-ec` response header only. Ordinary returning-user requests do not refresh the EC cookie and do not write KV solely to extend TTL. +4. If `ec_generated == true`: set `Set-Cookie` and `X-ts-ec`. KV create already happened inside `generate_if_needed()`; `ec_finalize_response()` does NOT write KV beyond explicit-withdrawal tombstones and Prebid EID ingestion. Also ingest Prebid EIDs from the `ts-eids` cookie if present. +5. Handler-built response headers (`X-ts-ec` set directly by `/_ts/api/v1/identify`) are preserved only when consent currently allows EC. -**Note on `kv_degraded`:** Not on `EcContext` — `read_from_request()` does not read KV. Handlers track degraded state locally. `/identify` returns `degraded: true` in the JSON body on KV read failure; the auction handler treats a failed read as `eids: []`. +**Note on `kv_degraded`:** Not on `EcContext` — `read_from_request()` does not read KV. Handlers track degraded state locally. `/_ts/api/v1/identify` returns `degraded: true` in the JSON body on KV read failure; the auction handler treats a failed read as `eids: []`. ```` @@ -324,7 +364,7 @@ impl EcContext { | Max-Age | `31536000` (1 year) | | HttpOnly | No | -### 5.2 Module: `ec/cookie.rs` +### 5.2 Module: `ec/cookies.rs` The `cookie_domain` parameter passed to all functions below is computed as `format!(".{}", settings.publisher.domain)`. Do **not** use @@ -341,24 +381,52 @@ pub fn create_ec_cookie(ec_value: &str, cookie_domain: &str) -> String; pub fn delete_ec_cookie(cookie_domain: &str) -> String; // Sets Max-Age=0 with same Domain/Path/Secure/SameSite attributes. +/// Sets only the `X-ts-ec` response header on a response. +pub fn set_ec_header_on_response(response: &mut Response, ec_value: &str); + /// Sets the EC cookie and `X-ts-ec` response header on a response. -pub fn set_ec_on_response(response: &mut Response, ec_value: &str, cookie_domain: &str); +pub fn set_ec_cookie_and_header_on_response(response: &mut Response, ec_value: &str, cookie_domain: &str); /// Removes the EC cookie and strips all EC-related response headers: /// `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, -/// and any `X-ts-` headers. Called on consent withdrawal to -/// prevent leaking EC identity in handler-built headers. +/// and any `X-ts-` headers. Called on explicit consent +/// withdrawal to prevent leaking EC identity in handler-built headers. pub fn clear_ec_on_response(response: &mut Response, cookie_domain: &str); ```` ### 5.3 Response header -`X-ts-ec: {ec_hash.suffix}` is set by `set_ec_on_response()`, which is called by `ec_finalize_response()` in two cases: (1) `ec_generated == true` (new EC minted this request), or (2) `cookie_ec_value.is_some()` (header/cookie mismatch reconciliation — overwrites cookie to match header). It is also set explicitly by `/identify` and `/auction` handlers on their own response paths when an EC is present. It is **not** set on ordinary returning-user requests where the cookie already matches the header (or no header is present). +`X-ts-ec: {64-hex}.{6-alnum}` is set when an EC is available for the response. In current behavior, returning users (`ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)`) receive the header only; newly generated ECs (`ec_generated == true`) receive both the header and `Set-Cookie`. `/_ts/api/v1/identify` and `/auction` also set EC-related headers on their response paths. This header is added to `INTERNAL_HEADERS` in `constants.rs` so it is stripped before proxying to downstream backends, consistent with existing `X-ts-*` handling. ### 5.4 Per-request EC lifecycle +**Phase 0 — bot gate** (always runs, all routes — pure in-memory, no KV I/O): + +``` +derive_device_signals(req) + ua = req.get_header_str("user-agent") + ja4 = req.get_tls_ja4() // Fastly SDK — full JA4 hash + h2_fp = req.get_client_h2_fingerprint() // Fastly SDK — raw H2 SETTINGS string + + DeviceSignals::derive(ua, ja4, h2_fp) + is_mobile = parse_is_mobile(ua) // 0=desktop, 1=mobile, 2=unknown + ja4_class = extract_ja4_section1(ja4) // split on '_', take [0] + platform_class = parse_platform_class(ua) // mac/windows/ios/android/linux/None + h2_fp_hash = sha256(h2_fp)[..6].hex() // 12 hex chars + known_browser = evaluate_known_browser(ja4_class, h2_fp_hash) // allowlist match + + is_real_browser = looks_like_browser() // ja4_class.is_some() && platform_class.is_some() + + if !is_real_browser: + log::debug("Bot gate: blocking EC operations") + kv_graph = None // suppress all KV operations + // ec_finalize_response() will be skipped + // pull sync will be skipped + // request still proxied to origin normally +``` + **Phase 1 — pre-routing** (always runs, all routes): ``` @@ -371,12 +439,13 @@ EcContext::read_from_request() If neither valid: ec_value = None ec_was_present = ec_value.is_some() cookie_was_present = ts-ec cookie raw key exists (regardless of validity) - ec_hash = ec_value.as_deref().and_then(ec_hash) // None on first visit or malformed - build_consent_context(jar, req, config, geo, ec_hash) → consent: ConsentContext - // ec_hash is the identity key for consent KV (renamed from synthetic_id in PR #479). - // When ec_hash is Some: consent KV fallback read + consent KV write (to consent store, not EC store). - // When ec_hash is None (first visit): no consent KV interaction — cookies/headers only. + ec_id = ec_value.as_deref() // None on first visit or malformed + build_consent_context(jar, req, config, geo, ec_id) → consent: ConsentContext + // Consent is interpreted from request-local cookies, headers, settings, and geo. + // No separate consent KV fallback or persistence runs in the EC lifecycle. ec_generated = false + + ec_context.set_device_signals(device_signals) // for KvDevice on creation ``` **Phase 2 — inside organic handlers only** (`handle_publisher_request`, `handle_proxy`): @@ -389,41 +458,42 @@ ec_context.generate_if_needed(settings, &kv) // best-effort — never 500s → generate_ec(passphrase, ip) → ec_value = Some(new_ec) → ec_generated = true - → kv.create_or_revive(ec_hash, &entry) (best-effort, log warn if fails) + → kv.create_or_revive(new_ec, &entry) (best-effort, log warn if fails) // create_or_revive overwrites a tombstone (ok=false) on re-consent // no-ops if a live entry (ok=true) already exists ``` -**`ec_finalize_response(settings, geo, ec_context, &kv, response)` — always runs, all routes:** +**`ec_finalize_response(settings, geo, ec_context, &kv, response)` — runs only when `is_real_browser == true`:** ``` - ├── !allows_ec_creation(&consent) && cookie_was_present? - │ → clear_ec_on_response() (delete cookie + strip ALL EC headers from response) - │ → // Tombstone all known valid EC hashes. May be 0, 1, or 2 hashes. - │ if let Some(cookie_hash) = cookie_ec_value.and_then(|v| ec_hash(&v)): - │ kv.write_withdrawal_tombstone(cookie_hash) // cookie-derived hash - │ if let Some(header_hash) = ec_value.and_then(|v| ec_hash(&v)): - │ if Some(header_hash) != cookie_hash: - │ kv.write_withdrawal_tombstone(header_hash) // header-derived hash (if different) - │ // When cookie is malformed and no valid header exists: no tombstone written. - │ // Cookie deletion is still the authoritative enforcement mechanism. - │ // Tombstone fails? log error, do NOT block — no retry possible on browser path. + // Bot gate: when !looks_like_browser(), this entire block is skipped. + // The response is proxied to origin without any cookie writes or KV operations. + + ├── !allows_ec_creation(&consent)? + │ → clear_ec_headers_on_response() (strip ALL EC headers from response) + │ → has_explicit_ec_withdrawal(&consent) && cookie_was_present? + │ → expire_ec_cookie() + │ → // Tombstone all known valid EC IDs. May be 0, 1, or 2 IDs. + │ if let Some(cookie_ec_id) = cookie_ec_value.filter(|v| is_valid_ec_id(v)): + │ kv.write_withdrawal_tombstone(cookie_ec_id) // cookie-derived EC ID + │ if let Some(header_ec_id) = ec_value.filter(|v| is_valid_ec_id(v)): + │ if Some(header_ec_id) != cookie_ec_id: + │ kv.write_withdrawal_tombstone(header_ec_id) // header-derived EC ID (if different) + │ // When cookie is malformed and no valid header exists: no tombstone written. + │ // Cookie deletion is still the authoritative enforcement mechanism. + │ // Tombstone fails? log error, do NOT block — no retry possible on browser path. + │ → return │ ├── ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)? - │ → kv.update_last_seen(ec_hash, now()) (returning user — debounced at 300s) - │ → if cookie_ec_value.is_some(): - │ // Header and cookie disagree — reconcile by overwriting cookie with header value. - │ // Prevents persistent split identity where user oscillates between two ECs - │ // depending on whether the forwarded header is present on subsequent requests. - │ set_ec_on_response() (Set-Cookie with ec_value, the header-derived identity) + │ → set_ec_header_on_response() (returning user — no cookie/KV TTL refresh) │ └── ec_generated == true? - → set_ec_on_response() (Set-Cookie + X-ts-ec on response) + → set_ec_cookie_and_header_on_response() (Set-Cookie + X-ts-ec on response) ``` -EC route handlers (`GET /sync`, `GET /identify`, `POST /api/v1/sync`, `POST /admin/*`) never call `generate_if_needed()`. `ec_finalize_response()` will still delete the cookie on those routes if consent is withdrawn — that is intentional. +EC route handlers (`GET /_ts/api/v1/identify`, `POST /_ts/api/v1/batch-sync`) never call `generate_if_needed()`. `ec_finalize_response()` will still delete the cookie on those routes if consent is explicitly withdrawn — that is intentional. -**Cookie write rule:** `Set-Cookie` is written in exactly two cases: (1) `ec_generated == true` (first-time generation), or (2) `cookie_ec_value.is_some()` (header/cookie mismatch — reconcile cookie to match the header-derived identity). There is no cookie refresh or Max-Age reset on ordinary returning users where cookie already matches. The PRD defers a blanket refresh-on-every-request strategy to a future iteration. +**Cookie write rule:** `Set-Cookie` is written for newly generated ECs and consent-withdrawal deletion only. Ordinary returning requests set `x-ts-ec` but do not refresh the cookie `Max-Age`. --- @@ -442,41 +512,51 @@ Consent decoding shipped in `#380` (already merged). This spec treats the follow ### 6.1.1 EC consent gating EC reuses the existing `allows_ec_creation(&ConsentContext) -> bool` function -from the consent module (`consent/mod.rs`). No parallel gating function is -introduced — EC calls `allows_ec_creation()` directly for all consent decisions -(EC generation, withdrawal detection, sync gating). +from the consent module (`consent/mod.rs`) for EC generation, header emission, +and other "may this request use ECs right now?" decisions. -There is no EC-specific consent gate and no behavior change to -`allows_ec_creation()` in this spec. Shared consent-policy semantics stay in -the consent module; EC only consumes that existing decision. - -**Consent pipeline integration:** +Explicit withdrawal semantics use a separate +`has_explicit_ec_withdrawal(&ConsentContext) -> bool` helper. This narrower +signal distinguishes authoritative opt-outs from fail-closed cases where EC use +must be blocked for the current request but an already-issued EC must not be +revoked (for example, unknown jurisdiction or missing/undecodable consent in a +regulated regime). -`EcContext::read_from_request()` calls `build_consent_context()` with the EC hash as the identity key, passed via `ConsentPipelineInput.ec_id` (renamed from `synthetic_id` in PR #479). The consent pipeline's KV persistence and fallback behavior works with EC hashes: +There is no new consent source or KV lookup in this spec. Shared +consent-policy semantics stay in the consent module; EC consumes the existing +request-local decision plus the explicit-withdrawal helper. -- **Returning user** (EC cookie present → `ec_hash` is `Some`): consent KV fallback read is available when consent cookies are absent; consent KV write persists cookie-sourced consent for future requests. Note: `build_consent_context()` calls `try_kv_write()` internally, so phase 1 writes to the **consent** KV store (not the EC identity store). -- **First visit** (no EC cookie → `ec_hash` is `None`): no consent KV interaction. Consent is evaluated purely from request cookies/headers. The gap: consent is not persisted to consent KV on the first request. This is accepted — in regulated jurisdictions (GDPR, US state), consent cookies/headers must be present for `allows_ec_creation()` to return `true`, so there is always a signal to persist on the next request. In non-regulated jurisdictions, `allows_ec_creation()` returns `true` without consent signals, so there is nothing to persist anyway. Consent KV persistence begins on the second request when the EC cookie is present. +**Consent pipeline integration:** -**Consent store keying:** Old consent KV entries under SyntheticID keys become orphaned after PR #479 ships. New entries are keyed by EC hash. Orphaned entries expire via TTL — no explicit migration is performed. +`EcContext::read_from_request()` calls `build_consent_context()` with request-local cookies, headers, settings, and geo data. Current runtime behavior does not use a separate consent KV store or consent KV fallback. Consent is interpreted from live request signals on every request; the EC identity store only keeps the minimal `KvEntry.consent` snapshot and withdrawal tombstones for S2S enforcement. -**Rollout impact:** At cutover, returning users who relied on consent KV fallback (consent cookies absent, consent loaded from KV under SyntheticID key) will lose that fallback until a new EC-keyed consent entry is written on a subsequent request where consent cookies are present. This is a one-time window: once the EC cookie is set and a request with consent cookies arrives, the consent KV entry is written under the EC hash and fallback works again. The window duration depends on how quickly users return with consent cookies. This is accepted — consent cookies are the primary signal; KV fallback is a secondary mechanism for when cookies are blocked or absent. +All downstream EC logic uses `allows_ec_creation(&self.consent)` for creation/forwarding decisions and `has_explicit_ec_withdrawal(&self.consent)` for cookie-expiry/tombstone decisions. No consent decoding or KV-backed gating logic is added in this epic. -All downstream EC logic calls `allows_ec_creation(&self.consent)`. No consent decoding or gating logic is added in this epic. +### 6.2 Consent withdrawal — explicit delete path -### 6.2 Consent withdrawal — KV delete +When `allows_ec_creation(&consent)` returns `false`, Trusted Server **always** +strips EC-related response headers for that request. This covers both explicit +revocation and fail-closed cases. -When `allows_ec_creation(&consent)` returns `false` for a user whose **`ts-ec` cookie** is present (`cookie_was_present == true`). A user identified only by the `X-ts-ec` request header is not subject to cookie deletion — there is no cookie to expire. +Cookie expiry and tombstone writes happen only when +`has_explicit_ec_withdrawal(&consent)` returns `true` **and** the request +carried a **`ts-ec` cookie** (`cookie_was_present == true`). A user identified +only by the `X-ts-ec` request header is not subject to cookie deletion or +`tombstoning` on this path — there is no browser cookie to revoke. -1. Issue `Set-Cookie: ts-ec=; Max-Age=0; ...` and strip all EC response headers (synchronous — must not fail silently). This always happens when `cookie_was_present == true`. -2. Write tombstone for each valid EC hash available (`cookie_ec_value` and/or `ec_value`). When neither is valid (malformed cookie, no header), **no tombstone is written** — cookie deletion alone is the enforcement mechanism. When at least one valid hash exists: `kv.write_withdrawal_tombstone(hash)` sets `consent.ok = false`, clears partner IDs, TTL 24h — approximately 25ms per write. +1. Strip all EC response headers (synchronous — must not fail silently) whenever `!allows_ec_creation(&consent)`. +2. If `has_explicit_ec_withdrawal(&consent) && cookie_was_present == true`, issue `Set-Cookie: ts-ec=; Max-Age=0; ...`. +3. In that same explicit-withdrawal + cookie-present case, write a tombstone for each valid EC ID available (`cookie_ec_value` and/or `ec_value`). When neither is valid (malformed cookie, no header), **no tombstone is written** — cookie deletion alone is the browser-side enforcement mechanism. When at least one valid EC ID exists: `kv.write_withdrawal_tombstone(ec_id)` sets `consent.ok = false`, clears partner IDs, TTL 24h — approximately 25ms per write. -The tombstone write runs in the request path (not async) to ensure real-time enforcement. Using a tombstone rather than a hard delete preserves the `consent_withdrawn` signal for batch sync clients for 24 hours — otherwise batch sync cannot distinguish consent withdrawal from an EC that never existed. +The tombstone write runs in the request path (not async) to ensure real-time enforcement for authoritative withdrawals. Using a tombstone rather than a hard delete preserves the `consent_withdrawn` signal for batch sync clients for 24 hours — otherwise batch sync cannot distinguish consent withdrawal from an EC that never existed. If the tombstone write fails: -- Log at `error` level with EC hash -- Do not block the response — cookie deletion is the primary enforcement mechanism -- **No retry is possible on the browser path.** Once the cookie is deleted, subsequent browser requests carry no EC value (`ec_hash()` returns `None`), so there is no hash to tombstone. A failed tombstone means batch sync clients may see `ec_hash_not_found` (after TTL expiry) rather than `consent_withdrawn` — this is accepted degradation. The cookie deletion remains the authoritative enforcement mechanism. +- Log at `error` level with EC ID +- Do not block the response — cookie deletion is the primary enforcement mechanism on explicit-withdrawal paths +- **No retry is possible on the browser path.** Once the cookie is deleted, subsequent browser requests carry no EC value (`ec_value` returns `None`), so there is no EC ID to tombstone. A failed tombstone means batch sync clients may see `ec_id_not_found` (after TTL expiry) rather than `consent_withdrawn` — this is accepted degradation. + +Fail-closed / unverifiable-consent cases keep the cookie intact and do not write tombstones; they only suppress EC use on that request. --- @@ -484,37 +564,56 @@ If the tombstone write fails: ### 7.1 Module: `ec/kv.rs` -Two KV stores are used. Their names are configured in `trusted-server.toml`: +One KV store is used for the identity graph. Its name is configured in `trusted-server.toml`: -| Store | TOML key | Purpose | -| ---------------- | ------------------ | ---------------------------------- | -| Identity graph | `ec.ec_store` | EC hash → identity JSON | -| Partner registry | `ec.partner_store` | Partner ID → config + API key hash | +| Store | TOML key | Purpose | +| -------------- | ------------- | --------------------- | +| Identity graph | `ec.ec_store` | EC ID → identity JSON | + +Partners are defined in config (`[[ec.partners]]` in TOML) and loaded into an in-memory `PartnerRegistry` at startup. There is no KV-backed partner store. ### 7.2 Identity graph schema -**KV key:** 64-character hex hash (the stable prefix from `ec_value`, without `.suffix`). +**KV key:** Full EC ID in `{64-char hex}.{6-char alphanumeric}` format. The random suffix is intentionally included to provide uniqueness for users behind the same NAT/proxy infrastructure who would otherwise share identical IP-derived hash prefixes. **KV value (JSON, max ~5KB):** ```json { "v": 1, - "created": 1741824000, - "last_seen": 1741910400, + "created": 1775162556, "consent": { "tcf": "CP...", "gpp": "DBA...", "ok": true, - "updated": 1741910400 + "updated": 1775162556 }, "geo": { "country": "US", - "region": "CA" + "region": "TN", + "asn": 7922, + "dma": 659 + }, + "device": { + "is_mobile": 0, + "ja4_class": "t13d1516h2", + "platform_class": "mac", + "h2_fp_hash": "a3f9d21c8b04", + "known_browser": true + }, + "pub_properties": { + "origin_domain": "autoblog.com", + "seen_domains": { + "autoblog.com": { "visits": 1 } + } + }, + "network": { + "cluster_size": 2 }, "ids": { - "ssp_x": { "uid": "abc123", "synced": 1741824000 }, - "liveramp": { "uid": "LR_xyz", "synced": 1741890000 } + "id5": { "uid": "ID5*qe8VHv..." }, + "trade_desk": { "uid": "226fb4b3-..." }, + "liveramp_ats": { "uid": "Ag2z1TDA..." } } } ``` @@ -522,12 +621,19 @@ Two KV stores are used. Their names are configured in `trusted-server.toml`: **KV metadata (max 2048 bytes, readable without streaming body):** ```json -{ "ok": true, "country": "US", "v": 1 } +{ + "ok": true, + "country": "US", + "v": 1, + "cluster_size": 2, + "is_mobile": 0, + "known_browser": true +} ``` -The `ok` field in metadata is a **historical consent record for S2S consumers only** — it is set to `false` by `write_withdrawal_tombstone()` so that batch sync clients (`POST /api/v1/sync`) can return `consent_withdrawn` rather than `ec_hash_not_found` during the 24-hour tombstone TTL. +The `ok` field in metadata is a **historical consent record for S2S consumers only** — it is set to `false` by `write_withdrawal_tombstone()` so that batch sync clients (`POST /_ts/api/v1/batch-sync`) can return `consent_withdrawn` rather than `ec_id_not_found` during the 24-hour tombstone TTL. -**`consent.ok` is NOT used to make the withdrawal decision on the main request path.** Consent withdrawal is determined entirely from `allows_ec_creation(&ec_context.consent)` on the current request. When withdrawal is detected, the cookie is deleted and `write_withdrawal_tombstone()` is called in-path (setting `ok = false`, 24h TTL — see §6.2). Engineers must not add a KV read to the consent withdrawal hot path based on this field. +**`consent.ok` is NOT used to make the withdrawal decision on the main request path.** Withdrawal enforcement is driven by current request-local consent: `allows_ec_creation(&ec_context.consent)` decides whether EC use and EC response headers are allowed on this request, and `has_explicit_ec_withdrawal(&ec_context.consent)` decides whether to expire the cookie and call `write_withdrawal_tombstone()` in-path (setting `ok = false`, 24h TTL — see §6.2). Engineers must not add a KV read to the consent withdrawal hot path based on this field. **Rust types:** @@ -535,9 +641,18 @@ The `ok` field in metadata is a **historical consent record for S2S consumers on pub struct KvEntry { pub v: u8, pub created: u64, - pub last_seen: u64, pub consent: KvConsent, pub geo: KvGeo, + /// Creation-time publisher property metadata. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pub_properties: Option, + /// Device class signals. Written once on creation — never updated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device: Option, + /// Network cluster disambiguation. Written only by /identify. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub ids: HashMap, } @@ -550,24 +665,86 @@ pub struct KvConsent { pub struct KvGeo { pub country: String, + #[serde(default, skip_serializing_if = "Option::is_none")] pub region: Option, + /// Autonomous System Number (e.g. 7922 = Comcast). + /// Primary signal for distinguishing home ISP vs. corporate VPN. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asn: Option, + /// DMA/metro code (e.g. 807 = San Francisco). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dma: Option, } pub struct KvPartnerId { pub uid: String, - pub synced: u64, +} + +/// Publisher property metadata captured when an EC entry is created. +pub struct KvPubProperties { + /// Apex domain where this EC entry was first created. + pub origin_domain: String, + /// Per-domain visit history, keyed by apex domain. + /// Capped at 50 entries; new domains silently dropped at cap. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub seen_domains: HashMap, +} + +pub struct KvDomainVisit { + /// Legacy visit count retained for schema compatibility. + pub visits: u32, +} + +/// Coarse device signals derived from TLS handshake and UA. +/// Written once on creation — never updated after. +pub struct KvDevice { + /// 0 = desktop, 1 = mobile, 2 = unknown (non-standard client). + pub is_mobile: u8, + /// JA4 Section 1 only (e.g. "t13d1516h2" = Chrome). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ja4_class: Option, + /// Coarse OS family: "mac", "windows", "ios", "android", "linux". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform_class: Option, + /// SHA256 prefix (12 hex chars) of H2 SETTINGS fingerprint. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub h2_fp_hash: Option, + /// true = known browser, false = known bot, None = unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub known_browser: Option, +} + +/// Network cluster disambiguation data. +/// Written only by /identify — too expensive for organic hot path. +pub struct KvNetwork { + /// Number of distinct EC suffixes sharing this hash prefix. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cluster_size: Option, } pub struct KvMetadata { pub ok: bool, pub country: String, pub v: u8, + /// Mirrors KvNetwork::cluster_size. None = not yet evaluated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cluster_size: Option, + /// Mirrors KvDevice::is_mobile. Enables propagation gating without body read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub is_mobile: Option, + /// Mirrors KvDevice::known_browser. Buyer-facing quality signal. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub known_browser: Option, } ``` +All new fields use `Option` types or `serde(default)`, so existing entries +deserialize without error. No schema version bump is needed — v1 has not +shipped yet. + ### 7.3 TTL -All KV writes use `time_to_live_sec = 31536000` (1 year), matching the cookie `Max-Age`. +New live entries use `time_to_live_sec = 31536000` (1 year), matching the initial cookie `Max-Age`. Ordinary returning-user page views do not refresh the EC cookie and do not write the KV entry solely to extend TTL. Real data mutations (for example, a changed partner UID or first cluster-size evaluation) still write the live entry with the live-entry TTL. Withdrawal tombstones use a 24-hour TTL. ### 7.4 Conflict resolution — atomic read-modify-write @@ -584,22 +761,34 @@ impl KvIdentityGraph { pub fn new(store_name: impl Into) -> Self; /// Reads the full entry, returning the generation marker for CAS writes. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn get( &self, - ec_hash: &str, + ec_id: &str, ) -> Result, Report>; /// Reads only the metadata fields (consent flag, country). + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn get_metadata( &self, - ec_hash: &str, + ec_id: &str, ) -> Result, Report>; /// Creates a new entry. Returns `Ok(())` if successful, `Err` if the key /// already exists (concurrent create) or on KV error. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn create( &self, - ec_hash: &str, + ec_id: &str, entry: &KvEntry, ) -> Result<(), Report>; @@ -615,9 +804,13 @@ impl KvIdentityGraph { /// Called by `generate_if_needed()` instead of `create()`. This ensures that /// re-consent recovery is immediate — a user who withdraws and then re-consents /// within the 24-hour tombstone window gets a fresh identity entry without delay. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn create_or_revive( &self, - ec_hash: &str, + ec_id: &str, entry: &KvEntry, ) -> Result<(), Report>; @@ -627,214 +820,491 @@ impl KvIdentityGraph { /// /// If the key does not exist, creates a minimal live entry first: /// `consent.ok = true`, `consent.tcf = None`, `consent.gpp = None`, - /// `created = synced`, `last_seen = synced`, `geo.country = "ZZ"`, - /// `geo.region = None`, and `ids = { partner_id: ... }`. + /// `created = now`, `geo.country = "ZZ"`, `geo.region = None`, + /// and `ids = { partner_id: ... }`. /// /// This recovery path is intentional: it materializes the graph later when /// the initial best-effort `create_or_revive()` on EC generation failed. /// Batch sync still performs its explicit existence/tombstone check before - /// calling this method, so `POST /api/v1/sync` retains its `ec_hash_not_found` + /// calling this method, so `POST /_ts/api/v1/batch-sync` retains its `ec_id_not_found` /// contract. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn upsert_partner_id( &self, - ec_hash: &str, + ec_id: &str, partner_id: &str, uid: &str, - synced: u64, ) -> Result<(), Report>; - /// Updates `last_seen` timestamp, but only if the stored value is more than - /// 300 seconds older than `timestamp`. This debounce prevents KV write - /// thrashing under bursty traffic — Fastly KV enforces a 1 write/sec limit - /// per key. Callers should log `warn` on failure and continue. - pub fn update_last_seen( + /// Upserts a partner ID only when the KV entry already exists. Used by + /// S2S batch sync. Returns `Unchanged` when the existing UID matches, + /// avoiding a KV write. Different UIDs overwrite the stored value; mapping + /// timestamps are not used for ordering because they are no longer stored + /// in the EC identity entry. + pub fn upsert_partner_id_if_exists( &self, - ec_hash: &str, - timestamp: u64, - ) -> Result<(), Report>; + ec_id: &str, + partner_id: &str, + uid: &str, + ) -> Result>; + + /// Counts the number of KV keys sharing a hash prefix via the list API. + /// Uses a single-page list with `limit(100)`. Returns the count, or + /// `None` if the list exceeds 100 keys (clearly a large network). + pub fn count_hash_prefix_keys( + &self, + hash_prefix: &str, + ) -> Result, Report>; + + /// Evaluates the network cluster size for an EC entry. + /// + /// Returns a stored `cluster_size` without a list call when present. If + /// missing, calls `count_hash_prefix_keys()` and writes the result to + /// `entry.network` via CAS. Returns the cluster size for inclusion in + /// the `/_ts/api/v1/identify` response. + pub fn evaluate_cluster( + &self, + ec_id: &str, + entry: &KvEntry, + generation: u64, + ) -> Result, Report>; /// Writes a withdrawal tombstone for consent enforcement. /// /// Instead of hard-deleting the KV entry, this overwrites it with /// `consent.ok = false`, clears all partner IDs, and sets a 24-hour TTL. - /// The tombstone allows batch sync clients (`POST /api/v1/sync`) to return - /// `consent_withdrawn` rather than `ec_hash_not_found` for the tombstone TTL. + /// The tombstone allows batch sync clients (`POST /_ts/api/v1/batch-sync`) to return + /// `consent_withdrawn` rather than `ec_id_not_found` for the tombstone TTL. /// /// After the 24-hour TTL expires, the entry is gone. Any subsequent `get()` - /// returns `None` (`ec_hash_not_found`) — the distinction is time-bounded. + /// returns `None` (`ec_id_not_found`) — the distinction is time-bounded. /// /// Caller must handle `Err` by logging at `error` level; the cookie deletion /// in `ec_finalize_response()` is the primary enforcement mechanism. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn write_withdrawal_tombstone( &self, - ec_hash: &str, + ec_id: &str, ) -> Result<(), Report>; /// Hard-deletes the entry. Used only for data deletion requests (IAB deletion /// framework — deferred). For consent withdrawal, use `write_withdrawal_tombstone()`. - pub fn delete(&self, ec_hash: &str) -> Result<(), Report>; + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. + pub fn delete(&self, ec_id: &str) -> Result<(), Report>; } ``` -`MAX_CAS_RETRIES = 3`. If all retries fail on a generation conflict, return `Err` — callers handle per-endpoint policy (§8.3 step 7 for pixel sync, §9.4 for batch sync). +`MAX_CAS_RETRIES = 5`. If all retries fail on a generation conflict, return `Err` — callers handle per-endpoint policy (§9.4 for batch sync, §8.4 for Prebid EID ingestion). ### 7.5 KV degraded behavior | Operation | KV unavailable | Action | | ---------------------------------- | -------------- | ---------------------------------------------------------------------------------------------- | | EC cookie creation | KV error | Set cookie. Skip KV create. Log `warn`. | -| `/sync` KV write | KV error | Redirect with `ts_synced=0&ts_reason=write_failed`. | -| `/identify` KV read | KV error | Return `200` with `ec` set, `degraded: true`, empty `uids`/`eids`. | -| `POST /api/v1/sync` | KV error | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | +| Prebid EID ingestion KV write | KV error | Skip write. Log `warn`. Retry on next qualifying request. | +| `/_ts/api/v1/identify` KV read | KV error | Return `200` with `ec` set, `degraded: true`, empty `uid`/`eid`. | +| `POST /_ts/api/v1/batch-sync` | KV error | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | | Pull sync KV write | KV error | Discard uid. Log `warn`. Retry on next qualifying request. | | Consent withdrawal tombstone write | KV error | Delete cookie (primary enforcement). Log `error`. Next request: no cookie → no EC regenerated. | --- -## 8. Pixel Sync Endpoint (`GET /sync`) +## 7A. Device Signals and Bot Gate + +### 7A.1 Overview + +Device signals provide coarse, non-PII browser classification derived from +the TLS handshake and User-Agent header at the Fastly edge. They serve two +purposes: + +1. **Bot gate** — block all KV identity operations for unrecognized clients + (bots, scrapers, non-standard HTTP clients). The request is still proxied + to the publisher origin normally — the bot receives valid HTML but leaves + no trace in the identity graph. +2. **Device class record** — store a write-once `KvDevice` on each EC entry + for future cross-browser propagation decisions and buyer-facing device + quality scoring. -### 8.1 Module: `ec/sync_pixel.rs` +All signal derivation is pure in-memory computation — no KV I/O. It runs on +every request before EC context creation. + +### 7A.2 Signal derivation + +No Client Hints are used — JA4 and UA platform parsing provide equivalent or +superior signal for every browser including Safari and Firefox, which do not +send Client Hints. + +**`is_mobile`** — derived in priority order: + +| Condition | Value | +| ---------------------------------------------- | -------------------------------------------------------------------------- | +| UA contains `iPhone`, `iPad`, or `Android` | `1` — confirmed mobile | +| UA contains `Macintosh`, `Windows`, or `Linux` | `0` — confirmed desktop | +| Neither pattern matches | `2` — genuinely unknown (rare; typically bots or heavily hardened clients) | + +Note: `is_mobile: 2` in practice signals a non-standard client rather than +Safari, since Safari always produces a recognizable UA platform string. + +**`platform_class`** — coarse OS family parsed from UA (checked in order): + +| UA segment | `platform_class` | +| ---------------------------------- | ---------------- | +| `iPhone` or `iPad` | `ios` | +| `Android` (checked before `Linux`) | `android` | +| `Macintosh` | `mac` | +| `Windows NT` | `windows` | +| `Linux` (non-Android) | `linux` | +| No match | `None` | + +**`ja4_class`** — Section 1 of the JA4 fingerprint only (e.g. `t13d1516h2`). +Available via `req.get_tls_ja4()` in the Fastly Compute Rust SDK. The full +JA4 format is `section1_section2_section3` separated by underscores; we split +on `_` and take `[0]`. Section 1 identifies browser family (cipher count, +extension count, ALPN) without uniquely fingerprinting a device. The full JA4 +is never stored. + +**`h2_fp_hash`** — first 12 hex characters of SHA256 of the raw HTTP/2 +SETTINGS fingerprint string, available via `req.get_client_h2_fingerprint()`. +Used alongside `ja4_class` to confirm browser family and detect bots. + +**`known_browser`** — set `true` when `ja4_class` + `h2_fp_hash` match a +known legitimate browser pattern from the allowlist below. `None` when +unknown. Both signals must be present for a match — if either is `None`, +returns `None`. + +### 7A.3 Known browser fingerprint allowlist + +Empirically derived from Fastly Compute production responses (2026-04-03): + +| Browser | `ja4_class` | `h2_fp` raw string | `known_browser` | +| ----------------------------------- | ------------ | -------------------------------- | --------------- | +| Chrome/Mac (v146) | `t13d1516h2` | `1:65536;2:0;4:6291456;6:262144` | `true` | +| Safari/Mac (v26) + Safari/iOS (v26) | `t13d2013h2` | `2:0;3:100;4:2097152` | `true` | +| Firefox/Mac (v149) | `t13d1717h2` | `1:65536;2:0;4:131072;5:16384` | `true` | + +Safari Mac and Safari iOS share identical TLS/H2 stacks — distinguished only +by `platform_class` (`mac` vs `ios`) and `is_mobile` (`0` vs `1`). + +This allowlist will expand as new browser versions are observed in production. +Entries not matching any allowlist row get `known_browser: None` (not `false`) +unless they match a confirmed bot pattern. + +The allowlist comparison works by hashing the known raw H2 SETTINGS strings +at evaluation time and comparing against the request's `h2_fp_hash`. The list +is small (3 entries) so the cost is negligible. + +### 7A.4 Bot gate behavior + +The bot gate checks for **signal presence** rather than matching against a +hardcoded fingerprint allowlist. Real browsers always produce a valid TLS +fingerprint (`ja4_class`) and a recognizable UA platform string +(`platform_class`). Raw HTTP clients (curl, Python requests, Go net/http, +headless scrapers) typically lack one or both. + +The gate uses `DeviceSignals::looks_like_browser()`: ```rust -pub async fn handle_sync( - settings: &Settings, - kv: &KvIdentityGraph, - partner_store: &PartnerStore, - req: &Request, - ec_context: &mut EcContext, -) -> Result>; +pub fn looks_like_browser(&self) -> bool { + self.ja4_class.is_some() && self.platform_class.is_some() +} +``` + +| Condition | EC operations | Example | +| ------------------------------------------------ | ------------- | -------------------------------- | +| `ja4_class` present AND `platform_class` present | **Allowed** | Any real browser on any OS | +| `ja4_class` missing OR `platform_class` missing | **Blocked** | curl, Python requests, Googlebot | + +`known_browser` (the fingerprint allowlist match) is still computed and stored +on `KvDevice` for analytics and future buyer-facing quality scoring, but it +does **not** gate identity operations. This avoids blocking legitimate browsers +whose JA4/H2 fingerprints are not yet in the allowlist. + +**Implementation in the Fastly adapter:** + +1. After `GeoInfo::from_request()`, call `derive_device_signals(req)` which + reads `User-Agent`, `req.get_tls_ja4()`, and + `req.get_client_h2_fingerprint()`. +2. If `!looks_like_browser()`: + - `kv_graph` is set to `None` (suppresses all KV reads and writes) + - `ec_finalize_response()` is skipped (no cookie set/deleted) + - Pull sync is skipped + - The request proceeds through normal routing — organic requests are + proxied to publisher origin, API endpoints respond normally (but + without EC identity data) +3. If `looks_like_browser()`: proceed normally. Device signals are set + on `EcContext` via `set_device_signals()` so they flow through to + `KvEntry` creation. + +**Current bot response:** the request is served normally (proxied to origin) +without any KV operations or cookie writes. The bot receives a valid HTML +response but leaves no trace in the identity graph. + +### 7A.5 `DeviceSignals` struct + +```rust +/// Device signals derived from a single request. +/// Computed in the Fastly adapter from raw TLS/H2/UA data. +pub struct DeviceSignals { + pub is_mobile: u8, + pub ja4_class: Option, + pub platform_class: Option, + pub h2_fp_hash: Option, + pub known_browser: Option, +} + +impl DeviceSignals { + /// Derives all device signals from raw request data. + pub fn derive(ua: &str, ja4: Option<&str>, h2_fp: Option<&str>) -> Self; + + /// Returns true when ja4_class and platform_class are both present. + /// Used by the bot gate — see §7A.4. + pub fn looks_like_browser(&self) -> bool; + + /// Converts to KvDevice for KV storage. + pub fn to_kv_device(&self) -> KvDevice; +} ``` -### 8.2 Query parameters +### 7A.6 `KvDevice` write policy + +`KvDevice` is written to `KvEntry.device` only during `generate_if_needed()` +(new EC creation). It is never updated after creation — device signals are a +first-seen record of how this EC entry was established. + +Existing entries (created before device signals were implemented) will have +`device: None`. Downstream consumers must handle `None` as "pre-device-signals +entry" rather than "unknown device." + +### 7A.7 Publisher property metadata (`KvPubProperties`) + +`KvPubProperties` records the publisher domain where the EC entry was created. +Earlier drafts treated `seen_domains` as mutable domain history, but the current +implementation avoids recurring organic-request KV writes. New entries seed only +the creation domain and runtime requests do not append domains or increment +visit counts. The `seen_domains`/`visits` shape remains for compatibility with +legacy records. + +```rust +pub struct KvPubProperties { + pub origin_domain: String, + pub seen_domains: HashMap, +} + +pub struct KvDomainVisit { + pub visits: u32, +} +``` + +**Written:** on `KvEntry::new()` / `create_or_revive()` for the creation domain +only. Ordinary returning-user requests do not update this structure. + +**Cap:** legacy `seen_domains` maps are capped at 50 entries +(`MAX_SEEN_DOMAINS`) during validation so old or malformed records cannot grow +unbounded. -| Parameter | Required | Description | -| --------- | -------- | ---------------------------------------------------------------------------- | -| `partner` | Yes | Partner ID — must exist in `partner_store` | -| `uid` | Yes | Partner's user ID for this user | -| `return` | Yes | Redirect-back URL (must match partner's `allowed_return_domains`) | -| `consent` | No | Fallback TCF/GPP string if `ec_context.consent.is_empty()` after pre-routing | +### 7A.8 Network cluster disambiguation (`KvNetwork`) -### 8.3 Flow +Tracks how many distinct EC entries share the same hash prefix. A high count +indicates a shared network (corporate VPN, campus); a low count indicates an +individual or household. +```rust +pub struct KvNetwork { + pub cluster_size: Option, +} ``` -1. Parse query params. Missing required params → 400. - -2. Require a valid cookie-held EC. - If `cookie_was_present == false` OR `ec_context.ec_hash().is_none()` - (cookie missing or malformed) → redirect to - {return}?ts_synced=0&ts_reason=no_ec - -3. Look up partner record in partner_store. - Not found → 400. - -4. Validate return URL host against partner.allowed_return_domains. - - Exact hostname match only — no suffix or wildcard. - - Mismatch → 400. - -5. Evaluate consent. Use `ec_context.consent` (built pre-routing via - `build_consent_context()`). The optional `consent` query param is a **fallback - only** — used solely when `ec_context.consent.is_empty()` returns `true`. - This is the actual contract from the consent module. It is broader than - “no cookies or headers on the wire”: if consent KV fallback, decoded objects, - GPP section IDs, AC string, raw US privacy, or GPC already populated the - context, `is_empty()` is `false` and the query param is ignored entirely. - - When the fallback applies: decode the query param into a **locally-built** - `ConsentContext` (same TCF/GPP/USP decoders, same jurisdiction inputs), then - assign that value into `ec_context.consent` for the remainder of this request. - This makes the sync write decision and `ec_finalize_response()` use the same - effective consent view, avoiding a same-request “write partner ID, then - withdraw EC” conflict. Do NOT re-call `build_consent_context()` — that would - trigger `try_kv_write()` and persist the query-param consent to the consent KV - store, which is not intended. The decoded fallback applies only to this `/sync` - request; it is not written to the consent KV store and does not change any - future request unless the client sends real consent cookies/headers again. - - `!allows_ec_creation(...)` → redirect to {return}?ts_synced=0&ts_reason=no_consent - -6. Check anti-stuffing rate limit (sync_rate_limit per EC hash per partner per hour). - Exceeded → `429 Too Many Requests` (no redirect — the `return` URL is never called). - -7. kv.upsert_partner_id(ec_hash, partner_id, uid, now()) - If the root KV entry is missing (e.g. initial `create_or_revive()` failed on - the organic page load), `upsert_partner_id()` creates a minimal live entry and - then writes `ids[partner_id]`. This is the recovery path for best-effort EC - creation misses. - KV write failure → redirect to {return}?ts_synced=0&ts_reason=write_failed - -8. Success → redirect to {return}?ts_synced=1 + +**Written:** only by the `/_ts/api/v1/identify` endpoint, never on the organic proxy path. +The prefix-match list API call required to compute `cluster_size` is too +expensive for the hot path. + +**Evaluation:** `evaluate_cluster()` on `KvIdentityGraph`: + +- Returns the stored `cluster_size` without a prefix-list call when present +- If `cluster_size` is missing, calls `count_hash_prefix_keys()` with `limit(100)` — a single list-page call +- Writes the computed result to `entry.network` via best-effort CAS +- `cluster_recheck_secs` is retained only as a legacy compatibility setting because no cluster-check timestamp is stored in the EC identity entry + +**Threshold guidance:** + +| Cluster size | Likely scenario | +| ------------ | ----------------------------------------- | +| 1–3 | Individual / household | +| 4–10 | Small shared space (family, small office) | +| 11–50 | Medium office, hotel, coworking | +| 50+ | Corporate VPN, university, campus | + +**Default trust threshold:** entries with `cluster_size <= 10` are treated as +individual users for identity resolution purposes. Configurable per publisher +via `trusted-server.toml`: + +```toml +[ec] +cluster_trust_threshold = 10 # default +# cluster_recheck_secs is legacy compatibility; cluster_size is computed once per entry ``` -`ts_synced` values: +### 7A.9 Geo extensions (`KvGeo`) + +`KvGeo` is extended with two non-PII network signals available from Fastly's +`geo_lookup()` on the client IP: -| Value | Meaning | -| ------------------------------------ | ----------------------------- | -| `ts_synced=1` | KV write succeeded | -| `ts_synced=0&ts_reason=no_ec` | No valid EC cookie present | -| `ts_synced=0&ts_reason=no_consent` | Consent absent or denied | -| `ts_synced=0&ts_reason=write_failed` | KV write failed after retries | +- **`asn: Option`** — Autonomous System Number (e.g. `7922` = Comcast). + Primary signal for distinguishing home ISP vs. corporate VPN. Populated from + `GeoInfo::asn` which reads `fastly::geo::Geo::as_number()`. A value of `0` + from the Fastly API is mapped to `None`. +- **`dma: Option`** — DMA/metro code (e.g. `807` = San Francisco). + Market-level targeting signal; not personal data. Populated from + `GeoInfo::metro_code` when non-zero. -Rate limit exceeded returns `429 Too Many Requests` directly — the partner's `return` URL is not called in this case. +Both fields are written on initial `KvEntry::new()` from `GeoInfo`. Never +updated after creation — geo is a first-seen signal, not a real-time one. -### 8.4 Return URL construction +### 7A.10 IP address storage policy -Append `ts_synced` (and optional `ts_reason`) to the `return` URL: +Raw IP addresses are personal data under GDPR (CJEU _Breyer v. Germany_, 2016) +and must not be stored in KV entries. The EC hash already derives from the IP +without persisting it. -- If the URL already has a query string, append `&ts_synced=...` -- If not, append `?ts_synced=...` +Permitted IP-derived signals (written at creation time): -Do not modify any other query parameters on the `return` URL. +- `geo.country` — ISO 3166-1 alpha-2 +- `geo.region` — ISO 3166-2 subdivision +- `geo.asn` — ASN number (network identifier, not personal data) +- `geo.dma` — DMA/metro code (market identifier, not personal data) -### 8.5 Security +### 7A.11 Privacy rationale -- `return` URL validated by exact hostname match against `partner.allowed_return_domains`. No subdomain wildcard matching. -- No HMAC signature required on inbound sync request. -- Rate limit: `partner.sync_rate_limit` writes per EC hash per partner per hour. Default: 100. Configurable per partner in `partner_store`. +`ja4_class` (Section 1 only) and `platform_class` are category signals, not +unique device identifiers. They are equivalent in precision to `geo.country` +— they identify a class of client, not an individual. The full JA4 fingerprint +(Sections 2 and 3) is never stored, as it approaches unique device +identification and would require explicit consent basis under GDPR Art. 4(1). --- -## 9. S2S Batch Sync API (`POST /api/v1/sync`) +## 8. Prebid EID Cookie Ingestion -### 9.1 Module: `ec/sync_batch.rs` +> **Note:** The pixel sync endpoint (`GET /_ts/api/v1/sync`) has been removed. Partner ID sync from the browser is now handled via the Prebid EID cookie, which is written client-side by the TSJS Prebid integration and ingested server-side in `ec_finalize_response()`. + +### 8.1 Module: `ec/prebid_eids.rs` ```rust -pub async fn handle_batch_sync( - settings: &Settings, +/// Parses a `ts-eids` cookie value and writes matched partner UIDs to KV. +/// +/// Best-effort: all errors are logged and swallowed so the main request +/// path is never affected. +pub fn ingest_prebid_eids( + cookie_value: &str, + ec_id: &str, kv: &KvIdentityGraph, - partner_store: &PartnerStore, + registry: &PartnerRegistry, +); +``` + +### 8.2 Cookie format + +| Attribute | Value | +| ---------- | -------------------------------------------------------------------------------------------- | +| Name | `ts-eids` | +| Format | Base64-encoded (standard RFC 4648) JSON array of OpenRTB-style EIDs (`{source, uids:[...]}`) | +| Max size | JS writer targets 3 KB; backend parser accepts up to 8 KiB raw cookie length | +| Written by | TSJS Prebid integration (client-side JS) | +| Read by | `ec_finalize_response()` (server-side, via `ingest_prebid_eids()`) | + +**Example decoded value:** + +```json +[ + { + "source": "uidapi.com", + "uids": [{ "id": "A4A...", "atype": 3 }] + }, + { + "source": "liveramp.com", + "uids": [{ "id": "LR_xyz", "atype": 3 }] + } +] +``` + +### 8.3 JS side + +The TSJS Prebid integration calls `pbjs.getUserIdsAsEids()` in the `bidsBackHandler` callback after each auction. The returned OpenRTB-style EID array is base64-encoded and written to the `ts-eids` cookie. This runs entirely client-side — no server round-trip is needed for the write. Current writers preserve the full `{source, uids:[...]}` shape; the backend remains backward-compatible with the earlier flattened `{source, id, atype}` payload during rollout. + +### 8.4 Backend side + +`ingest_prebid_eids()` is called from `ec_finalize_response()` on both returning-user and new-EC paths when a `ts-eids` cookie is present and consent is granted. The flow: + +1. Base64-decode the cookie value. +2. JSON-parse into OpenRTB-style `Eid` entries; if that parse fails, fall back to the earlier flattened `{source, id, atype}` payload for backward compatibility. +3. For each EID entry: + a. Look up `registry.find_by_source_domain(&eid.source)`. Skip if no match. + b. Find the first non-empty UID in `eid.uids`. Skip the source if none is present. + c. Skip oversized UID values. + d. Call `kv.upsert_partner_id(ec_id, &partner.id, &uid.id)`. The upsert skips the KV write when the stored UID already matches. +4. All errors are logged and swallowed — EID ingestion never blocks the response. + +### 8.5 Source domain matching + +Source domains are matched via `PartnerRegistry.find_by_source_domain()`, which performs a case-insensitive lookup against the `source_domain` field configured on each partner in `[[ec.partners]]`. The registry builds a `by_source_domain` HashMap at startup for O(1) lookups. + +### 8.6 Write suppression + +EC identity entries no longer store per-partner sync timestamps. Instead of a +time-based debounce, `upsert_partner_id()` skips the KV write when the stored UID +already matches the incoming UID. Different UIDs replace the stored value. + +--- + +## 9. S2S Batch Sync API (`POST /_ts/api/v1/batch-sync`) + +### 9.1 Module: `ec/batch_sync.rs` + +```rust +pub fn handle_batch_sync( + kv: &KvIdentityGraph, + registry: &PartnerRegistry, + rate_limiter: &dyn RateLimiter, req: Request, ) -> Result>; ``` ### 9.2 Authentication -`Authorization: Bearer ` header required. Auth flow: +`Authorization: Bearer ` header required. Auth flow: -1. Compute `sha256_hex(api_key)`. -2. Look up `partner_store.find_by_api_key_hash(hash)` — uses the `apikey:{hash}` secondary index (§13.1) for O(1) lookup instead of scanning all partners. -3. If the index returns a partner, verify the partner's stored `api_key_hash` matches the computed hash (constant-time comparison). This guards against stale index entries from key rotation. -4. If no match or verification fails → `401 Unauthorized` with no body processing. -5. If KV lookup fails (store unavailable) → `503 Service Unavailable`. +1. Compute `sha256_hex(api_token)`. +2. Look up `registry.find_by_api_key_hash(hash)` — the `PartnerRegistry` maintains a `by_api_key_hash` HashMap built at startup from `[[ec.partners]]` config for O(1) lookup. +3. If no match → `401 Unauthorized` with no body processing. -Key rotation does not require binary redeployment — partners update via `/admin/partners/register`, which handles old API-key index cleanup (§13.1). +Key rotation requires updating the `api_token` in `[[ec.partners]]` TOML and redeploying. ### 9.2.1 API-key rate limiting -After successful auth, check the API-key level rate limit: `partner.batch_rate_limit` requests per partner per minute (default 60). Uses the same Fastly rate-limiting API as pixel sync (§14.3), with key `batch:{partner_id}`. +After successful auth, check the API-key level rate limit: `partner.batch_rate_limit` requests per partner per minute (default 60). Uses Fastly's Edge Rate Limiting API (§14.3), with key `batch:{partner_id}`. Exceeded → `429 Too Many Requests` with body `{ "error": "rate_limit_exceeded" }`. No mappings are processed. ### 9.3 Request format ``` -POST /api/v1/sync +POST /_ts/api/v1/batch-sync Content-Type: application/json Authorization: Bearer { "mappings": [ { - "ec_hash": "<64-character hex hash>", + "ec_id": "", "partner_uid": "abc123", "timestamp": 1741824000 } @@ -846,13 +1316,13 @@ Maximum batch size: 1000 mappings. Requests exceeding this receive `400 Bad Requ ### 9.4 Processing -The authenticated partner's ID (from the `PartnerRecord` resolved via API key in §9.2) determines the `ids[partner_id]` namespace for all writes in this batch. A partner can only write to their own namespace. +The authenticated partner's ID (from the `PartnerConfig` resolved via API key hash in §9.2) determines the `ids[partner_id]` namespace for all writes in this batch. A partner can only write to their own namespace. For each mapping: -1. Validate `ec_hash` format (must be exactly 64 lowercase hex characters). Invalid format → reject with `reason: "invalid_ec_hash"`. -2. Read KV metadata for `ec_hash`. If not found → reject with `reason: "ec_hash_not_found"`. If `consent.ok = false` → reject with `reason: "consent_withdrawn"`. -3. `kv.upsert_partner_id(ec_hash, partner_id, partner_uid, timestamp)`. The upsert internally skips the write if the existing `ids[partner_id].synced ≥ timestamp` (idempotent — counted as accepted, no error). On KV failure → reject all remaining mappings with `reason: "kv_unavailable"`, return `207`. +1. Validate `ec_id` format (must match `{64-hex}.{6-alnum}` pattern). Invalid format → reject with `reason: "invalid_ec_id"`. +2. Read KV metadata for `ec_id`. If not found → reject with `reason: "ec_id_not_found"`. If `consent.ok = false` → reject with `reason: "consent_withdrawn"`. +3. `kv.upsert_partner_id_if_exists(ec_id, partner_id, partner_uid)`. Mapping `timestamp` is retained for API compatibility but is not used for ordering. The upsert skips the write if the existing UID already matches (counted as accepted). A different UID overwrites the stored value. On KV failure → reject all remaining mappings with `reason: "kv_unavailable"`, return `207`. ### 9.5 Response format @@ -861,7 +1331,7 @@ For each mapping: "accepted": 998, "rejected": 2, "errors": [ - { "index": 45, "reason": "ec_hash_not_found" }, + { "index": 45, "reason": "ec_id_not_found" }, { "index": 72, "reason": "consent_withdrawn" } ] } @@ -875,7 +1345,6 @@ HTTP status rules: | Some accepted, some rejected | `207 Multi-Status` | | All rejected (auth valid, batch valid) | `207 Multi-Status` with `accepted: 0` | | Auth invalid | `401 Unauthorized` | -| Auth KV lookup failed (store down) | `503 Service Unavailable` | | Malformed JSON or > 1000 mappings | `400 Bad Request` | | KV entirely unavailable | `207 Multi-Status`, all rejected with `kv_unavailable` | @@ -893,10 +1362,10 @@ pub struct BatchSyncError { #[derive(Debug, derive_more::Display)] pub enum BatchSyncRejection { - #[display("invalid_ec_hash")] - InvalidEcHash, - #[display("ec_hash_not_found")] - EcHashNotFound, + #[display("invalid_ec_id")] + InvalidEcId, + #[display("ec_id_not_found")] + EcIdNotFound, #[display("consent_withdrawn")] ConsentWithdrawn, #[display("kv_unavailable")] @@ -932,7 +1401,7 @@ impl PullSyncDispatcher { &self, ec_context: &EcContext, client_ip: IpAddr, - partners: &[PartnerRecord], + partners: &[&PartnerConfig], kv: &KvIdentityGraph, ); } @@ -940,9 +1409,9 @@ impl PullSyncDispatcher { /// Fires a single partner pull request via `send_async()`, waits for the /// response via `PendingRequest::wait()`, and writes the result to KV. fn pull_one_partner( - ec_hash: &str, + ec_id: &str, ip: IpAddr, - partner: &PartnerRecord, + partner: &PartnerConfig, kv: &KvIdentityGraph, ); ``` @@ -951,15 +1420,17 @@ fn pull_one_partner( A pull sync is dispatched for a partner when all of the following are true on a request: -1. The request was routed to an **organic handler** (`handle_publisher_request` or `integration_registry.handle_proxy`). Pull sync never fires on EC route handlers (`/sync`, `/identify`, `/api/v1/sync`, `/admin/*`) or `/auction`. This matches the PRD requirement that pull calls must not happen during the pixel sync flow. +1. The request was routed to an **organic handler** (`handle_publisher_request` or `integration_registry.handle_proxy`). Pull sync never fires on EC route handlers (`/_ts/api/v1/identify`, `/_ts/api/v1/batch-sync`) or `/auction`. 2. A valid EC is present (`ec_context.ec_hash().is_some()`). This includes an EC newly generated on the current organic request — pull sync may run immediately after first-page EC creation because the response cookie is flushed before the background dispatch starts. 3. `allows_ec_creation(&ec_context.consent) == true` 4. `partner.pull_sync_enabled == true` -5. Either: no entry exists for this partner in the KV graph, or the existing `synced` timestamp is older than `partner.pull_sync_ttl_sec` (default 86400 seconds) -6. Rate limit not exceeded: `partner.pull_sync_rate_limit` calls per EC hash per partner per hour (default 10) +5. The partner UID is missing from the KV graph. If `ids[partner_id]` is already present, pull sync is skipped. +6. Rate limit not exceeded: `partner.pull_sync_rate_limit` calls per EC ID per partner per hour (default 10) + +`partner.pull_sync_ttl_sec` is retained for configuration compatibility, but is not used by the current fill-missing-only behavior because EC entries no longer store per-partner sync timestamps. ### 10.3 Execution model @@ -967,18 +1438,18 @@ Pull calls are dispatched using Fastly's background task / `send_async` model af Maximum concurrent pull calls per request: `settings.ec.pull_sync_concurrency` (default 3). -**Architectural divergence from PRD:** The PRD describes excess partner calls being queued and dispatched on subsequent requests for the same user. A persistent queue is not implementable in the stateless Fastly WASM edge environment — there is no cross-request mutable state. This spec adapts the intent using a stateless rotating offset: sort qualifying partners by ID, then use `(unix_timestamp_secs / 3600) % partner_count` as the starting index (wrapping). This ensures different partners are prioritized across different requests without persisted state. Partners not called on a given request remain eligible on the next qualifying request per their `pull_sync_ttl_sec` condition. The practical outcome (all partners eventually called) matches the PRD intent; the mechanism differs due to the platform constraint. +**Architectural divergence from PRD:** The PRD describes excess partner calls being queued and dispatched on subsequent requests for the same user. A persistent queue is not implementable in the stateless Fastly WASM edge environment — there is no cross-request mutable state. This spec adapts the intent using a stateless rotating offset: sort qualifying partners by ID, then use `(unix_timestamp_secs / 3600) % partner_count` as the starting index (wrapping). This ensures different missing partners are prioritized across requests without persisted queue state. Once a partner UID is stored, that partner is no longer eligible for pull sync under the current fill-missing-only behavior. ### 10.4 Outbound request ``` -GET {partner.pull_sync_url}?ec_hash={64-char-hex}&ip={ip_address} +GET {partner.pull_sync_url}?ec_id={64-hex}.{6-alnum} Authorization: Bearer {partner.ts_pull_token} ``` -Before dispatching, `pull_sync.rs` validates that `pull_sync_url`'s hostname is present in `partner.pull_sync_allowed_domains`. If not, the call is skipped and an `error` is logged — this is a configuration error that should not occur at runtime if admin validation is working correctly (§13.2 step 3). +Before dispatching, `pull_sync.rs` validates that `pull_sync_url`'s hostname is present in `partner.pull_sync_allowed_domains`. If not, the call is skipped and an `error` is logged — this is a configuration error that should not occur at runtime if startup validation in `PartnerRegistry::from_config()` is working correctly. -Only the EC hash and IP are sent. No consent strings, geo data, or other partner IDs are included. +Only the full EC ID is sent. No client IP, consent strings, geo data, or other partner IDs are included. **Expected partner responses:** @@ -993,69 +1464,97 @@ Any other non-200 response is treated as a transient failure. No retry. The next ### 10.5 KV write on success -On a non-null `uid`: call `kv.upsert_partner_id(ec_hash, partner_id, uid, now())`. If the root entry is missing, the upsert creates a minimal live entry first (same recovery path as `/sync`). On KV failure: log `warn` and discard the result. Retry occurs on the next qualifying request. - -The write updates `ids[partner_id].synced` to the current timestamp, resetting the `pull_sync_ttl_sec` window. +On a non-null `uid`: call `kv.upsert_partner_id(ec_id, partner_id, uid)`. If the root entry is missing, the upsert creates a minimal live entry first. If the same UID is already stored, the upsert skips the KV write. On KV failure: log `warn` and discard the result. Retry occurs on the next qualifying request while the partner UID remains missing. --- -## 11. Identity Resolution Endpoint (`GET /identify`) +## 11. Identity Resolution Endpoint (`GET /_ts/api/v1/identify`) ### 11.1 Module: `ec/identify.rs` ```rust -pub async fn handle_identify( +pub fn handle_identify( settings: &Settings, kv: &KvIdentityGraph, - partner_store: &PartnerStore, + registry: &PartnerRegistry, req: &Request, ec_context: &EcContext, ) -> Result>; ``` -### 11.2 Call patterns +### 11.2 Authentication + +**Bearer token required.** The `Authorization: Bearer ` header identifies the requesting partner. Auth flow: + +1. Parse the Bearer token from the `Authorization` header. +2. Compute `sha256_hex(api_token)`. +3. Look up `registry.find_by_api_key_hash(hash)` — O(1) in-memory lookup. +4. If no match → `401 Unauthorized` with `{ "error": "invalid_token" }`. + +The authenticated partner determines which UID is returned — each partner sees only their own synced UID for the given EC, not all partners' UIDs. -**Browser-direct:** The browser sends the request to `ec.publisher.com/identify`. Cookies and consent cookies are sent automatically (same-site). No special header forwarding required. +### 11.2.1 Call patterns -**Server-side proxy (for use case 2):** The publisher's origin server must forward: +**Browser-direct:** The browser sends the request to `ec.publisher.com/_ts/api/v1/identify` with the partner's API token in the `Authorization` header. Cookies (including `ts-ec` and consent cookies) are sent automatically (same-site). + +**Server-side proxy:** The publisher's origin server must forward: | Header | Required | | --------------------------------------------------------- | -------------------------------------- | +| `Authorization: Bearer ` | Yes | | `Cookie: ts-ec=` or `X-ts-ec: ` | Yes | | `Cookie: euconsent-v2=` or `Cookie: __gpp=` | Yes for EU/UK/US users | | `X-consent-advertising: ` | Optional — takes precedence if present | ### 11.3 EC and consent handling -`/identify` follows `EcContext` retrieval priority (Section 4.2). It does **not** -generate a new EC, and the handler itself does not write cookies. However, -`ec_finalize_response()` still runs after the handler: on consent withdrawal it -deletes the EC cookie, and on header/cookie mismatch it may reconcile the cookie -to the authoritative header-derived EC. +`/_ts/api/v1/identify` follows `EcContext` retrieval priority (Section 4.2). It does **not** +generate a new EC, and the handler itself does not write cookies. After the +handler, `ec_finalize_response()` may still delete the EC cookie on consent +withdrawal. Ordinary returning-user responses set the `x-ts-ec` header only; +they do not refresh or repair the browser cookie. Consent is evaluated using the same logic as Section 6. ### 11.4 Response -**`200 OK` — EC present, consent granted:** +**`401 Unauthorized` — missing or invalid Bearer token:** + +```json +{ "error": "invalid_token" } +``` + +This is checked first, before consent or EC presence. + +**`200 OK` — EC present, consent granted, partner UID resolved:** ```json { "ec": "a1b2c3...AbC123", "consent": "ok", "degraded": false, - "uids": { - "uid2": "A4A...", - "liveramp": "LR_xyz" - }, - "eids": [ - { "source": "uidapi.com", "uids": [{ "id": "A4A...", "atype": 3 }] }, - { "source": "liveramp.com", "uids": [{ "id": "LR_xyz", "atype": 3 }] } - ] + "partner_id": "liveramp", + "uid": "LR_xyz", + "eid": { "source": "liveramp.com", "uids": [{ "id": "LR_xyz", "atype": 3 }] }, + "cluster_size": 2 +} +``` + +The response is scoped to the requesting partner only. `partner_id` identifies which partner was authenticated. `uid` is the partner's resolved UID for this EC. `eid` is the OpenRTB 2.6 EID object for this partner. `cluster_size` is included when the network cluster has been evaluated (see §7A.8); absent when not yet evaluated. + +**`200 OK` — EC present, consent granted, no UID for this partner:** + +```json +{ + "ec": "a1b2c3...AbC123", + "consent": "ok", + "degraded": false, + "partner_id": "liveramp", + "cluster_size": null } ``` -`uids` contains one key per partner with `bidstream_enabled: true` and a resolved UID in the KV graph. Partners with no resolved UID for this user are omitted. +`uid` and `eid` are omitted when the partner has no synced UID for this EC. **`200 OK` — KV unavailable (degraded):** @@ -1064,8 +1563,8 @@ Consent is evaluated using the same logic as Section 6. "ec": "a1b2c3...AbC123", "consent": "ok", "degraded": true, - "uids": {}, - "eids": [] + "partner_id": "liveramp", + "cluster_size": null } ``` @@ -1078,8 +1577,8 @@ This case occurs by design when `create_or_revive()` fails on EC generation (bes "ec": "a1b2c3...AbC123", "consent": "ok", "degraded": false, - "uids": {}, - "eids": [] + "partner_id": "liveramp", + "cluster_size": null } ``` @@ -1091,7 +1590,7 @@ Note: `degraded` is `false` because the KV read succeeded (it returned `None`, m { "consent": "denied" } ``` -Consent is evaluated **before** EC presence. If `!allows_ec_creation(&consent)`, return `403` immediately — do not fall through to the `204` branch. This ensures consent denial is always surfaced, even for users with no EC. +Consent is evaluated **after** auth but **before** EC presence. If `!allows_ec_creation(&consent)`, return `403` immediately — do not fall through to the `204` branch. This ensures consent denial is always surfaced, even for users with no EC. **`204 No Content` — no EC present, consent not denied.** No body. @@ -1099,18 +1598,15 @@ Consent is evaluated **before** EC presence. If `!allows_ec_creation(&consent)`, Set on `200` responses only: -| Header | Value | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `X-ts-ec` | `{ec_hash.suffix}` | -| `X-ts-eids` | Standard base64 (RFC 4648, with `=` padding) of the JSON array of OpenRTB 2.6 `user.eids` objects. Capped at **4 KB** after encoding. If the encoded value exceeds 4 KB, the array is truncated (fewest partners first — highest `synced` timestamp retained) until it fits, and a `x-ts-eids-truncated: true` header is added. | -| `X-ts-` | Resolved UID per partner (e.g., `X-ts-uid2`). One header per partner with a resolved UID. **Capped at 20 partners** — partners sorted by most-recently synced; excess partners are omitted silently. | -| `X-ts-ec-consent` | `ok` (always — denied consent returns `403`, not `200`) | +| Header | Value | +| --------- | --------------------------------- | +| `X-ts-ec` | `{64-hex}.{6-alnum}` — full EC ID | -These are supplementary — callers should read the JSON body as the primary contract. The 4 KB cap on `X-ts-eids` and the 20-partner cap on `X-ts-` headers reflect typical proxy and browser total-header-budget constraints. Both caps apply independently. +The JSON body is the primary contract. The `X-ts-ec` header is supplementary for proxy-layer consumers. ### 11.6 Performance target -`/identify` must respond within 30ms (excluding network latency) when EC is present and KV read succeeds. This requires the KV read to be on the fast path with no retries. +`/_ts/api/v1/identify` must respond within 30ms (excluding network latency) when EC is present and KV read succeeds. This requires the KV read to be on the fast path with no retries. CORS headers must be set to allow browser-direct calls from the publisher's page. The `Access-Control-Allow-Origin` header is dynamically reflected from the `Origin` request header if the origin is an exact match or a subdomain of `settings.publisher.domain`: @@ -1122,20 +1618,18 @@ CORS headers must be set to allow browser-direct calls from the publisher's page Access-Control-Allow-Origin: Access-Control-Allow-Credentials: true Access-Control-Allow-Methods: GET, OPTIONS -Access-Control-Allow-Headers: Cookie, X-ts-ec, X-consent-advertising -Access-Control-Expose-Headers: X-ts-ec, X-ts-eids, X-ts-ec-consent, X-ts-eids-truncated, +Access-Control-Allow-Headers: Authorization, X-ts-ec +Access-Control-Max-Age: 600 Vary: Origin ``` -**`Access-Control-Expose-Headers` note:** The dynamic `X-ts-` headers must be enumerated per-response, not as a static constant. The handler builds the expose list by iterating the partner IDs that have resolved UIDs in the response. `x-ts-eids-truncated` is always included in the expose list (browser JS should be able to detect truncation even when it occurs). - **Origin validation logic:** CORS headers are only relevant when the `Origin` request header is present (browser requests always send it; server-side proxy calls typically do not). -- **No `Origin` header present:** Process normally. No CORS headers added. No `403`. This is the server-side proxy path from §11.2 — origin-server calls forwarding `Cookie` and consent headers. +- **No `Origin` header present:** Process normally. No CORS headers added. No `403`. This is the server-side proxy path from §11.2.1 — origin-server calls forwarding `Cookie`, consent headers, and `Authorization`. - **`Origin` header present, hostname matches `publisher.domain` or ends with `.{publisher.domain}` and scheme is `https`:** Reflect origin in `Access-Control-Allow-Origin`. Add `Vary: Origin`. - **`Origin` header present but does not match:** Return `403`. No body. -Browser `fetch()` with `credentials: "include"` sends an `OPTIONS` preflight. The router handles `OPTIONS /identify` identically — returns `200 OK` with the CORS headers above and no body. +Browser `fetch()` with `credentials: "include"` sends an `OPTIONS` preflight. The router handles `OPTIONS /_ts/api/v1/identify` identically — returns `200 OK` with the CORS headers above and no body. --- @@ -1176,7 +1670,7 @@ let (user_id, eids) = match ec_context.ec_hash() { Some(hash) => { let kv_entry = kv.get(hash).ok().flatten(); let eids = match kv_entry { - Some((entry, _gen)) => build_eids_from_kv(&entry, partner_store), + Some((entry, _gen)) => build_eids_from_kv(&entry, ®istry), None => vec![], // KV read failed or no entry — degrade gracefully }; (ec_context.ec_value.clone(), eids) @@ -1224,7 +1718,7 @@ The current `/auction` path returns a JSON response inline to the JS caller (`en | Header | Value | | --------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `X-ts-ec` | `{ec_hash.suffix}` — when EC is present | +| `X-ts-ec` | `{64-hex}.{6-alnum}` — full EC ID, when EC is present | | `X-ts-eids` | Standard base64 (RFC 4648) of OpenRTB 2.6 `user.eids` JSON array. Capped at 4 KB — same truncation rules as §11.5. | | `X-ts-eids-truncated` | `true` — present only when `X-ts-eids` was truncated | | `X-ts-ec-consent` | `ok` — only present when consent granted; on withdrawal `ec_finalize_response()` strips all EC headers | @@ -1233,200 +1727,221 @@ The current `/auction` path returns a JSON response inline to the JS caller (`en --- -## 13. Partner Registry and Admin Endpoint +## 13. Partner Registry (Config-Based) + +### 13.1 Overview + +Partners are defined in `[[ec.partners]]` TOML configuration and loaded into an in-memory `PartnerRegistry` at startup. There is no KV-backed partner store and no admin registration endpoint. Partner changes require a config update and redeployment. + +### 13.2 Module: `ec/partner.rs` + +Contains only validation helpers and API key hashing. The full partner data model and registry live in `ec/registry.rs`. + +```rust +/// Validates a partner ID format and checks against reserved names. +/// +/// # Errors +/// +/// Returns a descriptive error string on validation failure. +pub fn validate_partner_id(id: &str) -> Result<(), String>; +// Must match `^[a-z0-9_-]{1,32}$`. Reserved names rejected: +// `ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, `version`, `env`. + +/// Computes the SHA-256 hex digest of an API key. +pub fn hash_api_key(api_key: &str) -> String; +``` -### 13.1 Module: `ec/partner.rs` +### 13.3 Module: `ec/registry.rs` ```rust -pub struct PartnerRecord { - /// Partner identifier. Must match `^[a-z0-9_-]{1,32}$` (lowercase, no spaces). - /// Used to build `X-ts-` response headers — header-safety is required. - /// Reserved names that would collide with existing managed headers are rejected - /// at registration: `ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, `version`, `env`. +/// Runtime-ready partner configuration with precomputed API key hash. +#[derive(Debug, Clone)] +pub struct PartnerConfig { pub id: String, pub name: String, - pub allowed_return_domains: Vec, - pub api_key_hash: String, // SHA-256 hex of the partner's API key + pub source_domain: String, + pub openrtb_atype: u8, pub bidstream_enabled: bool, - pub source_domain: String, // OpenRTB source (e.g., "liveramp.com") - pub openrtb_atype: u8, // typically 3 - pub sync_rate_limit: u32, // per EC hash per partner per hour - pub batch_rate_limit: u32, // API-key level: requests per partner per minute (default 60) + pub api_key_hash: String, // SHA-256 hex, precomputed at startup + pub batch_rate_limit: u32, // requests per partner per minute (default 60) pub pull_sync_enabled: bool, - pub pull_sync_url: Option, // required when pull_sync_enabled; validated at registration - pub pull_sync_allowed_domains: Vec, // allowlist of domains TS may call for this partner - pub pull_sync_ttl_sec: u64, // default 86400 - pub pull_sync_rate_limit: u32, // default 10 - pub ts_pull_token: Option, // required when pull_sync_enabled; outbound bearer token + pub pull_sync_url: Option, + pub pull_sync_allowed_domains: Vec, + pub pull_sync_ttl_sec: u64, // default 86400 + pub pull_sync_rate_limit: u32, // default 10 + pub ts_pull_token: Option, // outbound bearer token for pull sync } -pub struct PartnerStore { - store_name: String, -} - -impl PartnerStore { - pub fn new(store_name: impl Into) -> Self; - - /// Looks up a partner by ID. Returns `None` if not found. - pub fn get(&self, partner_id: &str) -> Result, Report>; - - /// Verifies an API key against the stored hash for a given partner. - /// Uses constant-time comparison. - pub fn verify_api_key(&self, partner_id: &str, api_key: &str) -> bool; - - /// Writes or updates a partner record. - /// Returns `true` if this was a new partner (create), `false` if an existing - /// partner was updated. The pre-read needed for index maintenance (old API key - /// deletion) also determines this. - pub fn upsert(&self, record: &PartnerRecord) -> Result>; - - /// Looks up the partner owning a given API key hash (for batch sync auth). - /// Uses the `apikey:{hash}` secondary index for O(1) lookup, then verifies the - /// stored `api_key_hash` matches (guards against stale index from key rotation). - pub fn find_by_api_key_hash(&self, hash: &str) -> Result, Report>; - - /// Returns all partner records with `pull_sync_enabled == true`. - /// Used by the pull sync dispatcher after each organic request. Implementations - /// must re-check `pull_sync_enabled` on the fetched record before returning it, - /// because the `_pull_enabled` secondary index is best-effort and may be stale. - pub fn pull_enabled_partners(&self) -> Result, Report>; +/// In-memory partner registry with O(1) lookups by ID, API key hash, +/// and source domain. +/// +/// Built once at startup from `[[ec.partners]]` in `trusted-server.toml`. +/// All validation happens during construction. +pub struct PartnerRegistry { + by_id: HashMap, + by_api_key_hash: HashMap, + by_source_domain: HashMap, } -``` -**Storage layout:** Partner records are stored as JSON values in `partner_store` KV, keyed by `partner_id`. Two operations require access patterns beyond single-key lookup: +impl PartnerRegistry { + /// Builds a registry from the config-defined partner list. + /// + /// # Errors + /// + /// Returns `TrustedServerError::Configuration` if any partner has an + /// invalid ID, duplicate ID, duplicate API token hash, duplicate source + /// domain, or invalid pull sync configuration. + pub fn from_config(partners: &[EcPartner]) -> Result>; -1. **`find_by_api_key_hash(hash)`** — batch sync auth needs to find the partner owning a given API key hash. Implementation: maintain a secondary index entry `apikey:{sha256_hex} → partner_id` in the same KV store. Written on `upsert()`, looked up on batch auth. **On key rotation:** `upsert()` must read the existing record first, and if the `api_key_hash` has changed, delete the old `apikey:{old_hash}` index entry before writing the new one. This prevents old API keys from remaining valid after rotation. + /// Returns an empty registry (no partners configured). + pub fn empty() -> Self; -2. **`pull_enabled_partners()`** — pull sync needs all partners with `pull_sync_enabled == true`. Implementation: maintain an index entry `_pull_enabled → [partner_id_1, partner_id_2, ...]` (JSON array of partner IDs) in the same KV store. Updated on `upsert()` when `pull_sync_enabled` changes. The dispatcher reads this list, then does individual `get()` calls for each partner record. This bounds the number of KV reads to `1 + pull_partner_count` per organic request. + /// Looks up a partner by ID. + pub fn get(&self, partner_id: &str) -> Option<&PartnerConfig>; -**Consistency model:** These index writes are **best-effort, not atomic** — Fastly KV does not support multi-key transactions. `upsert()` writes in order: (1) primary record, (2) old API-key index deletion (if key changed), (3) new API-key index, (4) `_pull_enabled` list. If the process fails mid-sequence, indexes may be stale. All readers handle this defensively: + /// Looks up a partner by the SHA-256 hex hash of their API token. + pub fn find_by_api_key_hash(&self, hash: &str) -> Option<&PartnerConfig>; -- `find_by_api_key_hash()`: if the index points to a partner whose stored `api_key_hash` does not match the lookup hash, treat as auth failure (stale index from a rotation). -- `pull_enabled_partners()`: if a listed partner ID returns `None` from `get()`, skip it silently. If the fetched record has `pull_sync_enabled == false`, also skip it silently — that is a stale `_pull_enabled` index entry. -- The `_pull_enabled` list is vulnerable to lost updates under concurrent registrations. This is accepted — partner registration is a low-frequency admin operation (not a hot path). If lost updates become an issue, a CAS-based read-modify-write can be added later. + /// Looks up a partner by their `source_domain` (case-insensitive). + /// Used by Prebid EID ingestion to match EID sources to partners. + pub fn find_by_source_domain(&self, domain: &str) -> Option<&PartnerConfig>; -### 13.2 Admin endpoint (`POST /admin/partners/register`) + /// Returns all partners with `pull_sync_enabled = true`. + pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig>; -**Module:** `ec/admin.rs` + /// Returns an iterator over all configured partners. + pub fn all(&self) -> impl Iterator; -> **Codebase invariant — requires test update:** `Settings::ADMIN_ENDPOINTS` in `settings.rs` lists routes that must be covered by a `[[handlers]]` Basic Auth entry. The existing test at `settings.rs:1504-1530` scans `main.rs` for **every** `/admin/` route string and asserts it appears in `ADMIN_ENDPOINTS`. When `/admin/partners/register` is added to `main.rs`, this test will fail. -> -> **Required changes:** -> -> 1. Do **NOT** add `/admin/partners/register` to `ADMIN_ENDPOINTS` — it uses bearer-token-in-handler auth. -> 2. Update the admin-route-scan test (`settings.rs:1504-1530`) to maintain an exclusion list of bearer-token-authed admin routes (e.g., `const BEARER_AUTH_ADMIN_ROUTES: &[&str] = &["/admin/partners/register"]`) and skip those when asserting `ADMIN_ENDPOINTS` coverage. -> 3. Narrow the `[[handlers]]` pattern in `trusted-server.toml` from `"^/admin"` to `"^/admin/keys"` so that `/admin/partners/register` is not intercepted by `enforce_basic_auth()` before reaching its bearer-token handler. + /// Returns the number of configured partners. + pub fn len(&self) -> usize; -```rust -pub async fn handle_register_partner( - settings: &Settings, - partner_store: &PartnerStore, - req: Request, -) -> Result>; + /// Returns true if no partners are configured. + pub fn is_empty(&self) -> bool; +} ``` -Authentication: `Authorization: Bearer ` header, validated inside the handler against `settings.ec.admin_token_hash` (SHA-256 constant-time comparison). This is a publisher-level admin credential — separate from partner API keys, and enforced in-handler (not via `[[handlers]]` Basic Auth). Returns `401 Unauthorized` with no body if the token is missing or invalid. - -**Request:** +### 13.4 TOML configuration -``` -POST /admin/partners/register -Authorization: Bearer -Content-Type: application/json +Partners are defined in `trusted-server.toml` as `[[ec.partners]]` array entries: -{ - "id": "ssp_x", - "name": "SSP Example", - "allowed_return_domains": ["sync.example-ssp.com"], - "api_key": "raw_key_to_hash_and_store", - "bidstream_enabled": true, - "source_domain": "example-ssp.com", - "openrtb_atype": 3, - "sync_rate_limit": 100, - "batch_rate_limit": 60, - "pull_sync_enabled": false, - "pull_sync_url": null, - "pull_sync_allowed_domains": [], - "pull_sync_ttl_sec": 86400, - "pull_sync_rate_limit": 10, - "ts_pull_token": null -} +```toml +[[ec.partners]] +id = "liveramp" +name = "LiveRamp ATS" +source_domain = "liveramp.com" +openrtb_atype = 3 +bidstream_enabled = true +api_token = "partner-api-token-here" +batch_rate_limit = 60 +pull_sync_enabled = true +pull_sync_url = "https://api.liveramp.com/resolve" +pull_sync_allowed_domains = ["api.liveramp.com"] +pull_sync_ttl_sec = 86400 +pull_sync_rate_limit = 10 +ts_pull_token = "outbound-bearer-token" + +[[ec.partners]] +id = "uid2" +name = "UID 2.0" +source_domain = "uidapi.com" +openrtb_atype = 3 +bidstream_enabled = true +api_token = "uid2-api-token" +batch_rate_limit = 60 ``` -**Processing:** +### 13.5 Startup validation -1. Validate `Authorization: Bearer `: SHA-256 hash the token and compare against `settings.ec.admin_token_hash` using constant-time comparison. `401` if missing or invalid. -2. Validate required fields (`id`, `name`, `allowed_return_domains`, `api_key`, `source_domain`). `400` on failure. - Validate `id` format: must match `^[a-z0-9_-]{1,32}$`. Must not be a reserved name - (`ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, `version`, `env`). `400` with descriptive message on failure. -3. If `pull_sync_enabled == true`, validate that both `pull_sync_url` and `ts_pull_token` are present and non-empty. `400` with `"pull_sync_url and ts_pull_token are required when pull_sync_enabled is true"` if either is missing. - If `pull_sync_url` is set, validate that its hostname is present in `pull_sync_allowed_domains`. `400` on failure with `"pull_sync_url domain must be in pull_sync_allowed_domains"`. This prevents TS from being directed to call arbitrary URLs — the allowlist must be declared in the same registration payload. -4. Hash `api_key` with SHA-256 before writing — never store plaintext. -5. `let created = partner_store.upsert(record)?`. `503` on KV failure. - `upsert()` returns `true` for a new partner, `false` for an update. -6. Return `201 Created` if new partner (`created == true`), or `200 OK` if update - (`created == false`). Use an explicit response DTO — do NOT serialize the full - `PartnerRecord` (which contains `api_key_hash` and `ts_pull_token`). +`PartnerRegistry::from_config()` validates during construction: -**Response:** +1. Each partner ID matches `^[a-z0-9_-]{1,32}$` and is not reserved. +2. No duplicate partner IDs. +3. No duplicate API token hashes (collision detection). +4. No duplicate source domains. +5. Rate limits are within valid bounds. +6. If `pull_sync_enabled`, both `pull_sync_url` and `ts_pull_token` must be present. +7. If `pull_sync_url` is set, its hostname must be in `pull_sync_allowed_domains`. -```json -{ - "id": "ssp_x", - "name": "SSP Example", - "pull_sync_enabled": false, - "bidstream_enabled": true, - "created": true -} -``` - -The response confirms the registration succeeded and echoes key fields. `api_key_hash`, `ts_pull_token`, and `api_key` are never returned. `PartnerRecord` does not have a `registered_at` field — use the `created` boolean to signal first registration vs. upsert update. +Any validation failure causes a startup error (`TrustedServerError::Configuration`). --- ## 14. Configuration -### 14.1 New `EdgeCookie` settings struct +### 14.1 `Ec` settings struct Added to `crates/trusted-server-core/src/settings.rs`: ```rust #[derive(Debug, Clone, Deserialize, Serialize, Validate)] -pub struct EdgeCookie { +pub struct Ec { /// Publisher passphrase used as HMAC key for EC generation. /// Must be identical across all of the publisher's owned domains. /// Publishers sharing this value with partners form an identity-federated consortium. - #[validate(custom(function = EdgeCookie::validate_passphrase))] - pub passphrase: String, + #[validate(custom(function = Ec::validate_passphrase))] + pub passphrase: Redacted, /// Fastly KV store name for the EC identity graph. - #[validate(length(min = 1))] - pub ec_store: String, - - /// Fastly KV store name for the partner registry. - #[validate(length(min = 1))] - pub partner_store: String, - - /// SHA-256 hex of the publisher admin token for `POST /admin/partners/register`. - /// The plaintext token is provided in the `Authorization: Bearer` header; - /// it is never stored in plaintext. - #[validate(custom(function = EdgeCookie::validate_sha256_hex))] - pub admin_token_hash: String, + #[serde(default)] + pub ec_store: Option, /// Maximum concurrent pull sync calls dispatched per request. - #[validate(range(min = 1))] - #[serde(default = "EdgeCookie::default_pull_sync_concurrency")] + #[serde(default = "Ec::default_pull_sync_concurrency")] pub pull_sync_concurrency: usize, + + /// Network cluster trust threshold. Entries with `cluster_size <= threshold` + /// are treated as individual users for identity resolution purposes. + /// B2B publishers should raise this to 50+ for office-heavy audiences. + #[serde(default = "Ec::default_cluster_trust_threshold")] + pub cluster_trust_threshold: u32, + + /// Seconds between cluster size re-evaluations per entry. + /// Avoids repeated list-prefix API calls on every /identify request. + #[serde(default = "Ec::default_cluster_recheck_secs")] + pub cluster_recheck_secs: u64, + + /// Partners (SSPs, DSPs, identity vendors) for EC identity sync. + #[serde(default)] + pub partners: Vec, } -impl EdgeCookie { +impl Ec { fn validate_passphrase(passphrase: &str) -> Result<(), ValidationError>; - // Rejects "passphrase" or empty string as placeholder. - - fn validate_sha256_hex(value: &str) -> Result<(), ValidationError>; - // Requires exactly 64 lowercase hex characters. + // Rejects known placeholder values as non-production passphrases. fn default_pull_sync_concurrency() -> usize { 3 } + fn default_cluster_trust_threshold() -> u32 { 10 } + fn default_cluster_recheck_secs() -> u64 { 3600 } +} +``` + +The `EcPartner` struct (see §13.4 for TOML format): + +```rust +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct EcPartner { + pub id: String, + pub name: String, + pub source_domain: String, + #[serde(default = "EcPartner::default_openrtb_atype")] + pub openrtb_atype: u8, // default 3 + #[serde(default)] + pub bidstream_enabled: bool, + pub api_token: Redacted, // hashed at startup + #[serde(default = "EcPartner::default_batch_rate_limit")] + pub batch_rate_limit: u32, // default 60 + #[serde(default)] + pub pull_sync_enabled: bool, + #[serde(default)] + pub pull_sync_url: Option, + #[serde(default)] + pub pull_sync_allowed_domains: Vec, + #[serde(default = "EcPartner::default_pull_sync_ttl_sec")] + pub pull_sync_ttl_sec: u64, // default 86400 + #[serde(default = "EcPartner::default_pull_sync_rate_limit")] + pub pull_sync_rate_limit: u32, // default 10 + #[serde(default)] + pub ts_pull_token: Option>, } ``` @@ -1436,11 +1951,11 @@ Added to `Settings`: pub struct Settings { // ... existing fields ... #[validate(nested)] - pub ec: EdgeCookie, // Required — omitting [ec] is a startup error + pub ec: Ec, // Required — omitting [ec] is a startup error } ``` -`EdgeCookie` does not derive `Default` — omitting the `[ec]` section from TOML is a deserialization error at startup. This is intentional: `passphrase`, `ec_store`, `partner_store`, and `admin_token_hash` have no safe defaults. The `#[validate(nested)]` attribute ensures `EdgeCookie::validate_passphrase()` runs when `settings.validate()` is called at startup (`settings_data.rs:28`), matching the pattern used by `Publisher` and `Rewrite` in the existing `Settings` struct (`Synthetic` is removed in PR #479). +`Ec` does not derive `Default` — omitting the `[ec]` section from TOML is a deserialization error at startup. This is intentional: `passphrase` has no safe default. The `#[validate(nested)]` attribute ensures `Ec::validate_passphrase()` runs when `settings.validate()` is called at startup, matching the pattern used by `Publisher` and `Rewrite` in the existing `Settings` struct. ### 14.2 TOML configuration example @@ -1448,24 +1963,42 @@ pub struct Settings { [ec] passphrase = "publisher-chosen-secret" ec_store = "ec_identity_store" -partner_store = "ec_partner_store" -admin_token_hash = "sha256-hex-of-publisher-admin-token" pull_sync_concurrency = 3 +# cluster_trust_threshold = 10 # raise to 50+ for B2B publishers +# cluster_recheck_secs = 3600 # legacy compatibility; cluster_size is computed once per entry + +[[ec.partners]] +id = "liveramp" +name = "LiveRamp ATS" +source_domain = "liveramp.com" +api_token = "partner-api-token-here" +bidstream_enabled = true +batch_rate_limit = 60 +pull_sync_enabled = true +pull_sync_url = "https://api.liveramp.com/resolve" +pull_sync_allowed_domains = ["api.liveramp.com"] +ts_pull_token = "outbound-bearer-token" + +[[ec.partners]] +id = "uid2" +name = "UID 2.0" +source_domain = "uidapi.com" +api_token = "uid2-api-token" +bidstream_enabled = true ``` ### 14.3 Rate Limit Storage -Pixel sync and pull sync rate limits (per EC hash per partner per hour) cannot use in-memory state in a WASM/Fastly Compute environment — there is no shared memory across requests. +Batch sync and pull sync rate limits cannot use in-memory state in a WASM/Fastly Compute environment — there is no shared memory across requests. -**Implementation:** Use Fastly's Edge Rate Limiting API (`fastly::erl::RateCounter`), which provides distributed per-key counting without KV latency and is designed for high-frequency counting without per-key write limits. +**Implementation:** Use Fastly's Edge Rate Limiting API (`fastly::erl::RateCounter`), which provides distributed per-key counting without KV latency and is designed for high-frequency counting without per-key write limits. The `RateLimiter` trait abstracts this for testability. | Counter | Key format | Window | | ---------- | ----------------------------- | -------- | -| Pixel sync | `{partner_id}:{ec_hash}` | 1 hour | -| Pull sync | `pull:{partner_id}:{ec_hash}` | 1 hour | | Batch sync | `batch:{partner_id}` | 1 minute | +| Pull sync | `pull:{partner_id}:{ec_hash}` | 1 hour | -Engineering must confirm `fastly::erl::RateCounter` availability in the target before implementation of Steps 7, 9, and 10 is considered complete. Do NOT silently skip rate limiting in production if ERL is unavailable. Do NOT fall back to KV-based counters — they would hit the same 1 write/sec/key limit that necessitates `update_last_seen()` debouncing, and would thrash under real sync traffic. If ERL is unavailable, the rate-limited routes are blocked on an approved alternative counting mechanism. +Engineering must confirm `fastly::erl::RateCounter` availability in the target before implementation is considered complete. Do NOT silently skip rate limiting in production if ERL is unavailable. Do NOT fall back to KV-based counters — they would hit the same 1 write/sec/key limit that motivated removing recurring organic-request KV writes, and would thrash under real sync traffic. If ERL is unavailable, the rate-limited routes are blocked on an approved alternative counting mechanism. ### 14.4 Deprecation note @@ -1478,33 +2011,25 @@ Engineering must confirm `fastly::erl::RateCounter` availability in the target b New constants in `crates/trusted-server-core/src/constants.rs`: ```rust -// EC cookie name -pub const COOKIE_EC: &str = "ts-ec"; - -// EC response header -pub const HEADER_X_TS_EC: &str = "x-ts-ec"; - -// Supplementary identity headers -pub const HEADER_X_TS_EIDS: &str = "x-ts-eids"; -pub const HEADER_X_TS_EC_CONSENT: &str = "x-ts-ec-consent"; -pub const HEADER_X_TS_EIDS_TRUNCATED: &str = "x-ts-eids-truncated"; - -// Consent cookies (must match existing constants in constants.rs) -pub const COOKIE_TCF: &str = "euconsent-v2"; -pub const COOKIE_GPP: &str = "__gpp"; -pub const COOKIE_GPP_SID: &str = "__gpp_sid"; -pub const COOKIE_US_PRIVACY: &str = "us_privacy"; - -// No EC-specific geo/IP header constants — use req.get_client_ip_addr() and GeoInfo::from_request(req). +// EC cookie names +pub const COOKIE_TS_EC: &str = "ts-ec"; +pub const COOKIE_TS_EIDS: &str = "ts-eids"; + +// EC response headers +pub const HEADER_X_TS_EC: HeaderName = HeaderName::from_static("x-ts-ec"); +pub const HEADER_X_TS_EIDS: HeaderName = HeaderName::from_static("x-ts-eids"); +pub const HEADER_X_TS_EC_CONSENT: HeaderName = HeaderName::from_static("x-ts-ec-consent"); +pub const HEADER_X_TS_EIDS_TRUNCATED: HeaderName = HeaderName::from_static("x-ts-eids-truncated"); ``` -The following EC headers must be added to `INTERNAL_HEADERS` in `constants.rs` to ensure they are stripped before proxying to downstream backends: +The following EC headers are included in `INTERNAL_HEADERS` in `constants.rs` to ensure they are stripped before proxying to downstream backends: + +- `x-ts-ec` +- `x-ts-eids` +- `x-ts-ec-consent` +- `x-ts-eids-truncated` -- `HEADER_X_TS_EC` (`x-ts-ec`) -- `HEADER_X_TS_EIDS` (`x-ts-eids`) -- `HEADER_X_TS_EC_CONSENT` (`x-ts-ec-consent`) -- `HEADER_X_TS_EIDS_TRUNCATED` (`x-ts-eids-truncated`) -- Dynamic `X-ts-` headers — these cannot be registered statically because partners are added at runtime via `/admin/partners/register`. The `INTERNAL_HEADERS` filter **must use prefix stripping** (`x-ts-` prefix match) rather than enumerating partner IDs. A startup snapshot would miss partners registered after deployment. The current filter in `http_util.rs` uses explicit header names — extend it to also strip any header matching the `x-ts-` prefix pattern. +The `INTERNAL_HEADERS` filter uses `x-ts-` prefix stripping in `http_util.rs` to also strip any dynamic `X-ts-` headers without needing to enumerate partner IDs. --- @@ -1525,14 +2050,14 @@ pub enum TrustedServerError { // Maps to StatusCode::INTERNAL_SERVER_ERROR (500) // Used for: EC-specific handler errors only (not organic-path generation) - /// Partner not found in partner_store. + /// Partner not found in registry. #[display("Partner not found: {partner_id}")] PartnerNotFound { partner_id: String }, // Maps to StatusCode::BAD_REQUEST (400) /// Partner API key authentication failed. - #[display("Invalid API key for partner: {partner_id}")] - PartnerAuthFailed { partner_id: String }, + #[display("Invalid API key")] + PartnerAuthFailed, // Maps to StatusCode::UNAUTHORIZED (401) } ``` @@ -1544,23 +2069,17 @@ pub enum TrustedServerError { New routes added to `route_request()` in `crates/trusted-server-adapter-fastly/src/main.rs`: ```rust -// EC sync pixel — no auth required (partner validation is internal) -(GET, "/sync") → handle_sync(settings, &kv, &partner_store, &req, &mut ec_context) - -// EC identity resolution — no auth required (consent-gated) -(GET, "/identify") → handle_identify(settings, &kv, &partner_store, &req, &ec_context) +// EC identity resolution — Bearer token auth (internal to handler) +(GET, "/_ts/api/v1/identify") → handle_identify(settings, &kv, ®istry, &req, &ec_context) // CORS preflight for /identify — must be registered explicitly, current router dispatches by exact method/path -(OPTIONS, "/identify") → cors_preflight_identify(settings, &req) +(OPTIONS, "/_ts/api/v1/identify") → cors_preflight_identify(settings, &req) // S2S batch sync — partner API key auth (internal to handler) -(POST, "/api/v1/sync") → handle_batch_sync(settings, &kv, &partner_store, req) - -// Partner registration — publisher admin auth enforced in-handler (Bearer token) -(POST, "/admin/partners/register") → handle_register_partner(settings, &partner_store, req) +(POST, "/_ts/api/v1/batch-sync") → handle_batch_sync(&kv, ®istry, &limiter, req) ``` -Route ordering: EC routes are inserted before the fallback `handle_publisher_request()`. The `/admin/partners/register` route uses bearer-token auth in-handler (not `[[handlers]]` Basic Auth). The current `trusted-server.toml` has `path = "^/admin"` which catches **all** `/admin/*` paths via `enforce_basic_auth()` before routing — this would block bearer-token requests to `/admin/partners/register`. **Required change:** narrow the existing `[[handlers]]` pattern from `"^/admin"` to `"^/admin/keys"` so it covers only `/admin/keys/rotate` and `/admin/keys/deactivate` (the routes in `Settings::ADMIN_ENDPOINTS`). `/admin/partners/register` then passes through `enforce_basic_auth()` unchallenged and reaches the bearer-token handler. +Route ordering: EC routes are inserted before the fallback `handle_publisher_request()`. ### 17.1 EC integration in `main.rs` @@ -1574,9 +2093,17 @@ EC follows the same pre-routing pattern as `GeoInfo::from_request()` (line 70). This is a supported Fastly Compute pattern — `Response::send_to_client()` flushes the response to the client immediately and allows the WASM invocation to continue. This is not a small wiring change; it restructures how the application returns responses. ```rust -async fn route_request(...) -> Result<(), Error> { +fn route_request(...) -> Result<(), Error> { let geo_info = GeoInfo::from_request(&req); + // Phase 0 — bot gate (pure in-memory, no KV I/O). See §7A. + let device_signals = derive_device_signals(&req); + let is_real_browser = device_signals.looks_like_browser(); + if !is_real_browser { + log::debug!("Bot gate: blocking EC operations (ja4={:?}, platform={:?})", + device_signals.ja4_class, device_signals.platform_class); + } + // Pre-routing — read only, no generation (matches GeoInfo pattern). // EcContext stores client_ip internally (same req.get_client_ip_addr() // already called by GeoInfo::from_request() above). @@ -1584,20 +2111,35 @@ async fn route_request(...) -> Result<(), Error> { let mut ec_context = match ec_context_result { Ok(ctx) => ctx, Err(e) => { - // Pre-routing failure — no route matched yet, but we still need to - // send an HTTP error response. Construct one and flush immediately. log::error!("EcContext initialization failed: {e:?}"); let mut response = to_error_response(&e); response.send_to_client(); return Ok(()); } }; - let kv = KvIdentityGraph::new(&settings.ec.ec_store); - let partner_store = PartnerStore::new(&settings.ec.partner_store); - let pull_sync_dispatcher = PullSyncDispatcher::new(settings.ec.pull_sync_concurrency); + + // Pass device signals through for KvDevice on creation. + ec_context.set_device_signals(device_signals); + + // Build partner registry from config at startup. + let registry = PartnerRegistry::from_config(&settings.ec.partners)?; + + // Extract ts-eids cookie before routing consumes the request. + let eids_cookie = extract_cookie_value(&req, COOKIE_TS_EIDS); + + // Bot gate: suppress all KV operations for unrecognized clients. + let kv = if is_real_browser { + settings.ec.ec_store.as_deref().map(KvIdentityGraph::new) + } else { + None + }; + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); if let Some(mut response) = enforce_basic_auth(settings, &req) { - ec_finalize_response(settings, geo_info.as_ref(), &ec_context, &kv, &mut response); + // Bot gate: skip EC cookie writes for unrecognized clients. + if is_real_browser { + ec_finalize_response(settings, &ec_context, kv.as_ref(), ®istry, eids_cookie.as_deref(), &mut response); + } response.send_to_client(); return Ok(()); } @@ -1609,49 +2151,37 @@ async fn route_request(...) -> Result<(), Error> { // is_organic tracks whether pull sync should fire (organic routes only — §10.2). let mut is_organic = false; let result = match (method, path.as_str()) { - // EC-specific routes — all read-only except /sync which takes &mut. - // /sync may assign fallback consent into ec_context.consent when the - // query param is the only signal — see §8.3. - (GET, "/sync") => handle_sync(settings, &kv, &partner_store, &req, &mut ec_context).await, - (GET, "/identify") => handle_identify(settings, &kv, &partner_store, &req, &ec_context).await, - (OPTIONS, "/identify") => cors_preflight_identify(settings, &req), - (POST, "/api/v1/sync") => handle_batch_sync(settings, &kv, &partner_store, req).await, - (POST, "/admin/partners/register") => handle_register_partner(settings, &partner_store, req).await, - - // /auction — EC-read-only; never generates EC. - // NOTE: handle_auction signature changes from (settings, orchestrator, req) to - // (settings, orchestrator, &kv, req, &ec_context) — this is a call-graph change, - // not just wiring. See §12 for the full auction integration. - (POST, "/auction") => handle_auction(settings, orchestrator, &kv, req, &ec_context).await, - - // Organic routes — generate EC if needed (best-effort, never 500s), then dispatch + (GET, "/_ts/api/v1/identify") => handle_identify(settings, kv.as_ref(), ®istry, &req, &ec_context), + (OPTIONS, "/_ts/api/v1/identify") => cors_preflight_identify(settings, &req), + (POST, "/_ts/api/v1/batch-sync") => handle_batch_sync(kv.as_ref(), ®istry, &limiter, req), + (POST, "/auction") => handle_auction(settings, orchestrator, kv.as_ref(), req, &ec_context), + (m, path) if integration_registry.has_route(&m, path) => { is_organic = true; - ec_context.generate_if_needed(settings, &kv); - integration_registry.handle_proxy(&m, path, settings, req, &ec_context).await + ec_context.generate_if_needed(settings, kv.as_ref()); + integration_registry.handle_proxy(&m, path, settings, req, &ec_context) }, _ => { is_organic = true; - ec_context.generate_if_needed(settings, &kv); + ec_context.generate_if_needed(settings, kv.as_ref()); handle_publisher_request(settings, integration_registry, req, &ec_context) }, }; - // Unwrap result — errors become error responses (matches existing pattern) let mut response = result.unwrap_or_else(|e| to_error_response(&e)); - // finalize_response runs on every route — enforces cookie write/deletion/last_seen - ec_finalize_response(settings, geo_info.as_ref(), &ec_context, &kv, &mut response); + // Bot gate: skip EC cookie writes and finalize for unrecognized clients. + if is_real_browser { + ec_finalize_response(settings, &ec_context, kv.as_ref(), ®istry, eids_cookie.as_deref(), &mut response); + } - // Flush response to client; WASM continues for background pull sync. response.send_to_client(); - // Background pull sync — organic routes only (§10.2). Never fires on /sync, - // /identify, /auction, /api/v1/sync, or /admin/* routes. - // Fires outbound HTTP calls via send_async(), blocks on PendingRequest::wait(). - if is_organic { - if let (Some(ip), Ok(pull_partners)) = (ec_context.client_ip, partner_store.pull_enabled_partners()) { - pull_sync_dispatcher.dispatch_background(&ec_context, ip, &pull_partners, &kv); + // Background pull sync — organic routes only, real browsers only (§7A.4, §10.2). + if is_real_browser && is_organic { + if let Some(ip) = ec_context.client_ip { + let pull_partners = registry.pull_enabled_partners(); + pull_sync_dispatcher.dispatch_background(&ec_context, ip, &pull_partners, kv.as_ref()); } } @@ -1659,7 +2189,7 @@ async fn route_request(...) -> Result<(), Error> { } ``` -The existing `finalize_response()` in `main.rs` becomes `ec_finalize_response()` with the extended signature that accepts `ec_context` and `kv`. The `#[fastly::main]` entrypoint changes to call `route_request()` and return `Ok(())` (the response is already sent via `send_to_client()`). +The existing `finalize_response()` in `main.rs` becomes `ec_finalize_response()` with the extended signature that accepts `ec_context`, `kv`, `registry`, and `eids_cookie`. The `#[fastly::main]` entrypoint changes to call `route_request()` and return `Ok(())` (the response is already sent via `send_to_client()`). The `PartnerRegistry` is built once at startup via `PartnerRegistry::from_config(&settings.ec.partners)` and passed by reference throughout the request lifecycle. `PullSyncDispatcher::dispatch_background` uses `Request::send_async()` to fire outbound HTTP calls, then calls `PendingRequest::wait()` (blocking) on each handle under `settings.ec.pull_sync_concurrency` concurrency. No async runtime is needed — this is synchronous blocking code running after `send_to_client()` has flushed the response. The Fastly WASM invocation stays alive until `dispatch_background` returns. This does not add latency to the user-facing response. @@ -1673,53 +2203,55 @@ Follow the project's **Arrange-Act-Assert** pattern. Test both happy paths and e Each module in `ec/` has a `#[cfg(test)]` module covering: -| Module | Key test cases | -| --------------- | --------------------------------------------------------------------------------------------------------- | -| `identity.rs` | IPv4/IPv6 normalization, /64 truncation, HMAC determinism, output format | -| `finalize.rs` | `ec_finalize_response()`: cookie write on generation, deletion on withdrawal, `update_last_seen` debounce | -| `cookie.rs` | Cookie string format, Max-Age=0 for deletion, domain derivation | -| `kv.rs` | Serialization/deserialization roundtrip, CAS merge logic, metadata extraction | -| `partner.rs` | API key hash verification (constant-time), record serialization | -| `sync_pixel.rs` | All `ts_synced` redirect codes, 429 rate limit, return URL construction | -| `sync_batch.rs` | Status code selection (200/207/401/400/429), per-mapping rejection reasons, API-key rate limit | -| `pull_sync.rs` | Trigger conditions, null/404 no-op, dispatch limit | -| `identify.rs` | All response codes (200/403/204), degraded flag, `uids` filtering | +| Module | Key test cases | +| ---------------- | --------------------------------------------------------------------------------------------------------------------- | +| `generation.rs` | IPv4/IPv6 normalization, /64 truncation, HMAC determinism, output format | +| `finalize.rs` | `ec_finalize_response()`: cookie write on generation, deletion on withdrawal, returning-user EC header, EID ingestion | +| `cookies.rs` | Cookie string format, Max-Age=0 for deletion, domain derivation | +| `kv.rs` | Serialization/deserialization roundtrip, CAS merge logic, metadata extraction | +| `partner.rs` | Partner ID validation, API key hashing | +| `registry.rs` | `from_config()` validation, duplicate detection, O(1) lookups by ID/hash/domain | +| `prebid_eids.rs` | Base64 decode, JSON parse, source domain matching, debounce | +| `batch_sync.rs` | Status code selection (200/207/401/400/429), per-mapping rejection reasons, API-key rate limit | +| `pull_sync.rs` | Trigger conditions, null/404 no-op, dispatch limit | +| `identify.rs` | Bearer auth (200/401/403/204), scoped partner response, degraded flag, CORS | ### 18.2 Integration tests KV behavior is tested with Viceroy (local Fastly Compute simulator) using real KV store operations. Key scenarios: -- Consent withdrawal: cookie deletion + tombstone write (`write_withdrawal_tombstone()`) + all EC response headers stripped — in same request +- Explicit consent withdrawal: cookie deletion + tombstone write (`write_withdrawal_tombstone()`) + all EC response headers stripped — in same request - Concurrent writes: CAS retry logic under simulated generation conflicts - KV degraded: EC cookie still set when KV `create_or_revive()` fails (best-effort) -- Sync-then-identify flow: pixel sync writes partner ID, then `/identify` returns it +- Prebid EID ingestion: `ts-eids` cookie parsed, source domain matched, partner UID written to KV +- Batch sync then identify: batch sync writes partner UID, then `/_ts/api/v1/identify` returns it for that partner **Eventually-consistent caveat:** Fastly KV does not guarantee read-after-write consistency. The sync→identify scenario may not be immediately visible on production — Viceroy may behave differently. Tests for this flow should use retry with backoff (up to 1s) and be documented as Viceroy-only consistency. Do not write assertions that assume immediate visibility after a KV write. ### 18.3 JS tests (if applicable) -If any JS changes are made for EC (e.g., publisher-side `/identify` fetch helper in `crates/js/`), use Vitest with `vi.hoisted()` for mocks. +If any JS changes are made for EC (e.g., publisher-side `/_ts/api/v1/identify` fetch helper in `crates/js/`), use Vitest with `vi.hoisted()` for mocks. --- ## 19. Implementation Order -Suggested order to minimize risk and allow incremental testing. Each step should pass `cargo test --workspace` before the next begins. - -| Step | Scope | Deliverable | -| ---- | --------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| 1 | `ec/identity.rs` + constants + settings | `generate_ec()`, `normalize_ip()`, `EcContext` | -| 2 | `ec/finalize.rs` | `ec_finalize_response()` (cookie write, deletion, tombstone, last_seen) | -| 3 | `ec/cookie.rs` | Cookie creation, deletion, response header | -| 4 | `ec/kv.rs` | `KvIdentityGraph` CRUD with CAS | -| 5 | `ec/partner.rs` + `ec/admin.rs` | `PartnerStore`, `/admin/partners/register` | -| 6 | EC middleware in `main.rs`, `publisher.rs`, `registry.rs` | `EcContext::read_from_request()` pre-routing, `generate_if_needed()`, `ec_finalize_response()` | -| 7 | `ec/sync_pixel.rs` | `GET /sync` handler + route | -| 8 | `ec/identify.rs` | `GET /identify` handler + route | -| 9 | `ec/sync_batch.rs` | `POST /api/v1/sync` handler + route | -| 10 | `ec/pull_sync.rs` | Background pull sync dispatch (blocking, after `send_to_client()`) | -| 11 | Auction integration | EC injection into `user.id`, `user.eids`, `user.consent` | -| 12 | End-to-end integration tests | Viceroy-based flow tests | +Implementation was completed in the following order. Each step passed `cargo test --workspace` before the next began. + +| Step | Scope | Deliverable | +| ---- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| 1 | `ec/generation.rs` + constants + settings | `generate_ec()`, `normalize_ip()`, `EcContext` | +| 2 | `ec/cookies.rs` | Cookie creation, deletion, response header | +| 3 | `ec/kv.rs` + `ec/kv_types.rs` | `KvIdentityGraph` CRUD with CAS | +| 4 | `ec/finalize.rs` | `ec_finalize_response()` (cookie write on generation, deletion, tombstone, returning-user header) | +| 5 | `ec/partner.rs` + `ec/registry.rs` | `PartnerRegistry` (config-based), partner validation helpers | +| 6 | EC middleware in `main.rs`, `publisher.rs`, `registry.rs` | `EcContext::read_from_request()` pre-routing, `generate_if_needed()`, `ec_finalize_response()` | +| 7 | `ec/prebid_eids.rs` | Prebid EID cookie ingestion (replaces pixel sync) | +| 8 | `ec/identify.rs` | `GET /_ts/api/v1/identify` handler + route (Bearer auth, scoped response) | +| 9 | `ec/batch_sync.rs` + `ec/rate_limiter.rs` | `POST /_ts/api/v1/batch-sync` handler + route | +| 10 | `ec/pull_sync.rs` | Background pull sync dispatch (blocking, after `send_to_client()`) | +| 11 | Auction integration | EC injection into `user.id`, `user.eids`, `user.consent` | +| 12 | End-to-end integration tests | Viceroy-based flow tests | --- @@ -1733,7 +2265,7 @@ and auction decoration — without relying on third-party cookies. **Done when:** All 12 stories below are complete, `cargo test --workspace` and `cargo clippy` pass with no warnings, and the end-to-end Viceroy flow tests -cover the full sync → identify → auction path. +cover the full EID ingestion → identify → auction path. **Spec ref:** This document. PRD: `docs/internal/ssc-prd.md`. @@ -1744,7 +2276,7 @@ cover the full sync → identify → auction path. Implement the core EC data types, generation logic, and per-request context struct that all subsequent stories depend on. -**Scope:** `ec/identity.rs`, `ec/mod.rs`, `trusted-server.toml` `[ec]` section, +**Scope:** `ec/generation.rs`, `ec/mod.rs`, `trusted-server.toml` `[ec]` section, `Settings` struct update. **Acceptance criteria:** @@ -1764,7 +2296,7 @@ struct that all subsequent stories depend on. Calls `build_consent_context()` with the EC hash as identity key and stores the result as `consent: ConsentContext` (see §6.1.1). Does not generate. Does not write to EC identity KV. (Note: `build_consent_context()` may write - to the consent KV store when an EC hash is available.) + using the request-local consent context.) - `EcContext::generate_if_needed(settings, kv)` generates a new EC when `ec_value == None && allows_ec_creation(&consent)`, sets `ec_generated = true`, and writes the initial KV entry via `kv.create_or_revive()` (best-effort). @@ -1774,8 +2306,8 @@ struct that all subsequent stories depend on. without setting `ec_generated`. It never returns an error — organic traffic must not 500 on EC failure. - `[ec]` settings block parses from TOML: `passphrase`, `ec_store`, - `partner_store`, `admin_token_hash`, `pull_sync_concurrency`. -- All unit tests in `identity.rs` pass (HMAC determinism, format, IP normalization). + `pull_sync_concurrency`, `partners`. +- All unit tests in `generation.rs` pass (HMAC determinism, format, IP normalization). **Spec ref:** §2, §3, §4, §5.4, §14.1 @@ -1784,26 +2316,19 @@ struct that all subsequent stories depend on. ### Story 2 — EC finalize response Implement `ec_finalize_response()` — the post-routing function that enforces -cookie writes, deletions, tombstones, and last-seen updates on every response. +cookie writes on generation, cookie deletion on withdrawal, tombstones, returning-user `x-ts-ec` headers, and EID ingestion on responses. **Scope:** `ec/finalize.rs` (new file) **Acceptance criteria:** - `ec_finalize_response(settings, geo, ec_context, kv, response)` runs on every route. -- Consent gating uses the existing `allows_ec_creation()` — no new gating function. -- When `!allows_ec_creation(&consent) && cookie_was_present`: calls - `clear_ec_on_response()` (deletes cookie and strips all EC response headers) - and writes tombstone for each valid EC hash available. When the cookie is - malformed and no valid header exists, no tombstone is written — cookie - deletion alone enforces withdrawal (see §6.2). -- When `ec_was_present && !ec_generated && allows_ec_creation(&consent)`: calls - `kv.update_last_seen(ec_hash, now())` (debounced at 300s). If `cookie_ec_value` - is set (header/cookie mismatch), also calls `set_ec_on_response()` to reconcile - the browser cookie to the header-derived identity. -- When `ec_generated == true`: calls `set_ec_on_response()`. -- Unit tests cover all four branches: withdrawal (with and without valid hash), - returning-user last_seen + mismatch reconciliation, and new-EC generation. +- Consent gating uses `allows_ec_creation()` for current-request EC usage and `has_explicit_ec_withdrawal()` for cookie-expiry/tombstone decisions. +- When `!allows_ec_creation(&consent)`: strips all EC response headers. +- When `has_explicit_ec_withdrawal(&consent) && cookie_was_present`: additionally expires the cookie and writes tombstones for each valid EC ID available. When the cookie is malformed and no valid header exists, no tombstone is written — cookie deletion alone enforces withdrawal (see §6.2). +- When `ec_was_present && !ec_generated && allows_ec_creation(&consent)`: sets the `x-ts-ec` response header only. It does not refresh the EC cookie, repair header/cookie mismatches, or write KV solely to extend TTL. +- When `ec_generated == true`: calls `set_ec_cookie_and_header_on_response()`. +- Unit tests cover explicit-withdrawal, fail-closed header stripping, returning-user header behavior, and new-EC generation. **Spec ref:** §5.4, §6.2 @@ -1814,7 +2339,7 @@ cookie writes, deletions, tombstones, and last-seen updates on every response. Implement the low-level functions that create and delete the `ts-ec` cookie and set EC response headers. These are called by `ec_finalize_response()` (Story 2). -**Scope:** `ec/cookie.rs` +**Scope:** `ec/cookies.rs` **Acceptance criteria:** @@ -1822,7 +2347,7 @@ and set EC response headers. These are called by `ec_finalize_response()` (Story `Max-Age=31536000`, `SameSite=Lax; Secure`. `HttpOnly` is NOT set (JS on the publisher page must be able to read the cookie). - `delete_ec_cookie()` produces a cookie with `Max-Age=0`, same attributes. -- `set_ec_on_response()` sets `Set-Cookie` and `X-ts-ec` response headers. +- `set_ec_header_on_response()` sets only `X-ts-ec`; `set_ec_cookie_and_header_on_response()` sets both `Set-Cookie` and `X-ts-ec`. - `clear_ec_on_response()` sets `Set-Cookie` with `Max-Age=0` **and** strips all EC-related response headers: `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, and any `X-ts-` headers. This prevents @@ -1854,19 +2379,14 @@ CAS-based concurrent write protection and consent withdrawal delete. - `KvIdentityGraph::create_or_revive(ec_hash, &entry)` creates a new entry OR overwrites an existing tombstone (`consent.ok = false`) with a fresh entry; no-ops if a live entry already exists. Called by `generate_if_needed()`. -- `KvIdentityGraph::update_last_seen(ec_hash, timestamp)` updates `last_seen` - without overwriting partner IDs (CAS merge), and only writes if the stored - value is more than 300s older than `timestamp` (debounce to avoid 1 write/sec - KV limit). Callers pass `now()` as `timestamp`. +- Returning-user page views do not update a last-seen field; EC entries no longer store `last_seen` or mutable publisher-domain visit timestamps. - `KvIdentityGraph::write_withdrawal_tombstone(ec_hash)` sets `consent.ok = false`, clears partner IDs, and applies a 24-hour TTL (see §6.2). Returns `Result` — callers log `error` on failure and continue (cookie deletion is the primary enforcement mechanism). - `KvIdentityGraph::delete(ec_hash)` hard-deletes the entry — used only for IAB data deletion requests, not for consent withdrawal (which uses tombstones). -- `kv.upsert_partner_id(ec_hash, partner_id, uid, timestamp)` writes to - `ids[partner_id]`, creating a minimal live root entry first if the key is - absent, and skips if existing `synced >= timestamp` (idempotent). +- `kv.upsert_partner_id(ec_hash, partner_id, uid)` writes to `ids[partner_id]`, creating a minimal live root entry first if the key is absent, and skips writes when the existing UID already matches (idempotent). - KV schema matches §7 exactly (JSON roundtrip test). - Unit tests cover CAS merge logic, tombstone write, tombstone error handling, serialization/deserialization roundtrip, metadata extraction. @@ -1875,39 +2395,31 @@ CAS-based concurrent write protection and consent withdrawal delete. --- -### Story 5 — Partner registry and admin endpoint +### Story 5 — Partner registry (config-based) -Implement `PartnerRecord`, `PartnerStore`, and the admin registration endpoint -that operators use to onboard ID sync partners. +Implement partner ID validation, API key hashing, and the in-memory +`PartnerRegistry` that replaces the KV-backed `PartnerStore`. -**Scope:** `ec/partner.rs`, `ec/admin.rs`, router update +**Scope:** `ec/partner.rs`, `ec/registry.rs` **Acceptance criteria:** -- `PartnerRecord` contains all fields from §13.1 including +- `validate_partner_id()` enforces `^[a-z0-9_-]{1,32}$` and rejects reserved + names (`ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, + `version`, `env`). +- `hash_api_key()` computes SHA-256 hex of the plaintext API token. +- `PartnerConfig` contains all fields from §13.3 including `pull_sync_allowed_domains` and `batch_rate_limit`. -- `PartnerStore::get()`, `upsert()`, `find_by_api_key_hash()` operate on - `partner_store` KV. -- `pull_enabled_partners()` re-checks `pull_sync_enabled == true` on fetched - records so stale `_pull_enabled` index entries do not dispatch disabled partners. -- API key stored as SHA-256 hex; plaintext never written to KV. -- `verify_api_key()` uses constant-time comparison. -- `POST /admin/partners/register` validates `Authorization: Bearer ` inside - the handler against `settings.ec.admin_token_hash` (constant-time SHA-256 comparison). - Returns `401` if missing or invalid — before any request body is read. -- Admin endpoint validates: `pull_sync_url` hostname must be in - `pull_sync_allowed_domains` when set — returns `400` otherwise. -- Returns `201 Created` on new partner or `200 OK` on update, with an explicit - response DTO (see §13.2 step 6 — do NOT serialize full `PartnerRecord`). - Returns `400` on validation failure; `503` on KV failure. -- `/admin/partners/register` is **NOT** added to `Settings::ADMIN_ENDPOINTS` — - it uses bearer-token-in-handler auth, not `[[handlers]]` Basic Auth. -- The admin-route-scan test (`settings.rs:1504-1530`) must be updated to exclude - bearer-token-authed routes from its `ADMIN_ENDPOINTS` assertion. Add an exclusion - list (see §13.2 codebase invariant note). -- The `[[handlers]]` pattern in `trusted-server.toml` must be narrowed from - `"^/admin"` to `"^/admin/keys"` (see §13.2). -- Unit tests cover API key hash verification and record serialization. +- `PartnerRegistry::from_config()` builds the registry from `Vec` + with O(1) `by_id`, `by_api_key_hash`, and `by_source_domain` indexes. +- Startup validation catches: invalid IDs, duplicate IDs, duplicate API token + hashes, duplicate source domains, invalid pull sync configuration. +- `get()`, `find_by_api_key_hash()`, `find_by_source_domain()` return + `Option<&PartnerConfig>`. +- `pull_enabled_partners()` returns only partners with `pull_sync_enabled = true`. +- No admin endpoint — partner changes require config update and redeployment. +- Unit tests cover partner ID validation, hash computation, registry + construction, and duplicate detection. **Spec ref:** §13 @@ -1925,22 +2437,17 @@ Wire `EcContext` into the request pipeline following the two-phase model - `EcContext::read_from_request()` is called before the route match on every request, passed the existing `geo_info` (no duplicate geo header parsing). -- EC route handlers receive `ec_context` without EC generation. `/identify`, - `/auction`, `/api/v1/sync`, and `/admin/*` use read-only `&EcContext` and - never mutate it. **Exception:** `/sync` receives `&mut EcContext`; when the - consent query-param fallback applies (`ec_context.consent.is_empty()`), it - assigns the locally-decoded consent into `ec_context.consent` so that both - the sync write decision and `ec_finalize_response()` share the same effective - consent view. This prevents a same-request "write partner ID, then withdraw - EC" conflict. See §8.3 for full details. +- EC route handlers receive `ec_context` without EC generation. `/_ts/api/v1/identify`, + `/auction`, and `/_ts/api/v1/batch-sync` use read-only `&EcContext` and + never mutate it. - `/auction` consumes EC identity but never bootstraps it. - `handle_publisher_request()` and `integration_registry.handle_proxy()` call `ec_context.generate_if_needed(settings, &kv)` before their handler logic (best-effort, never 500s). - `ec_finalize_response()` receives `ec_context` and `kv` and: - - Deletes the EC cookie and writes a withdrawal tombstone when `!allows_ec_creation(&consent) && cookie_was_present` (runs on all routes). - - Calls `kv.update_last_seen(ec_hash, now())` when `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)` (returning user with valid consent). - - Calls `set_ec_on_response()` when `ec_context.ec_generated == true`, and also - on returning-user mismatch reconciliation when `cookie_ec_value.is_some()`. + - Strips EC response headers whenever `!allows_ec_creation(&consent)`. + - Additionally deletes the EC cookie and writes a withdrawal tombstone when `has_explicit_ec_withdrawal(&consent) && cookie_was_present` (runs on all routes). + - Sets `x-ts-ec` header when `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)` (returning user with valid consent). Also ingests Prebid EIDs from `ts-eids` cookie. + - Calls `set_ec_cookie_and_header_on_response()` when `ec_context.ec_generated == true` (newly generated ECs). Returning-user mismatch repair is not performed. Also ingests Prebid EIDs. - `route_request()` return type changes from `Result` to `Result<(), Error>`; response is flushed via `response.send_to_client()` instead of being returned. The `#[fastly::main]` entrypoint must also change to match. @@ -1962,102 +2469,84 @@ Wire `EcContext` into the request pipeline following the two-phase model --- -### Story 7 — Pixel sync (`GET /sync`) - -Implement the pixel-based ID sync endpoint that partners use to write their -user ID against an EC hash. +### Story 7 — Prebid EID cookie ingestion -**Scope:** `ec/sync_pixel.rs`, router update +Implement the server-side ingestion of the `ts-eids` cookie, which replaces +the pixel sync endpoint as the browser-side ID sync mechanism. +**Scope:** `ec/prebid_eids.rs`, `ec/finalize.rs` update **Acceptance criteria:** -- Missing required query params (`partner`, `uid`, `return`) → `400`. -- No valid `ts-ec` cookie (missing or malformed) → redirect to - `{return}?ts_synced=0&ts_reason=no_ec`. -- Unknown `partner` ID → `400`. -- `return` URL hostname not in `partner.allowed_return_domains` → `400`. -- Consent uses `ec_context.consent`. The optional `consent` query param is a fallback - only: it is used exclusively when `ec_context.consent.is_empty()` returns `true` - — meaning no consent signals of any kind are present (no TCF string, no GPP - string, no US Privacy string, no AC string, no GPC, no decoded consent objects). - Use the `ConsentContext::is_empty()` method directly; do not reimplement the - check from this description. If consent KV fallback or any other pre-routing - source has already populated `ec_context.consent`, `is_empty()` is `false` and - the param is ignored. - When the fallback applies, decode the consent string locally into a - `ConsentContext` and **assign it into `ec_context.consent`** so that both - the sync write and `ec_finalize_response()` share the same effective consent - (prevents a same-request "write partner ID, then withdraw EC" conflict). - Do NOT re-call `build_consent_context()` (that would trigger consent KV writes). - Denied or absent → redirect to `{return}?ts_synced=0&ts_reason=no_consent`. -- Rate limit exceeded → `429 Too Many Requests` (no redirect). -- KV write failure → redirect to `{return}?ts_synced=0&ts_reason=write_failed`. -- `kv.upsert_partner_id()` creates a minimal live root entry first when the EC - exists in the cookie but the identity graph key is still missing because the - original best-effort `create_or_revive()` failed on generation. -- Success → redirect to `{return}?ts_synced=1`. -- Return URL construction correctly appends `&` or `?` based on existing query string. -- Rate counter key: `{partner_id}:{ec_hash}`, 1-hour window, via `fastly::erl::RateCounter`. -- Unit tests cover all redirect/response codes and return URL construction. +- `ingest_prebid_eids(cookie_value, ec_id, kv, registry)` decodes a base64 JSON + array of OpenRTB-style `{source, uids:[...]}` objects and syncs matched partners to KV. The backend also accepts the earlier flattened `{source, id, atype}` payload for backward compatibility. +- Source domain matching via `registry.find_by_source_domain()` (case-insensitive). +- Sources with no non-empty UID are skipped. +- Idempotent write suppression: if the stored UID already matches the incoming UID, the write is skipped for that partner. +- KV write via `kv.upsert_partner_id()` — best-effort, errors logged at `warn`. +- Called from `ec_finalize_response()` on both returning-user and new-EC paths + when a `ts-eids` cookie is present and consent is granted. +- JS writer target size: 3 KB; backend parser raw-cookie limit: 8 KiB. +- All errors are logged and swallowed — never blocks the response. +- Unit tests cover base64 decode, JSON parse, source domain matching, size limits, + and empty/oversized UID handling. **Spec ref:** §8 --- -### Story 8 — Identity lookup (`GET /identify`) +### Story 8 — Identity lookup (`GET /_ts/api/v1/identify`) -Implement the browser-facing endpoint that publishers call to retrieve the EC -hash and synced partner UIDs for the current user. +Implement the partner-facing endpoint that authenticated partners call to +retrieve their own synced UID for the current EC. **Scope:** `ec/identify.rs`, router update **Acceptance criteria:** +- **Bearer token required.** Missing or invalid `Authorization: Bearer` → `401` + with `{ "error": "invalid_token" }`. Auth uses `registry.find_by_api_key_hash()`. - `!allows_ec_creation(consent)` (consent denied, regardless of EC presence) → `403 Forbidden`. - When EC is present but consent is denied, the handler returns `403` and - `ec_finalize_response()` deletes the cookie and writes a tombstone. + When the denial is an explicit withdrawal signal and a `ts-ec` cookie was present, `ec_finalize_response()` also deletes the cookie and writes a tombstone. Fail-closed / unverifiable-consent cases still return `403`, but they strip EC headers only. - No EC present (`ec_was_present == false`) and consent not denied → `204 No Content`. -- Valid EC, consent granted, KV read succeeds with entry → `200` with full JSON body - including `ec`, `consent`, `uids`, `eids`. -- Valid EC, consent granted, KV read succeeds but no entry (never synced or - `create_or_revive()` failed on generation) → `200` with `degraded: false`, - empty `uids`/`eids`. This is not an error — see §11.4. -- `uids` filtered to partners where `bidstream_enabled = true` and consent - granted. -- KV read error (store unavailable) → `200` with `degraded: true` and empty - `uids`/`eids`. +- Valid EC, consent granted, KV read succeeds with entry → `200` with scoped JSON body + including `ec`, `consent`, `partner_id`, `uid` (single partner's UID), `eid` + (single partner's OpenRTB EID object), `cluster_size`. +- Valid EC, consent granted, KV read succeeds but no entry for this partner → + `200` with `degraded: false`, `uid` and `eid` absent. Not an error — see §11.4. +- KV read error (store unavailable) → `200` with `degraded: true`, `uid` and + `eid` absent. +- Response scoped to the authenticated partner only — no multi-partner `uids`/`eids` maps. +- `X-ts-ec` response header set on `200` responses. - No `Origin` header (server-side proxy): process normally, no CORS headers, no `403`. - `Origin` header present and matches `publisher.domain` or subdomain: reflect in `Access-Control-Allow-Origin` + `Vary: Origin`. - `Origin` header present but does not match: `403`, no body. -- `OPTIONS /identify` preflight → `200` with CORS headers, no body. -- `generate_if_needed()` is never called — no new EC is generated. The handler - itself does not write cookies, but `ec_finalize_response()` may still delete - the cookie on withdrawal or reconcile it on header/cookie mismatch. +- `Access-Control-Allow-Headers` includes `Authorization, X-ts-ec`. +- `OPTIONS /_ts/api/v1/identify` preflight → `200` with CORS headers, no body. +- `generate_if_needed()` is never called — no new EC is generated. - Response time target: 30ms p95 (documented, not gate). -- Unit tests cover all response codes, degraded flag, `uids` filtering, - CORS origin validation. +- Unit tests cover Bearer auth (200/401/403/204), scoped partner response, + degraded flag, CORS origin validation. **Spec ref:** §11 --- -### Story 9 — S2S batch sync (`POST /api/v1/sync`) +### Story 9 — S2S batch sync (`POST /_ts/api/v1/batch-sync`) Implement the server-to-server batch sync endpoint for partners to bulk-write their UIDs against a list of EC hashes. -**Scope:** `ec/sync_batch.rs`, router update +**Scope:** `ec/batch_sync.rs`, `ec/rate_limiter.rs`, router update **Acceptance criteria:** -- Missing or invalid `Authorization: Bearer` → `401`. Auth uses index-based - lookup via `find_by_api_key_hash()` (§9.2) with constant-time hash verification. -- Auth KV lookup failure (store unavailable) → `503 Service Unavailable`. +- Missing or invalid `Authorization: Bearer` → `401`. Auth uses in-memory + lookup via `registry.find_by_api_key_hash()` (§9.2). - API-key rate limit exceeded (`batch_rate_limit` per partner per minute) → `429` with `{ "error": "rate_limit_exceeded" }`. - More than 1000 mappings → `400`. -- Per-mapping rejections: `invalid_ec_hash`, `ec_hash_not_found`, +- Per-mapping rejections: `invalid_ec_id`, `ec_id_not_found`, `consent_withdrawn`, `kv_unavailable`. - KV write failure aborts remaining mappings with `kv_unavailable`; partial results returned as `207`. @@ -2085,7 +2574,7 @@ runtime). Only fires on organic routes (§10.2). - Dispatch only when: EC present (including an EC generated on the current organic request), consent granted, `pull_sync_enabled = true`, and either no - existing partner entry or existing `synced` is older than `pull_sync_ttl_sec`. + existing partner entry; existing partner UIDs are not refreshed by pull sync. - Rate limit: `pull_sync_rate_limit` per EC hash per partner per hour; counter key `pull:{partner_id}:{ec_hash}`. - Maximum concurrent pulls per request: `settings.ec.pull_sync_concurrency` @@ -2100,7 +2589,7 @@ runtime). Only fires on organic routes (§10.2). - Dispatch runs after `send_to_client()` — does not add latency to the user-facing response. Uses `send_async()` + `PendingRequest::wait()` (blocking). - Only fires on organic routes (`handle_publisher_request`, `handle_proxy`) — - never on `/sync`, `/identify`, `/auction`, `/api/v1/sync`, or `/admin/*`. + never on `/_ts/api/v1/identify`, `/_ts/api/v1/batch-sync`, or `/auction`. - Unit tests cover trigger conditions, null/404 no-op, domain allowlist check, dispatch limit enforcement. @@ -2143,21 +2632,21 @@ across multiple handlers in a single simulated environment. **Acceptance criteria:** -- **Full flow:** First-party page load → EC generated → pixel sync writes - partner UID → `/identify` returns that UID → auction includes EID. +- **Full flow:** First-party page load → EC generated → Prebid EID cookie + ingestion writes partner UID → `/_ts/api/v1/identify` returns that UID + (scoped to authenticated partner) → auction includes EID. - **Consent withdrawal:** Request with denied consent clears EC cookie and writes a KV tombstone (`consent.ok = false`, 24h TTL) in the same request; subsequent - `/identify` with consent still denied returns `403` (consent denied → §11.4); + `/_ts/api/v1/identify` with consent still denied returns `403` (consent denied → §11.4); batch sync returns `consent_withdrawn` within the tombstone TTL. - **KV create failure:** EC cookie is still set when `create_or_revive()` fails - (best-effort). Subsequent `/identify` returns `200` with `degraded: false` and + (best-effort). Subsequent `/_ts/api/v1/identify` returns `200` with `degraded: false` and empty `uids`/`eids` (KV read succeeds — entry simply does not exist). -- **KV read failure:** `/identify` returns `200` with `degraded: true` and empty +- **KV read failure:** `/_ts/api/v1/identify` returns `200` with `degraded: true` and empty `uids`/`eids` (store unavailable, entry might exist but can't be read). - **Concurrent writes:** Two simultaneous EC creates for the same hash resolve without data loss (CAS retry). -- **Rate limits:** Pixel sync returns `429` after `sync_rate_limit` is - exceeded; batch sync returns `429` after `batch_rate_limit` is exceeded. +- **Rate limits:** Batch sync returns `429` after `batch_rate_limit` is exceeded. - **Pull sync no-op:** Partner returning `{ "uid": null }` produces no KV write and no error log. - All tests pass under `cargo test --workspace` with Viceroy. diff --git a/fastly.toml b/fastly.toml index 9d6c0f269..2ea512a6a 100644 --- a/fastly.toml +++ b/fastly.toml @@ -19,17 +19,32 @@ build = """ [local_server] address = "127.0.0.1:7676" - + [local_server.backends] [local_server.kv_stores] + [[local_server.kv_stores.counter_store]] + key = "placeholder" + data = "placeholder" + + [[local_server.kv_stores.opid_store]] + key = "placeholder" + data = "placeholder" + [[local_server.kv_stores.creative_store]] key = "placeholder" data = "placeholder" - [[local_server.kv_stores.consent_store]] + [[local_server.kv_stores.ec_identity_store]] key = "placeholder" data = "placeholder" + + # Pre-seeded test EC entry for local script testing (test-prebid-eids.sh). + # Matches the TEST_EC_ID used in that script. + [[local_server.kv_stores.ec_identity_store]] + key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01" + data = '{"v":1,"created":1700000000,"last_seen":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US"}}' + [local_server.secret_stores] [[local_server.secret_stores.signing_keys]] key = "ts-2025-10-A" diff --git a/scripts/integration-tests-browser.sh b/scripts/integration-tests-browser.sh index 888adb13d..fb1289d3e 100755 --- a/scripts/integration-tests-browser.sh +++ b/scripts/integration-tests-browser.sh @@ -32,7 +32,7 @@ echo "==> Validating shared integration-test dependency versions..." echo "==> Building WASM binary (origin=http://127.0.0.1:$ORIGIN_PORT)..." TRUSTED_SERVER__PUBLISHER__ORIGIN_URL="http://127.0.0.1:$ORIGIN_PORT" \ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET="integration-test-proxy-secret" \ -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY="integration-test-secret-key" \ +TRUSTED_SERVER__EC__PASSPHRASE="integration-test-ec-secret" \ TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false \ cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index 3b9ec974b..318b9323c 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -53,7 +53,7 @@ fi echo "==> Building WASM binary (origin=http://127.0.0.1:$ORIGIN_PORT)..." TRUSTED_SERVER__PUBLISHER__ORIGIN_URL="http://127.0.0.1:$ORIGIN_PORT" \ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET="integration-test-proxy-secret" \ -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY="integration-test-secret-key" \ +TRUSTED_SERVER__EC__PASSPHRASE="integration-test-ec-secret" \ TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false \ cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 diff --git a/trusted-server.toml b/trusted-server.toml index d9189aaa2..72f0672bf 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -4,7 +4,7 @@ username = "user" password = "pass" [[handlers]] -path = "^/admin" +path = "^/_ts/admin" username = "admin" password = "changeme" @@ -14,8 +14,45 @@ cookie_domain = ".test-publisher.com" origin_url = "https://origin.test-publisher.com" proxy_secret = "change-me-proxy-secret" -[edge_cookie] -secret_key = "trusted-server" +[ec] +passphrase = "trusted-server" +ec_store = "ec_identity_store" +pull_sync_concurrency = 3 +# cluster_trust_threshold = 10 # Entries with cluster_size <= this are individual users +# cluster_recheck_secs = 3600 # Re-evaluate cluster_size after this many seconds + +# [[ec.partners]] +# id = "liveramp" +# name = "LiveRamp" +# source_domain = "liveramp.com" +# openrtb_atype = 3 +# bidstream_enabled = true +# api_token = "partner-api-token" +# batch_rate_limit = 60 +# pull_sync_enabled = false + +[[ec.partners]] +id = "sharedid" +name = "Prebid SharedID" +source_domain = "sharedid.org" +openrtb_atype = 1 +bidstream_enabled = true +api_token = "sharedid-internal-token" + +# Integration test partners (used by crates/integration-tests) +[[ec.partners]] +id = "inttest" +name = "Integration Test Partner" +source_domain = "inttest.example.com" +bidstream_enabled = true +api_token = "inttest-api-key-1" + +[[ec.partners]] +id = "inttest2" +name = "Integration Test Partner 2" +source_domain = "inttest2.example.com" +bidstream_enabled = true +api_token = "inttest2-api-key-2" # Custom headers to be included in every response # Allows publishers to include tags such as X-Robots-Tag: noindex @@ -126,8 +163,9 @@ rewrite_script = true # mode = "restrictive" # "restrictive" | "newest" | "permissive" # freshness_threshold_days = 30 -# KV Store consent persistence (requires a KV store named "consent_store" in fastly.toml) -# consent_store = "consent_store" +# Consent is interpreted from request cookies, headers, geolocation, and these +# policy settings. EC identity lifecycle state and withdrawal tombstones are +# stored in the KV store configured by [ec].ec_store. # Rewrite configuration for creative HTML/CSS processing # [rewrite] @@ -189,5 +227,3 @@ timeout_ms = 1000 # query parameter name. Arrays are joined with commas. [integrations.adserver_mock.context_query_params] permutive_segments = "permutive" - -