diff --git a/.gitignore b/.gitignore index cd4d019..921ce02 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ src/*.html !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json +.specstory diff --git a/CHANGELOG.md b/CHANGELOG.md index 57e7085..a6e560a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added basic unit tests - Added publisher config - Add AI assist rules. Based on https://github.com/hashintel/hash +- Added ability to construct GAM requests from static permutive segments with test pages ### Changed - Upgrade to rust 1.87.0 diff --git a/Cargo.lock b/Cargo.lock index c929cb4..2689050 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -124,6 +139,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -199,7 +235,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.16", "once_cell", "tiny-keccak", ] @@ -630,6 +666,18 @@ dependencies = [ "wasi 0.11.1+wasi-snapshot-preview1", ] +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + [[package]] name = "gimli" version = "0.31.1" @@ -1158,6 +1206,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "redox_syscall" version = "0.5.13" @@ -1578,6 +1632,7 @@ dependencies = [ name = "trusted-server-common" version = "0.1.0" dependencies = [ + "brotli", "chrono", "config", "cookie", @@ -1598,6 +1653,8 @@ dependencies = [ "temp-env", "tokio", "url", + "urlencoding", + "uuid", ] [[package]] @@ -1655,12 +1712,29 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +dependencies = [ + "getrandom 0.3.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 2cae701..383d52e 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -9,6 +9,7 @@ publish = false license = "Apache-2.0" [dependencies] +brotli = "3.3" chrono = "0.4" config = "0.15.11" cookie = "0.18.1" @@ -27,6 +28,8 @@ serde_json = "1.0.91" sha2 = "0.10.9" tokio = { version = "1.46", features = ["sync", "macros", "io-util", "rt", "time"] } url = "2.4.1" +uuid = { version = "1.0", features = ["v4"] } +urlencoding = "2.1" [build-dependencies] serde = { version = "1.0", features = ["derive"] } diff --git a/crates/common/src/gam.rs b/crates/common/src/gam.rs new file mode 100644 index 0000000..bd42ac0 --- /dev/null +++ b/crates/common/src/gam.rs @@ -0,0 +1,675 @@ +use crate::gdpr::get_consent_from_request; +use crate::settings::Settings; +use fastly::http::{header, Method, StatusCode}; +use fastly::{Error, Request, Response}; +use serde_json::json; +use std::collections::HashMap; +use uuid::Uuid; + +/// GAM request builder for server-side ad requests +pub struct GamRequest { + pub publisher_id: String, + pub ad_units: Vec, + pub page_url: String, + pub correlator: String, + pub prmtvctx: Option, // Permutive context - initially hardcoded, then dynamic + pub user_agent: String, + pub synthetic_id: String, +} + +impl GamRequest { + /// Create a new GAM request with default parameters + pub fn new(settings: &Settings, req: &Request) -> Result { + let correlator = Uuid::new_v4().to_string(); + let page_url = req.get_url().to_string(); + let user_agent = req + .get_header(header::USER_AGENT) + .and_then(|h| h.to_str().ok()) + .unwrap_or("Mozilla/5.0 (compatible; TrustedServer/1.0)") + .to_string(); + + // Get synthetic ID from request headers + let synthetic_id = req + .get_header("X-Synthetic-Trusted-Server") + .and_then(|h| h.to_str().ok()) + .unwrap_or("unknown") + .to_string(); + + Ok(Self { + publisher_id: settings.gam.publisher_id.clone(), + ad_units: settings + .gam + .ad_units + .iter() + .map(|u| u.name.clone()) + .collect(), + page_url, + correlator, + prmtvctx: None, // Will be set later with captured value + user_agent, + synthetic_id, + }) + } + + /// Set the Permutive context (initially hardcoded from captured request) + pub fn with_prmtvctx(mut self, prmtvctx: String) -> Self { + self.prmtvctx = Some(prmtvctx); + self + } + + /// Build the GAM request URL for the "Golden URL" replay phase + pub fn build_golden_url(&self) -> String { + // This will be replaced with the actual captured URL from autoblog.com + // For now, using a template based on the captured Golden URL + let mut params = HashMap::new(); + + // Core GAM parameters (based on captured URL) + params.insert("pvsid".to_string(), "3290837576990024".to_string()); // Publisher Viewability ID + params.insert("correlator".to_string(), self.correlator.clone()); + params.insert( + "eid".to_string(), + "31086815,31093089,95353385,31085777,83321072".to_string(), + ); // Event IDs + params.insert("output".to_string(), "ldjh".to_string()); // Important: not 'json' + params.insert("gdfp_req".to_string(), "1".to_string()); + params.insert("vrg".to_string(), "202506170101".to_string()); // Version/Region + params.insert("ptt".to_string(), "17".to_string()); // Page Type + params.insert("impl".to_string(), "fifs".to_string()); // Implementation + + // Ad unit parameters (simplified version of captured format) + params.insert( + "iu_parts".to_string(), + format!("{},{},homepage", self.publisher_id, "trustedserver"), + ); + params.insert( + "enc_prev_ius".to_string(), + "/0/1/2,/0/1/2,/0/1/2".to_string(), + ); + params.insert("prev_iu_szs".to_string(), "320x50|300x250|728x90|970x90|970x250|1x2,320x50|300x250|728x90|970x90|970x250|1x2,320x50|300x250|728x90|970x90|970x250|1x2".to_string()); + params.insert("fluid".to_string(), "height,height,height".to_string()); + + // Browser context (simplified) + params.insert("biw".to_string(), "1512".to_string()); + params.insert("bih".to_string(), "345".to_string()); + params.insert("u_tz".to_string(), "-300".to_string()); + params.insert("u_cd".to_string(), "30".to_string()); + params.insert("u_sd".to_string(), "2".to_string()); + + // Page context + params.insert("url".to_string(), self.page_url.clone()); + params.insert( + "dt".to_string(), + chrono::Utc::now().timestamp_millis().to_string(), + ); + + // Add Permutive context if available (in cust_params like the captured URL) + if let Some(ref prmtvctx) = self.prmtvctx { + let cust_params = format!("permutive={}&puid={}", prmtvctx, self.synthetic_id); + params.insert("cust_params".to_string(), cust_params); + } + + // Build query string + let query_string = params + .iter() + .map(|(k, v)| format!("{}={}", k, urlencoding::encode(v))) + .collect::>() + .join("&"); + + format!("{}?{}", self.get_base_url(), query_string) + } + + /// Get the base GAM server URL + pub fn get_base_url(&self) -> String { + // This will be updated with the actual GAM endpoint from captured request + "https://securepubads.g.doubleclick.net/gampad/ads".to_string() + } + + /// Send the GAM request and return the response + pub async fn send_request(&self, _settings: &Settings) -> Result { + let url = self.build_golden_url(); + log::info!("Sending GAM request to: {}", url); + + // Create the request + let mut req = Request::new(Method::GET, &url); + + // Set headers to mimic a browser request (using only Fastly-compatible headers) + req.set_header(header::USER_AGENT, &self.user_agent); + req.set_header(header::ACCEPT, "application/json, text/plain, */*"); + req.set_header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9"); + req.set_header(header::ACCEPT_ENCODING, "gzip, deflate, br"); + req.set_header(header::REFERER, &self.page_url); + req.set_header(header::ORIGIN, &self.page_url); + req.set_header("X-Synthetic-ID", &self.synthetic_id); + + // Send the request to the GAM backend + let backend_name = "gam_backend"; + log::info!("Sending request to backend: {}", backend_name); + + match req.send(backend_name) { + Ok(mut response) => { + log::info!( + "Received GAM response with status: {}", + response.get_status() + ); + + // Log response headers for debugging + log::debug!("GAM Response headers:"); + for (name, value) in response.get_headers() { + log::debug!(" {}: {:?}", name, value); + } + + // Handle response body safely + let body_bytes = response.take_body_bytes(); + let body = match std::str::from_utf8(&body_bytes) { + Ok(body_str) => body_str.to_string(), + Err(e) => { + log::warn!("Could not read response body as UTF-8: {:?}", e); + + // Try to decompress if it's Brotli compressed + let mut decompressed = Vec::new(); + match brotli::BrotliDecompress( + &mut std::io::Cursor::new(&body_bytes), + &mut decompressed, + ) { + Ok(_) => match std::str::from_utf8(&decompressed) { + Ok(decompressed_str) => { + log::debug!( + "Successfully decompressed Brotli response: {} bytes", + decompressed_str.len() + ); + decompressed_str.to_string() + } + Err(e2) => { + log::warn!( + "Could not read decompressed body as UTF-8: {:?}", + e2 + ); + format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) + } + }, + Err(e2) => { + log::warn!("Could not decompress Brotli response: {:?}", e2); + // Return a placeholder since we can't parse the binary response + format!("{{\"error\": \"compression_failed\", \"message\": \"Could not decompress response\", \"original_error\": \"{:?}\"}}", e2) + } + } + } + }; + + log::debug!("GAM Response body length: {} bytes", body.len()); + + // For debugging, log first 500 chars of response + if body.len() > 500 { + log::debug!("GAM Response preview: {}...", &body[..500]); + } else { + log::debug!("GAM Response: {}", body); + } + + Ok(Response::from_status(response.get_status()) + .with_header(header::CONTENT_TYPE, "application/json") + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-GAM-Test", "true") + .with_header("X-Synthetic-ID", &self.synthetic_id) + .with_header("X-Correlator", &self.correlator) + .with_header("x-compress-hint", "on") + .with_body(body)) + } + Err(e) => { + log::error!("Error sending GAM request: {:?}", e); + Err(e.into()) + } + } + } +} + +/// Handle GAM test requests (Phase 1: Capture & Replay) +pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result { + log::info!("Starting GAM test request handling"); + + // Debug: Log all request headers + log::debug!("GAM Test - All request headers:"); + for (name, value) in req.get_headers() { + log::debug!(" {}: {:?}", name, value); + } + + // Check consent status from cookie (more reliable than header) + let consent = get_consent_from_request(&req).unwrap_or_default(); + let advertising_consent = consent.advertising; + + log::debug!("GAM Test - Consent from cookie: {:?}", consent); + log::debug!( + "GAM Test - Advertising consent from cookie: {}", + advertising_consent + ); + + // Also check header as fallback + let header_consent = req + .get_header("X-Consent-Advertising") + .and_then(|h| h.to_str().ok()) + .map(|v| v == "true") + .unwrap_or(false); + + log::debug!( + "GAM Test - Advertising consent from header: {}", + header_consent + ); + + // Use cookie consent as primary, header as fallback + let final_consent = advertising_consent || header_consent; + log::info!("GAM Test - Final advertising consent: {}", final_consent); + + if !final_consent { + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "No advertising consent", + "message": "GAM requests require advertising consent", + "debug": { + "cookie_consent": consent, + "header_consent": header_consent, + "final_consent": final_consent + } + }))?); + } + + // Create GAM request + let gam_req = match GamRequest::new(settings, &req) { + Ok(req) => { + log::info!("Successfully created GAM request"); + req + } + Err(e) => { + log::error!("Error creating GAM request: {:?}", e); + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to create GAM request", + "details": format!("{:?}", e) + }))?); + } + }; + + // For Phase 1, we'll use a hardcoded prmtvctx value from captured request + // This will be replaced with the actual value from autoblog.com + let gam_req_with_context = gam_req.with_prmtvctx("129627,137412,138272,139095,139096,139218,141364,143196,143210,143211,143214,143217,144331,144409,144438,144444,144488,144543,144663,144679,144731,144824,144916,145933,146347,146348,146349,146350,146351,146370,146383,146391,146392,146393,146424,146995,147077,147740,148616,148627,148628,149007,150420,150663,150689,150690,150692,150752,150753,150755,150756,150757,150764,150770,150781,150862,154609,155106,155109,156204,164183,164573,165512,166017,166019,166484,166486,166487,166488,166492,166494,166495,166497,166511,167639,172203,172544,173548,176066,178053,178118,178120,178121,178133,180321,186069,199642,199691,202074,202075,202081,233782,238158,adv,bhgp,bhlp,bhgw,bhlq,bhlt,bhgx,bhgv,bhgu,bhhb,rts".to_string()); + + log::info!( + "Sending GAM request with correlator: {}", + gam_req_with_context.correlator + ); + + match gam_req_with_context.send_request(settings).await { + Ok(response) => { + log::info!("GAM request successful"); + Ok(response) + } + Err(e) => { + log::error!("GAM request failed: {:?}", e); + Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to send GAM request", + "details": format!("{:?}", e) + }))?) + } + } +} + +/// Handle GAM golden URL replay (for testing captured requests) +pub async fn handle_gam_golden_url(_settings: &Settings, _req: Request) -> Result { + log::info!("Handling GAM golden URL replay"); + + // This endpoint will be used to test the exact captured URL from autoblog.com + // For now, return a placeholder response + Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "status": "golden_url_replay", + "message": "Ready for captured URL testing", + "next_steps": [ + "1. Capture complete GAM request URL from autoblog.com", + "2. Replace placeholder URL in GamRequest::build_golden_url()", + "3. Test with exact captured parameters" + ] + }))?) +} + +/// Handle GAM custom URL testing (for testing captured URLs directly) +pub async fn handle_gam_custom_url( + _settings: &Settings, + mut req: Request, +) -> Result { + log::info!("Handling GAM custom URL test"); + + // Check consent status from cookie + let consent = get_consent_from_request(&req).unwrap_or_default(); + let advertising_consent = consent.advertising; + + if !advertising_consent { + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "No advertising consent", + "message": "GAM requests require advertising consent" + }))?); + } + + // Parse the request body to get the custom URL + let body = req.take_body_str(); + let url_data: serde_json::Value = serde_json::from_str(&body).map_err(|e| { + log::error!("Error parsing request body: {:?}", e); + fastly::Error::msg("Invalid JSON in request body") + })?; + + let custom_url = url_data["url"] + .as_str() + .ok_or_else(|| fastly::Error::msg("Missing 'url' field in request body"))?; + + log::info!("Testing custom GAM URL: {}", custom_url); + + // Create a request to the custom URL + let mut gam_req = Request::new(Method::GET, custom_url); + + // Set headers to mimic a browser request + gam_req.set_header( + header::USER_AGENT, + "Mozilla/5.0 (compatible; TrustedServer/1.0)", + ); + gam_req.set_header(header::ACCEPT, "application/json, text/plain, */*"); + gam_req.set_header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9"); + gam_req.set_header(header::ACCEPT_ENCODING, "gzip, deflate, br"); + gam_req.set_header(header::REFERER, "https://www.autoblog.com/"); + gam_req.set_header(header::ORIGIN, "https://www.autoblog.com"); + + // Send the request to the GAM backend + let backend_name = "gam_backend"; + log::info!("Sending custom URL request to backend: {}", backend_name); + + match gam_req.send(backend_name) { + Ok(mut response) => { + log::info!( + "Received GAM response with status: {}", + response.get_status() + ); + + // Log response headers for debugging + log::debug!("GAM Response headers:"); + for (name, value) in response.get_headers() { + log::debug!(" {}: {:?}", name, value); + } + + // Handle response body safely + let body_bytes = response.take_body_bytes(); + let body = match std::str::from_utf8(&body_bytes) { + Ok(body_str) => body_str.to_string(), + Err(e) => { + log::warn!("Could not read response body as UTF-8: {:?}", e); + + // Try to decompress if it's Brotli compressed + let mut decompressed = Vec::new(); + match brotli::BrotliDecompress( + &mut std::io::Cursor::new(&body_bytes), + &mut decompressed, + ) { + Ok(_) => match std::str::from_utf8(&decompressed) { + Ok(decompressed_str) => { + log::debug!( + "Successfully decompressed Brotli response: {} bytes", + decompressed_str.len() + ); + decompressed_str.to_string() + } + Err(e2) => { + log::warn!("Could not read decompressed body as UTF-8: {:?}", e2); + format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) + } + }, + Err(e2) => { + log::warn!("Could not decompress Brotli response: {:?}", e2); + // Return a placeholder since we can't parse the binary response + format!("{{\"error\": \"compression_failed\", \"message\": \"Could not decompress response\", \"original_error\": \"{:?}\"}}", e2) + } + } + } + }; + + log::debug!("GAM Response body length: {} bytes", body.len()); + + Ok(Response::from_status(response.get_status()) + .with_header(header::CONTENT_TYPE, "application/json") + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-GAM-Test", "true") + .with_header("X-Custom-URL", "true") + .with_header("x-compress-hint", "on") + .with_body_json(&json!({ + "status": "custom_url_test", + "original_url": custom_url, + "response_status": response.get_status().as_u16(), + "response_body": body, + "message": "Custom URL test completed" + }))?) + } + Err(e) => { + log::error!("Error sending custom GAM request: {:?}", e); + Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to send custom GAM request", + "details": format!("{:?}", e), + "original_url": custom_url + }))?) + } + } +} + +/// Handle GAM response rendering in iframe +pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result { + log::info!("Handling GAM response rendering"); + + // Check consent status from cookie + let consent = get_consent_from_request(&req).unwrap_or_default(); + let advertising_consent = consent.advertising; + + if !advertising_consent { + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "No advertising consent", + "message": "GAM requests require advertising consent" + }))?); + } + + // Create GAM request and get response + let gam_req = match GamRequest::new(settings, &req) { + Ok(req) => req.with_prmtvctx("129627,137412,138272,139095,139096,139218,141364,143196,143210,143211,143214,143217,144331,144409,144438,144444,144488,144543,144663,144679,144731,144824,144916,145933,146347,146348,146349,146350,146351,146370,146383,146391,146392,146393,146424,146995,147077,147740,148616,148627,148628,149007,150420,150663,150689,150690,150692,150752,150753,150755,150756,150757,150764,150770,150781,150862,154609,155106,155109,156204,164183,164573,165512,166017,166019,166484,166486,166487,166488,166492,166494,166495,166497,166511,167639,172203,172544,173548,176066,178053,178118,178120,178121,178133,180321,186069,199642,199691,202074,202075,202081,233782,238158,adv,bhgp,bhlp,bhgw,bhlq,bhlt,bhgx,bhgv,bhgu,bhhb,rts".to_string()), + Err(e) => { + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to create GAM request", + "details": format!("{:?}", e) + }))?); + } + }; + + // Get GAM response + let gam_response = match gam_req.send_request(settings).await { + Ok(response) => response, + Err(e) => { + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to get GAM response", + "details": format!("{:?}", e) + }))?); + } + }; + + // Parse the GAM response to extract HTML + let response_body = gam_response.into_body_str(); + log::info!("Parsing GAM response for HTML extraction"); + + // The GAM response format is: {"/ad_unit_path":["html",0,null,null,0,90,728,0,0,null,null,null,null,null,[...],null,null,null,null,null,null,null,0,null,null,null,null,null,null,"creative_id","line_item_id"],"..."} + // We need to extract the HTML part after the JSON array + + let html_content = if response_body.contains("") { + // Find the start of HTML content + if let Some(html_start) = response_body.find("") { + let html = &response_body[html_start..]; + log::debug!("Extracted HTML content: {} bytes", html.len()); + html.to_string() + } else { + format!("

Error: Could not find HTML content in GAM response

{}
", + response_body.chars().take(500).collect::()) + } + } else { + // Fallback: return the raw response in a safe HTML wrapper + format!( + "

GAM Response (no HTML found):

{}
", + response_body.chars().take(1000).collect::() + ) + }; + + // Create a safe HTML page that renders the ad content in an iframe + let render_page = format!( + r#" + + + + + GAM Ad Render Test + + + +
+
+

🎯 GAM Ad Render Test

+

Rendering Google Ad Manager response in iframe

+
+ +
+ Status: Ad content loaded successfully
+ Response Size: {} bytes
+ Timestamp: {} +
+ +
+ + + +
+ + + + +
+ + + +"#, + html_content.len(), + chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"), + html_content.replace("\"", """).replace("'", "'"), + html_content.len(), + html_content.chars().take(200).collect::() + ); + + Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-GAM-Render", "true") + .with_header("X-Synthetic-ID", &gam_req.synthetic_id) + .with_header("X-Correlator", &gam_req.correlator) + .with_body(render_page)) +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index d425c2c..89f58b4 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -21,6 +21,7 @@ pub mod constants; pub mod cookies; pub mod error; +pub mod gam; pub mod gdpr; pub mod models; pub mod prebid; diff --git a/crates/common/src/settings.rs b/crates/common/src/settings.rs index 05f372e..f218162 100644 --- a/crates/common/src/settings.rs +++ b/crates/common/src/settings.rs @@ -27,6 +27,22 @@ pub struct Prebid { pub server_url: String, } +#[derive(Debug, Default, Deserialize, Serialize)] +#[allow(unused)] +pub struct GamAdUnit { + pub name: String, + pub size: String, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +#[allow(unused)] +pub struct Gam { + pub publisher_id: String, + pub server_url: String, + pub ad_units: Vec, +} + +#[allow(unused)] #[derive(Debug, Default, Deserialize, Serialize)] pub struct Synthetic { pub counter_store: String, @@ -40,6 +56,7 @@ pub struct Settings { pub ad_server: AdServer, pub publisher: Publisher, pub prebid: Prebid, + pub gam: Gam, pub synthetic: Synthetic, } diff --git a/crates/common/src/templates.rs b/crates/common/src/templates.rs index 76f5a65..c4aae3c 100644 --- a/crates/common/src/templates.rs +++ b/crates/common/src/templates.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + pub const HTML_TEMPLATE: &str = r#" @@ -174,6 +176,9 @@ pub const HTML_TEMPLATE: &str = r#" } function saveConsent(consent) { + // Set the cookie first + document.cookie = `gdpr_consent=${JSON.stringify(consent)}; path=/; max-age=31536000`; // 1 year expiry + fetch('/gdpr/consent', { method: 'POST', headers: { @@ -184,16 +189,20 @@ pub const HTML_TEMPLATE: &str = r#" document.getElementById('gdpr-banner').classList.remove('visible'); document.getElementById('gdpr-preferences').classList.remove('visible'); document.querySelector('.overlay').classList.remove('visible'); - location.reload(); + // Remove the reload - we'll let the page continue with the new consent + }).catch(error => { + console.error('Error saving consent:', error); }); } // Load ads and tracking based on consent window.addEventListener('load', function() { - showGdprBanner(); + const consent = getCookie('gdpr_consent'); + if (!consent) { + showGdprBanner(); + } // Get consent status - const consent = getCookie('gdpr_consent'); const consentData = consent ? JSON.parse(consent) : { advertising: false, functional: false }; // Always make the prebid request, but include consent information @@ -319,3 +328,415 @@ pub const HTML_TEMPLATE: &str = r#" "#; + +pub const GAM_TEST_TEMPLATE: &str = r#" + + + + + + GAM Test - Trusted Server + + + +
+

GAM Test - Headless GPT PoC

+ +
+

📋 Instructions for Capture & Replay Phase

+

Phase 1 Goal: Capture a complete, successful ad request URL from autoblog.com and replay it from our server.

+
    +
  1. Open browser dev tools on autoblog.com
    2. +
  2. Go to Network tab and filter by "g.doubleclick.net"
    3. +
  3. Refresh the page and look for successful ad requests
    4. +
  4. Copy the complete URL with all parameters
    5. +
  5. Use the "Test Golden URL" button below to test it
    6. +
+
+ +
+

Phase 1: Capture & Replay (Golden URL)

+

Test the exact captured URL from autoblog.com to prove network connectivity.

+ +
+

Golden URL Test

+

Paste the captured GAM URL from autoblog.com below and test it:

+
+ +
+ + + +
+
+ +
+

Phase 2: Dynamic Request Building

+

Test dynamic parameter generation with hardcoded prmtvctx value.

+ +
+

Dynamic GAM Request

+

Test server-side GAM request with dynamic correlator and synthetic ID.

+ + +
+
+ +
+

Phase 3: Ad Rendering in iFrame

+

Render the GAM response HTML content in a sandboxed iframe for visual testing.

+ +
+

Ad Render Test

+

Test rendering the GAM response as an actual ad in an iframe:

+ + + +
+
+ +
+

Debug Information

+
+

Request Headers

+
+ +

Synthetic ID Status

+
+ Checking synthetic ID... +
+
+
+
+ + + + +"#; +// GAM Configuration Template +#[allow(dead_code)] +struct GamConfigTemplate { + publisher_id: String, + ad_units: Vec, + page_context: PageContext, + data_providers: Vec, +} +#[allow(dead_code)] +struct AdUnitConfig { + name: String, + sizes: Vec, + position: String, + targeting: HashMap, +} +#[allow(dead_code)] +struct PageContext { + page_type: String, + section: String, + keywords: Vec, +} +#[allow(dead_code)] +enum DataProvider { + Permutive(PermutiveConfig), + Lotame(LotameConfig), + Neustar(NeustarConfig), + Custom(CustomProviderConfig), +} +#[allow(dead_code)] +struct PermutiveConfig {} +#[allow(dead_code)] +struct LotameConfig {} +#[allow(dead_code)] +struct NeustarConfig {} +#[allow(dead_code)] +struct CustomProviderConfig {} +#[allow(dead_code)] +trait DataProviderTrait { + fn get_user_segments(&self, user_id: &str) -> Vec; +} + +#[allow(dead_code)] +struct RequestContext { + user_id: String, + page_url: String, + consent_status: bool, +} + +#[allow(dead_code)] +struct DynamicGamBuilder { + base_config: GamConfigTemplate, + context: RequestContext, + data_providers: Vec>, +} + +// Instead of hardcoded strings, use templates: +// "cust_params": "{{#each data_providers}}{{name}}={{segments}}&{{/each}}puid={{user_id}}" + +// This could generate: +// "permutive=129627,137412...&lotame=segment1,segment2&puid=abc123" + +// let context = data_provider_manager.build_context(&user_id, &request_context); +// let gam_req_with_context = gam_req.with_dynamic_context(context); diff --git a/crates/common/src/test_support.rs b/crates/common/src/test_support.rs index 67f7262..1b8da25 100644 --- a/crates/common/src/test_support.rs +++ b/crates/common/src/test_support.rs @@ -1,6 +1,6 @@ #[cfg(test)] pub mod tests { - use crate::settings::{AdServer, Prebid, Publisher, Settings, Synthetic}; + use crate::settings::{AdServer, Gam, GamAdUnit, Prebid, Publisher, Settings, Synthetic}; pub fn crate_test_settings_str() -> String { r#" @@ -16,6 +16,16 @@ pub mod tests { [prebid] server_url = "https://test-prebid.com/openrtb2/auction" + [gam] + publisher_id = "3790" + server_url = "https://securepubads.g.doubleclick.net/gampad/ads" + ad_units = [ + { name = "Flex8:1", size = "flexible" }, + { name = "Fixed728x90", size = "728x90" }, + { name = "Static8:1", size = "flexible" }, + { name = "Static728x90", size = "728x90" } + ] + [synthetic] counter_store = "test-counter-store" opid_store = "test-opid-store" @@ -38,6 +48,11 @@ pub mod tests { prebid: Prebid { server_url: "https://test-prebid.com/openrtb2/auction".to_string(), }, + gam: Gam { + publisher_id: "test-publisher-id".to_string(), + server_url: "https://securepubads.g.doubleclick.net/gampad/ads".to_string(), + ad_units: vec![GamAdUnit { name: "test-ad-unit".to_string(), size: "300x250".to_string() }], + }, synthetic: Synthetic { counter_store: "test_counter_store".to_string(), opid_store: "test-opid-store".to_string(), diff --git a/crates/fastly/src/main.rs b/crates/fastly/src/main.rs index e32d593..77cb614 100644 --- a/crates/fastly/src/main.rs +++ b/crates/fastly/src/main.rs @@ -17,6 +17,9 @@ use trusted_server_common::constants::{ HEADER_X_GEO_INFO_AVAILABLE, HEADER_X_GEO_METRO_CODE, }; use trusted_server_common::cookies::create_synthetic_cookie; +use trusted_server_common::gam::{ + handle_gam_custom_url, handle_gam_golden_url, handle_gam_render, handle_gam_test, +}; // Note: TrustedServerError is used internally by the common crate use trusted_server_common::gdpr::{ get_consent_from_request, handle_consent_request, handle_data_subject_request, GdprConsent, @@ -26,11 +29,12 @@ use trusted_server_common::prebid::PrebidRequest; use trusted_server_common::privacy::PRIVACY_TEMPLATE; use trusted_server_common::settings::Settings; use trusted_server_common::synthetic::{generate_synthetic_id, get_or_generate_synthetic_id}; -use trusted_server_common::templates::HTML_TEMPLATE; +use trusted_server_common::templates::{GAM_TEST_TEMPLATE, HTML_TEMPLATE}; use trusted_server_common::why::WHY_TEMPLATE; #[fastly::main] fn main(req: Request) -> Result { + // Print Settings only once at the beginning let settings = match Settings::new() { Ok(s) => s, Err(e) => { @@ -39,6 +43,12 @@ fn main(req: Request) -> Result { } }; log::info!("Settings {settings:?}"); + // Print User IP address immediately after Fastly Service Version + let client_ip = req + .get_client_ip_addr() + .map(|ip| ip.to_string()) + .unwrap_or_else(|| "Unknown".to_string()); + log::info!("User IP: {}", client_ip); futures::executor::block_on(async { log::info!( @@ -50,6 +60,14 @@ fn main(req: Request) -> Result { (&Method::GET, "/") => handle_main_page(&settings, req), (&Method::GET, "/ad-creative") => handle_ad_request(&settings, req), (&Method::GET, "/prebid-test") => handle_prebid_test(&settings, req).await, + (&Method::GET, "/gam-test") => handle_gam_test(&settings, req).await, + (&Method::GET, "/gam-golden-url") => handle_gam_golden_url(&settings, req).await, + (&Method::POST, "/gam-test-custom-url") => handle_gam_custom_url(&settings, req).await, + (&Method::GET, "/gam-render") => handle_gam_render(&settings, req).await, + (&Method::GET, "/gam-test-page") => Ok(Response::from_status(StatusCode::OK) + .with_body(GAM_TEST_TEMPLATE) + .with_header(header::CONTENT_TYPE, "text/html") + .with_header("x-compress-hint", "on")), (&Method::GET, "/gdpr/consent") => handle_consent_request(&settings, req), (&Method::POST, "/gdpr/consent") => handle_consent_request(&settings, req), (&Method::GET, "/gdpr/data") => handle_data_subject_request(&settings, req), @@ -376,11 +394,10 @@ fn handle_ad_request(settings: &Settings, mut req: Request) -> Result Result