Skip to content

Commit 1a7eeb3

Browse files
committed
Merge branch 'master' into release/0.17
2 parents d3e6f81 + 6ec1aae commit 1a7eeb3

File tree

8 files changed

+295
-33
lines changed

8 files changed

+295
-33
lines changed

Dockerfile

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
# NOTE: Ensure builder's Rust version matches CI's in .circleci/config.yml # RUST_VER
2-
FROM docker.io/lukemathwalker/cargo-chef:0.1.67-rust-1.78-bookworm as chef
1+
# NOTE: Ensure builder's Rust version matches CI's in .circleci/config.yml
2+
FROM docker.io/lukemathwalker/cargo-chef:0.1.67-rust-1.78-bullseye as chef
33
WORKDIR /app
44

55
FROM chef AS planner
@@ -14,7 +14,7 @@ RUN \
1414
# Fetch and load the MySQL public key. We need to install libmysqlclient-dev to build syncstorage-rs
1515
# which wants the mariadb
1616
wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
17-
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
17+
echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
1818
apt-get -q update && \
1919
apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake
2020

@@ -31,14 +31,14 @@ COPY --from=cacher $CARGO_HOME /app/$CARGO_HOME
3131
RUN \
3232
# Fetch and load the MySQL public key
3333
wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
34-
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
34+
echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
3535
# mysql_pubkey.asc from:
3636
# https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html
3737
# related:
3838
# https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup
3939
apt-get -q update && \
40-
apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake golang-go pkg-config python3-dev python3-pip python3-setuptools python3-wheel && \
41-
pip3 install --break-system-packages -r /app/requirements.txt && \
40+
apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \
41+
pip3 install -r requirements.txt && \
4242
rm -rf /var/lib/apt/lists/*
4343

4444
ENV PATH=$PATH:/root/.cargo/bin
@@ -49,7 +49,7 @@ RUN \
4949
cargo install --path ./syncserver --no-default-features --features=syncstorage-db/$DATABASE_BACKEND --features=py_verifier --locked --root /app && \
5050
if [ "$DATABASE_BACKEND" = "spanner" ] ; then cargo install --path ./syncstorage-spanner --locked --root /app --bin purge_ttl ; fi
5151

52-
FROM docker.io/library/debian:bookworm-slim
52+
FROM docker.io/library/debian:bullseye-slim
5353
WORKDIR /app
5454
COPY --from=builder /app/requirements.txt /app
5555
# Due to a build error that occurs with the Python cryptography package, we
@@ -69,17 +69,17 @@ RUN \
6969
apt-get -q update && \
7070
# and ca-certificates needed for https://repo.mysql.com
7171
apt-get install -y gnupg ca-certificates wget && \
72-
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
7372
# Fetch and load the MySQL public key
73+
echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
7474
wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
7575
# update again now that we trust repo.mysql.com
7676
apt-get -q update && \
77-
apt-get -q install -y build-essential libmysqlclient-dev libssl-dev libffi-dev libcurl4 pkg-config python3-dev python3-pip python3-setuptools python3-wheel cargo curl jq && \
77+
apt-get -q install -y build-essential libmysqlclient-dev libssl-dev libffi-dev libcurl4 python3-dev python3-pip python3-setuptools python3-wheel cargo curl jq && \
7878
# The python3-cryptography debian package installs version 2.6.1, but we
7979
# we want to use the version specified in requirements.txt. To do this,
8080
# we have to remove the python3-cryptography package here.
8181
apt-get -q remove -y python3-cryptography && \
82-
pip3 install --break-system-packages -r /app/requirements.txt && \
82+
pip3 install -r /app/requirements.txt && \
8383
rm -rf /var/lib/apt/lists/*
8484

8585
COPY --from=builder /app/bin /app/bin
@@ -92,10 +92,8 @@ COPY --from=builder /app/scripts/start_mock_fxa_server.sh /app/scripts/start_moc
9292
COPY --from=builder /app/syncstorage-spanner/src/schema.ddl /app/schema.ddl
9393

9494
RUN chmod +x /app/scripts/prepare-spanner.sh
95-
RUN \
96-
pip3 install --break-system-packages -r /app/tools/integration_tests/requirements.txt
97-
RUN \
98-
pip3 install --break-system-packages -r /app/tools/tokenserver/requirements.txt
95+
RUN pip3 install -r /app/tools/integration_tests/requirements.txt
96+
RUN pip3 install -r /app/tools/tokenserver/requirements.txt
9997

10098
USER app:app
10199

syncserver/src/server/test.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ fn create_hawk_header(method: &str, port: u16, path: &str) -> String {
174174
user_id: 42,
175175
fxa_uid: format!("xxx_test_uid_{}", *RAND_UID),
176176
fxa_kid: format!("xxx_test_kid_{}", *RAND_UID),
177+
hashed_fxa_uid: format!("xxx_test_hashed_fxa_uid_{}", *RAND_UID),
177178
hashed_device_id: "xxx_test".to_owned(),
178179
tokenserver_origin: Default::default(),
179180
};

syncserver/src/server/user_agent.rs

Lines changed: 255 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use std::fmt;
2+
13
use woothee::parser::{Parser, WootheeResult};
24

35
// List of valid user-agent attributes to keep, anything not in this
@@ -11,6 +13,185 @@ const VALID_UA_BROWSER: &[&str] = &["Chrome", "Firefox", "Safari", "Opera"];
1113
// field). Windows has many values and we only care that its Windows
1214
const VALID_UA_OS: &[&str] = &["Firefox OS", "Linux", "Mac OSX"];
1315

16+
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
17+
pub enum Platform {
18+
FirefoxDesktop,
19+
Fenix,
20+
FirefoxIOS,
21+
#[default]
22+
Other,
23+
}
24+
25+
impl fmt::Display for Platform {
26+
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
27+
let name = format!("{:?}", self).to_lowercase();
28+
write!(fmt, "{}", name)
29+
}
30+
}
31+
32+
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
33+
pub enum DeviceFamily {
34+
Desktop,
35+
Mobile,
36+
Tablet,
37+
#[default]
38+
Other,
39+
}
40+
41+
impl fmt::Display for DeviceFamily {
42+
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
43+
let name = format!("{:?}", self).to_lowercase();
44+
write!(fmt, "{}", name)
45+
}
46+
}
47+
48+
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
49+
pub enum OsFamily {
50+
Windows,
51+
MacOs,
52+
Linux,
53+
IOS,
54+
Android,
55+
#[default]
56+
Other,
57+
}
58+
59+
impl fmt::Display for OsFamily {
60+
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
61+
let name = format!("{:?}", self).to_lowercase();
62+
write!(fmt, "{}", name)
63+
}
64+
}
65+
66+
#[derive(Debug, Default, Eq, PartialEq)]
67+
pub struct DeviceInfo {
68+
pub platform: Platform,
69+
pub device_family: DeviceFamily,
70+
pub os_family: OsFamily,
71+
pub firefox_version: u32,
72+
}
73+
74+
impl DeviceInfo {
75+
/// Determine if the device is a desktop device based on either the form factor or OS.
76+
pub fn is_desktop(&self) -> bool {
77+
matches!(&self.device_family, DeviceFamily::Desktop)
78+
|| matches!(
79+
&self.os_family,
80+
OsFamily::MacOs | OsFamily::Windows | OsFamily::Linux
81+
)
82+
}
83+
84+
/// Determine if the device is a mobile phone based on either the form factor or OS.
85+
pub fn is_mobile(&self) -> bool {
86+
matches!(&self.device_family, DeviceFamily::Mobile)
87+
&& matches!(&self.os_family, OsFamily::Android | OsFamily::IOS)
88+
}
89+
90+
/// Determine if the device is iOS based on either the form factor or OS.
91+
pub fn is_ios(&self) -> bool {
92+
matches!(&self.device_family, DeviceFamily::Mobile)
93+
&& matches!(&self.os_family, OsFamily::IOS)
94+
}
95+
96+
/// Determine if the device is an android (Fenix) device based on either the form factor or OS.
97+
pub fn is_fenix(&self) -> bool {
98+
matches!(&self.device_family, DeviceFamily::Mobile)
99+
&& matches!(&self.os_family, OsFamily::Android)
100+
}
101+
}
102+
103+
/// Parses user agents from headers and returns a DeviceInfo struct containing
104+
/// DeviceFamily, OsFamily, Platform, and Firefox Version.
105+
///
106+
/// Intended to handle standard user agent strings but also accomodates the non-standard,
107+
/// Firefox-specific user agents for iOS and desktop.
108+
///
109+
/// It is theoretically possible to have an invalid user agent that is non-Firefox in the
110+
/// case of an invalid UA, bot, or scraper.
111+
/// There is a check for this to return an empty result as opposed to failing.
112+
///
113+
/// Parsing logic for non-standard iOS strings are in the form Firefox-iOS-FxA/24 and
114+
/// manually modifies WootheeResult to match with correct enums for iOS platform and OS.
115+
/// FxSync/<...>.desktop result still parses natively with Woothee and doesn't require intervention.
116+
pub fn get_device_info(user_agent: &str) -> DeviceInfo {
117+
let mut w_result: WootheeResult<'_> = Parser::new().parse(user_agent).unwrap_or_default();
118+
119+
// Current Firefox-iOS logic outputs the `user_agent` in the following formats:
120+
// Firefox-iOS-Sync/108.1b24234 (iPad; iPhone OS 16.4.1) (Firefox)
121+
// OR
122+
// Firefox-iOS-FxA/24
123+
// Both contain prefix `Firefox-iOS` and are not successfully parsed by Woothee.
124+
// This custom logic accomodates the current state (Q4 - 2024)
125+
// This may be a discussion point for future client-side adjustment to have a more standardized
126+
// user_agent string.
127+
if user_agent.to_lowercase().starts_with("firefox-ios") {
128+
w_result.name = "firefox";
129+
w_result.category = "smartphone";
130+
w_result.os = "iphone";
131+
}
132+
133+
// NOTE: Firefox on iPads report back the Safari "desktop" UA
134+
// (e.g. `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15
135+
// (KHTML, like Gecko) Version/13.1 Safari/605.1.15)`
136+
// therefore we have to accept that one. This does mean that we may presume
137+
// that a mac safari UA is an iPad.
138+
if w_result.name.to_lowercase() == "safari" && !user_agent.to_lowercase().contains("firefox/") {
139+
w_result.name = "firefox";
140+
w_result.category = "smartphone";
141+
w_result.os = "ipad";
142+
}
143+
144+
// Check if the user agent is not Firefox and return empty.
145+
if !["firefox"].contains(&w_result.name.to_lowercase().as_str()) {
146+
return DeviceInfo::default();
147+
}
148+
149+
let os = w_result.os.to_lowercase();
150+
let os_family = match os.as_str() {
151+
_ if os.starts_with("windows") => OsFamily::Windows,
152+
"mac osx" => OsFamily::MacOs,
153+
"linux" => OsFamily::Linux,
154+
"iphone" | "ipad" => OsFamily::IOS,
155+
"android" => OsFamily::Android,
156+
_ => OsFamily::Other,
157+
};
158+
159+
let device_family = match w_result.category {
160+
"pc" => DeviceFamily::Desktop,
161+
"smartphone" if os.as_str() == "ipad" => DeviceFamily::Tablet,
162+
"smartphone" => DeviceFamily::Mobile,
163+
_ => DeviceFamily::Other,
164+
};
165+
166+
let platform = match device_family {
167+
DeviceFamily::Desktop => Platform::FirefoxDesktop,
168+
DeviceFamily::Mobile => match os_family {
169+
OsFamily::IOS => Platform::FirefoxIOS,
170+
OsFamily::Android => Platform::Fenix,
171+
_ => Platform::Other,
172+
},
173+
DeviceFamily::Tablet => match os_family {
174+
OsFamily::IOS => Platform::FirefoxIOS,
175+
_ => Platform::Other,
176+
},
177+
DeviceFamily::Other => Platform::Other,
178+
};
179+
180+
let firefox_version = w_result
181+
.version
182+
.split('.')
183+
.next()
184+
.and_then(|v| v.parse::<u32>().ok())
185+
.unwrap_or(0);
186+
187+
DeviceInfo {
188+
platform,
189+
device_family,
190+
os_family,
191+
firefox_version,
192+
}
193+
}
194+
14195
pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) {
15196
let parser = Parser::new();
16197
let wresult = parser.parse(agent).unwrap_or_else(|| WootheeResult {
@@ -41,7 +222,9 @@ pub fn parse_user_agent(agent: &str) -> (WootheeResult<'_>, &str, &str) {
41222

42223
#[cfg(test)]
43224
mod tests {
44-
use super::parse_user_agent;
225+
use crate::server::user_agent::{DeviceFamily, OsFamily, Platform};
226+
227+
use super::{get_device_info, parse_user_agent};
45228

46229
#[test]
47230
fn test_linux() {
@@ -81,4 +264,75 @@ mod tests {
81264
assert_eq!(metrics_browser, "Other");
82265
assert_eq!(ua_result.name, "UNKNOWN");
83266
}
267+
268+
#[test]
269+
fn test_windows_desktop() {
270+
let user_agent = r#"Firefox/130.0.1 (Windows NT 10.0; Win64; x64) FxSync/1.132.0.20240913135723.desktop"#;
271+
let device_info = get_device_info(user_agent);
272+
assert_eq!(device_info.platform, Platform::FirefoxDesktop);
273+
assert_eq!(device_info.device_family, DeviceFamily::Desktop);
274+
assert_eq!(device_info.os_family, OsFamily::Windows);
275+
assert_eq!(device_info.firefox_version, 130);
276+
}
277+
278+
#[test]
279+
fn test_macos_desktop() {
280+
let user_agent =
281+
r#"Firefox/130.0.1 (Intel Mac OS X 10.15) FxSync/1.132.0.20240913135723.desktop"#;
282+
let device_info = get_device_info(user_agent);
283+
assert_eq!(device_info.platform, Platform::FirefoxDesktop);
284+
assert_eq!(device_info.device_family, DeviceFamily::Desktop);
285+
assert_eq!(device_info.os_family, OsFamily::MacOs);
286+
assert_eq!(device_info.firefox_version, 130);
287+
}
288+
289+
#[test]
290+
fn test_fenix() {
291+
let user_agent = r#"Mozilla/5.0 (Android 13; Mobile; rv:130.0) Gecko/130.0 Firefox/130.0"#;
292+
let device_info = get_device_info(user_agent);
293+
assert_eq!(device_info.platform, Platform::Fenix);
294+
assert_eq!(device_info.device_family, DeviceFamily::Mobile);
295+
assert_eq!(device_info.os_family, OsFamily::Android);
296+
assert_eq!(device_info.firefox_version, 130);
297+
}
298+
299+
#[test]
300+
fn test_firefox_ios() {
301+
let user_agent = r#"Firefox-iOS-FxA/24"#;
302+
let device_info = get_device_info(user_agent);
303+
assert_eq!(device_info.platform, Platform::FirefoxIOS);
304+
assert_eq!(device_info.device_family, DeviceFamily::Mobile);
305+
assert_eq!(device_info.os_family, OsFamily::IOS);
306+
assert_eq!(device_info.firefox_version, 0);
307+
}
308+
309+
#[test]
310+
fn test_firefox_ios_alternate_user_agent() {
311+
let user_agent = r#"Firefox-iOS-Sync/115.0b32242 (iPhone; iPhone OS 17.7) (Firefox)"#;
312+
let device_info = get_device_info(user_agent);
313+
assert_eq!(device_info.platform, Platform::FirefoxIOS);
314+
assert_eq!(device_info.device_family, DeviceFamily::Mobile);
315+
assert_eq!(device_info.os_family, OsFamily::IOS);
316+
assert_eq!(device_info.firefox_version, 0);
317+
}
318+
319+
#[test]
320+
fn test_platform_other() {
321+
let user_agent = r#"Mozilla/5.0 (Linux; Android 9; SM-A920F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4216.0 Mobile Safari/537.36"#;
322+
let device_info = get_device_info(user_agent);
323+
assert_eq!(device_info.platform, Platform::Other);
324+
assert_eq!(device_info.device_family, DeviceFamily::Other);
325+
assert_eq!(device_info.os_family, OsFamily::Other);
326+
assert_eq!(device_info.firefox_version, 0);
327+
}
328+
329+
#[test]
330+
fn test_non_firefox_platform_other() {
331+
let user_agent = r#"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)"#;
332+
let device_info = get_device_info(user_agent);
333+
assert_eq!(device_info.platform, Platform::Other);
334+
assert_eq!(device_info.device_family, DeviceFamily::Other);
335+
assert_eq!(device_info.os_family, OsFamily::Other);
336+
assert_eq!(device_info.firefox_version, 0);
337+
}
84338
}

syncserver/src/web/auth.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ pub struct HawkPayload {
5353
#[serde(default)]
5454
pub fxa_kid: String,
5555

56+
#[serde(default)]
57+
pub hashed_fxa_uid: String,
58+
5659
#[serde(default)]
5760
pub hashed_device_id: String,
5861

@@ -156,6 +159,7 @@ impl HawkPayload {
156159
user_id,
157160
fxa_uid: "xxx_test".to_owned(),
158161
fxa_kid: "xxx_test".to_owned(),
162+
hashed_fxa_uid: "xxx_test".to_owned(),
159163
hashed_device_id: "xxx_test".to_owned(),
160164
tokenserver_origin: Default::default(),
161165
}
@@ -508,6 +512,7 @@ mod tests {
508512
user_id: 1,
509513
fxa_uid: "319b98f9961ff1dbdd07313cd6ba925a".to_owned(),
510514
fxa_kid: "de697ad66d845b2873c9d7e13b8971af".to_owned(),
515+
hashed_fxa_uid: "0e8df5d41398a389913bd8402435649518af46493da1d4a437a46dc1784c501a".to_owned(),
511516
hashed_device_id: "2bcb92f4d4698c3d7b083a3c698a16ccd78bc2a8d20a96e4bb128ddceaf4e0b6".to_owned(),
512517
tokenserver_origin: Default::default(),
513518
},

0 commit comments

Comments
 (0)