Skip to content

Commit 62b49c3

Browse files
jcscottiiijcscottiii
authored
Refactor: Decouple web-features processing from raw schema (#1827)
Background: v3.0 schema is stable now. And introducing it directly was a headache. So I refactored the structs. This commit refactors the web feature ingestion process to decouple it from the raw, auto-generated JSON schema types from the `web-platform-dx/web-features` repository. This change adds support for the new v3 schema format while maintaining backward compatibility with the old format. Previously, the application directly used structs generated from the JSON schema. This created a tight coupling, making the system brittle and difficult to adapt to schema changes. To address this, the following changes were made: - A new set of internal, stable structs has been introduced in `lib/webdxfeaturetypes/types.go`. These structs define the canonical, application-specific representation of web feature data, independent of any specific schema version. - The data parser in `workflows/steps/services/web_feature_consumer/pkg/data/parser.go` has been updated. It now includes logic to process both the old schema and the new v3 schema, transforming the raw, version-specific data into the new stable `webdxfeaturetypes` structs. - All consumers of web feature data throughout the codebase (including Spanner adapters, Datastore adapters, and workflows) have been updated to use the new stable `webdxfeaturetypes` instead of the auto-generated types. - The `StringOrStringArray` type from the source data is now consistently normalized into a `[]string` in the new intermediate representation, simplifying downstream logic. This decoupling provides a robust abstraction layer that isolates the core application from upstream schema changes, making it significantly easier to support future versions of the `web-features` data format with minimal impact on the rest of the system.
1 parent 0bccb66 commit 62b49c3

26 files changed

+1323
-674
lines changed

‎Makefile‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,10 @@ node-openapi:
170170
################################
171171
JSONSCHEMA_OUT_DIR = lib/gen/jsonschema
172172

173+
# TODO: Once v3 is merged into main, removed the old defs.schema.json and change the ref from v3.0 to main.
173174
download-schemas:
175+
wget -O jsonschema/web-platform-dx_web-features/v3.data.schema.json \
176+
https://raw.githubusercontent.com/web-platform-dx/web-features/refs/heads/v3.0/schemas/data.schema.json
174177
wget -O jsonschema/web-platform-dx_web-features/defs.schema.json \
175178
https://raw.githubusercontent.com/web-platform-dx/web-features/refs/heads/main/schemas/data.schema.json
176179
wget -O jsonschema/mdn_browser-compat-data/browsers.schema.json \
@@ -185,7 +188,15 @@ jsonschema: clean-jsonschema
185188
--out $(JSONSCHEMA_OUT_DIR)/web_platform_dx__web_features/feature_data.go \
186189
--package web_platform_dx__web_features \
187190
--field-tags json
188-
cp web_platform_dx__web_features_extras.txt $(JSONSCHEMA_OUT_DIR)/web_platform_dx__web_features/extras.go
191+
192+
npx quicktype \
193+
--src jsonschema/web-platform-dx_web-features/v3.data.schema.json \
194+
--src-lang schema \
195+
--lang go \
196+
--top-level FeatureData \
197+
--out $(JSONSCHEMA_OUT_DIR)/web_platform_dx__web_features_v3/feature_data.go \
198+
--package web_platform_dx__web_features_v3 \
199+
--field-tags json
189200

190201
npx quicktype \
191202
--src jsonschema/mdn_browser-compat-data/browsers.schema.json \

‎jsonschema/web-platform-dx_web-features/v3.data.schema.json‎

Lines changed: 319 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,319 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema",
3+
"description": "The top-level web-features data package",
4+
"type": "object",
5+
"properties": {
6+
"browsers": {
7+
"description": "Browsers and browser release data",
8+
"type": "object",
9+
"properties": {
10+
"chrome": {
11+
"$ref": "#/definitions/BrowserData"
12+
},
13+
"chrome_android": {
14+
"$ref": "#/definitions/BrowserData"
15+
},
16+
"edge": {
17+
"$ref": "#/definitions/BrowserData"
18+
},
19+
"firefox": {
20+
"$ref": "#/definitions/BrowserData"
21+
},
22+
"firefox_android": {
23+
"$ref": "#/definitions/BrowserData"
24+
},
25+
"safari": {
26+
"$ref": "#/definitions/BrowserData"
27+
},
28+
"safari_ios": {
29+
"$ref": "#/definitions/BrowserData"
30+
}
31+
},
32+
"required": [
33+
"chrome",
34+
"chrome_android",
35+
"edge",
36+
"firefox",
37+
"firefox_android",
38+
"safari",
39+
"safari_ios"
40+
],
41+
"additionalProperties": false
42+
},
43+
"features": {
44+
"description": "Feature identifiers and data",
45+
"type": "object",
46+
"additionalProperties": {
47+
"oneOf": [
48+
{
49+
"$ref": "#/definitions/FeatureData"
50+
},
51+
{
52+
"$ref": "#/definitions/FeatureMovedData"
53+
},
54+
{
55+
"$ref": "#/definitions/FeatureSplitData"
56+
}
57+
],
58+
"$comment": "Use the `kind` property as a discriminator."
59+
}
60+
},
61+
"groups": {
62+
"description": "Group identifiers and data",
63+
"type": "object",
64+
"additionalProperties": {
65+
"$ref": "#/definitions/GroupData"
66+
}
67+
},
68+
"snapshots": {
69+
"description": "Snapshot identifiers and data",
70+
"type": "object",
71+
"additionalProperties": {
72+
"$ref": "#/definitions/SnapshotData"
73+
}
74+
}
75+
},
76+
"required": ["browsers", "features", "groups", "snapshots"],
77+
"additionalProperties": false,
78+
"definitions": {
79+
"Discouraged": {
80+
"type": "object",
81+
"properties": {
82+
"according_to": {
83+
"description": "Links to a formal discouragement notice, such as specification text, intent-to-unship, etc.",
84+
"$ref": "#/definitions/Strings"
85+
},
86+
"alternatives": {
87+
"description": "IDs for features that substitute some or all of this feature's utility",
88+
"$ref": "#/definitions/Strings"
89+
}
90+
},
91+
"required": ["according_to"],
92+
"additionalProperties": false
93+
},
94+
"BrowserData": {
95+
"description": "Browser information",
96+
"type": "object",
97+
"properties": {
98+
"name": {
99+
"description": "The name of the browser, as in \"Edge\" or \"Safari on iOS\"",
100+
"type": "string"
101+
},
102+
"releases": {
103+
"type": "array",
104+
"items": {
105+
"$ref": "#/definitions/Release"
106+
}
107+
}
108+
},
109+
"required": ["name", "releases"],
110+
"additionalProperties": false
111+
},
112+
"FeatureData": {
113+
"description": "A feature data entry",
114+
"type": "object",
115+
"properties": {
116+
"kind": {
117+
"const": "feature"
118+
},
119+
"name": {
120+
"description": "Short name",
121+
"type": "string"
122+
},
123+
"description": {
124+
"description": "Short description of the feature, as a plain text string",
125+
"type": "string"
126+
},
127+
"description_html": {
128+
"description": "Short description of the feature, as an HTML string",
129+
"type": "string"
130+
},
131+
"spec": {
132+
"description": "Specification URLs",
133+
"$ref": "#/definitions/Strings"
134+
},
135+
"group": {
136+
"description": "Group identifiers",
137+
"$ref": "#/definitions/Strings"
138+
},
139+
"snapshot": {
140+
"description": "Snapshot identifiers",
141+
"$ref": "#/definitions/Strings"
142+
},
143+
"caniuse": {
144+
"description": "caniuse.com identifiers",
145+
"$ref": "#/definitions/Strings"
146+
},
147+
"compat_features": {
148+
"description": "Sources of support data for this feature",
149+
"$ref": "#/definitions/Strings"
150+
},
151+
"status": {
152+
"description": "Whether a feature is considered a \"Baseline\" web platform feature and when it achieved that status",
153+
"$ref": "#/definitions/StatusHeadline"
154+
},
155+
"discouraged": {
156+
"description": "Whether developers are formally discouraged from using this feature",
157+
"$ref": "#/definitions/Discouraged"
158+
}
159+
},
160+
"required": [
161+
"kind",
162+
"name",
163+
"description",
164+
"description_html",
165+
"spec",
166+
"status"
167+
],
168+
"additionalProperties": false
169+
},
170+
"FeatureMovedData": {
171+
"description": "A feature has permanently moved to exactly one other ID",
172+
"type": "object",
173+
"properties": {
174+
"kind": {
175+
"const": "moved"
176+
},
177+
"redirect_target": {
178+
"description": "The new ID for this feature",
179+
"type": "string"
180+
}
181+
},
182+
"required": ["kind", "redirect_target"],
183+
"additionalProperties": false
184+
},
185+
"FeatureSplitData": {
186+
"description": "A feature has split into two or more other features",
187+
"type": "object",
188+
"properties": {
189+
"kind": {
190+
"const": "split"
191+
},
192+
"redirect_targets": {
193+
"description": "The new IDs for this feature",
194+
"$ref": "#/definitions/Strings"
195+
}
196+
},
197+
"required": ["kind", "redirect_targets"],
198+
"additionalProperties": false
199+
},
200+
"GroupData": {
201+
"type": "object",
202+
"properties": {
203+
"name": {
204+
"description": "Short name",
205+
"type": "string"
206+
},
207+
"parent": {
208+
"description": "Identifier of parent group",
209+
"type": "string"
210+
}
211+
},
212+
"required": ["name"],
213+
"additionalProperties": false
214+
},
215+
"Release": {
216+
"description": "Browser release information",
217+
"type": "object",
218+
"properties": {
219+
"version": {
220+
"description": "The version string, as in \"10\" or \"17.1\"",
221+
"type": "string"
222+
},
223+
"date": {
224+
"description": " The release date, as in \"2023-12-11\"",
225+
"type": "string"
226+
}
227+
},
228+
"required": ["version", "date"],
229+
"additionalProperties": false
230+
},
231+
"SnapshotData": {
232+
"type": "object",
233+
"properties": {
234+
"name": {
235+
"description": "Short name",
236+
"type": "string"
237+
},
238+
"spec": {
239+
"description": "Specification",
240+
"type": "string"
241+
}
242+
},
243+
"required": ["name", "spec"],
244+
"additionalProperties": false
245+
},
246+
"Status": {
247+
"type": "object",
248+
"properties": {
249+
"baseline": {
250+
"description": "Whether the feature is Baseline (low substatus), Baseline (high substatus), or not (false)",
251+
"enum": ["high", "low", false]
252+
},
253+
"baseline_high_date": {
254+
"description": "Date the feature achieved Baseline high status",
255+
"type": "string"
256+
},
257+
"baseline_low_date": {
258+
"description": "Date the feature achieved Baseline low status",
259+
"type": "string"
260+
},
261+
"support": {
262+
"description": "Browser versions that most-recently introduced the feature",
263+
"type": "object",
264+
"properties": {
265+
"chrome": {
266+
"type": "string"
267+
},
268+
"chrome_android": {
269+
"type": "string"
270+
},
271+
"edge": {
272+
"type": "string"
273+
},
274+
"firefox": {
275+
"type": "string"
276+
},
277+
"firefox_android": {
278+
"type": "string"
279+
},
280+
"safari": {
281+
"type": "string"
282+
},
283+
"safari_ios": {
284+
"type": "string"
285+
}
286+
},
287+
"additionalProperties": false
288+
}
289+
},
290+
"required": ["baseline", "support"]
291+
},
292+
"StatusHeadline": {
293+
"type": "object",
294+
"allOf": [{ "$ref": "#/definitions/Status" }],
295+
"properties": {
296+
"baseline": {},
297+
"baseline_high_date": {},
298+
"baseline_low_date": {},
299+
"support": {},
300+
"by_compat_key": {
301+
"description": "Statuses for each key in the feature's compat_features list, if applicable. Not available to the npm release of web-features.",
302+
"type": "object",
303+
"additionalProperties": {
304+
"$ref": "#/definitions/Status"
305+
}
306+
}
307+
},
308+
"required": ["baseline", "support"],
309+
"additionalProperties": false
310+
},
311+
"Strings": {
312+
"type": "array",
313+
"items": {
314+
"type": "string"
315+
},
316+
"minItems": 1
317+
}
318+
}
319+
}

‎lib/gcpspanner/spanneradapters/chromium_historgram_enum_consumer.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ import (
2121
"strings"
2222

2323
"github.com/GoogleChrome/webstatus.dev/lib/gcpspanner"
24-
"github.com/GoogleChrome/webstatus.dev/lib/gen/jsonschema/web_platform_dx__web_features"
2524
"github.com/GoogleChrome/webstatus.dev/lib/metricdatatypes"
25+
"github.com/GoogleChrome/webstatus.dev/lib/webdxfeaturetypes"
2626
"golang.org/x/text/cases"
2727
"golang.org/x/text/language"
2828
)
@@ -53,7 +53,7 @@ type ChromiumHistogramEnumsClient interface {
5353
const logMissingFeatureIDMetricMsg = "unable to find feature ID. skipping mapping"
5454

5555
func (c *ChromiumHistogramEnumConsumer) GetAllMovedWebFeatures(
56-
ctx context.Context) (map[string]web_platform_dx__web_features.FeatureMovedData, error) {
56+
ctx context.Context) (map[string]webdxfeaturetypes.FeatureMovedData, error) {
5757
movedFeatures, err := c.client.GetAllMovedWebFeatures(ctx)
5858
if err != nil {
5959
return nil, err
@@ -165,7 +165,7 @@ func migrateMovedFeaturesForChromiumHistograms(
165165
ctx context.Context,
166166
histogramsToEnumMap map[metricdatatypes.HistogramName]map[int64]*string,
167167
histogramsToAllFeatureKeySet map[metricdatatypes.HistogramName]map[string]metricdatatypes.HistogramEnumValue,
168-
movedFeatures map[string]web_platform_dx__web_features.FeatureMovedData,
168+
movedFeatures map[string]webdxfeaturetypes.FeatureMovedData,
169169
) error {
170170
for histogram, allFeaturesKeySet := range histogramsToAllFeatureKeySet {
171171
logger := slog.With("histogram", histogram)

0 commit comments

Comments
 (0)