Skip to content

Commit ff2f96a

Browse files
NiamsycdhawkestevenMevansleonardogavaudanshortcuts
authored
feat(specs): add abtests v3 (#4394)
Co-authored-by: Christopher Hawke <[email protected]> Co-authored-by: Steven Evans <[email protected]> Co-authored-by: Leonardo Gavaudan <[email protected]> Co-authored-by: Clément Vannicatte <[email protected]>
1 parent 7226589 commit ff2f96a

File tree

16 files changed

+1090
-1
lines changed

16 files changed

+1090
-1
lines changed

‎scripts/common.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ export const LANGUAGES = [...new Set(Object.values(GENERATORS).map((gen) => gen.
7373

7474
// `crawler` is manually added so we can still bundled and validate the specs
7575
// the entry can be removed once at least one client is generated
76-
export const CLIENTS = [...new Set(Object.values(GENERATORS).map((gen) => gen.client)), 'crawler'];
76+
export const CLIENTS = [...new Set(Object.values(GENERATORS).map((gen) => gen.client)), 'crawler', 'abtesting-v3'];
7777

7878
export async function run(command: string, { errorMessage, cwd, language }: RunOptions = {}): Promise<string> {
7979
const realCwd = path.resolve(ROOT_DIR, cwd ?? '.');

‎specs/abtesting-v3/common/parameters.yml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
# path
2+
ID:
3+
in: path
4+
name: id
5+
description: Unique A/B test identifier.
6+
required: true
7+
schema:
8+
$ref: '#/abTestID'
9+
10+
# misc
11+
index:
12+
type: string
13+
description: Index name of the A/B test variant (case-sensitive).
14+
example: 'delcourt_production'
15+
16+
abTestID:
17+
type: integer
18+
description: Unique A/B test identifier.
19+
example: 224
20+
21+
abTestScheduleID:
22+
type: integer
23+
description: Unique scheduled A/B test identifier.
24+
example: 224
25+
26+
endAt:
27+
type: string
28+
description: End date and time of the A/B test, in RFC 3339 format.
29+
example: 2023-06-17T00:00:00Z
30+
31+
createdAt:
32+
type: string
33+
description: Date and time when the A/B test was created, in RFC 3339 format.
34+
example: 2023-06-15T15:06:04.249906Z
35+
36+
updatedAt:
37+
type: string
38+
description: Date and time when the A/B test was last updated, in RFC 3339 format.
39+
example: 2023-06-15T15:06:44.400601Z
40+
41+
scheduledAt:
42+
type: string
43+
description: Date and time when the A/B test is scheduled to start, in RFC 3339 format.
44+
example: 2023-06-15T15:06:44.400601Z
45+
46+
name:
47+
type: string
48+
description: A/B test name.
49+
example: Custom ranking sales rank test
50+
51+
description:
52+
type: string
53+
description: Description for this variant.
54+
example: Current production index
55+
56+
trafficPercentage:
57+
type: integer
58+
description: Percentage of search requests each variant receives.
59+
minimum: 1
60+
maximum: 99
61+
example: 60
62+
63+
currencies:
64+
type: object
65+
description: A/B test currencies.
66+
example:
67+
USD:
68+
currency: USD
69+
revenue: 120.0
70+
mean: 53.7
71+
standardDeviation: 12.3
72+
EUR:
73+
currency: EUR
74+
revenue: 100.0
75+
mean: 43.7
76+
standardDeviation: 10.3
77+
additionalProperties:
78+
$ref: '#/currency'
79+
x-additionalPropertiesName: currency code
80+
81+
currency:
82+
type: object
83+
properties:
84+
currency:
85+
type: string
86+
description: Currency code.
87+
example: 'USD'
88+
revenue:
89+
type: number
90+
format: double
91+
description: Revenue for this currency.
92+
example: 120.0
93+
mean:
94+
type: number
95+
format: double
96+
description: Mean for this currency.
97+
example: 53.7
98+
standardDeviation:
99+
type: number
100+
format: double
101+
description: Standard deviation for this currency.
102+
example: 12.3
103+
104+
filterEffects:
105+
type: object
106+
description: A/B test filter effects resulting from configuration settings.
107+
properties:
108+
outliers:
109+
title: outliersFilter
110+
type: object
111+
description: Outliers removed from the A/B test as a result of configuration settings.
112+
example:
113+
usersCount: 1
114+
trackedSearchesCount: 237
115+
properties:
116+
usersCount:
117+
type: integer
118+
description: Number of users removed from the A/B test.
119+
example: 1
120+
trackedSearchesCount:
121+
type: integer
122+
description: Number of tracked searches removed from the A/B test.
123+
example: 237
124+
emptySearch:
125+
title: emptySearchFilter
126+
type: object
127+
description: Empty searches removed from the A/B test as a result of configuration settings.
128+
example:
129+
usersCount: 1
130+
trackedSearchesCount: 237
131+
properties:
132+
usersCount:
133+
type: integer
134+
description: Number of users removed from the A/B test.
135+
example: 1
136+
trackedSearchesCount:
137+
type: integer
138+
description: Number of tracked searches removed from the A/B test.
139+
example: 237
140+
141+
metric:
142+
type: object
143+
description: Defines a metric to be retrieved during an A/B test.
144+
properties:
145+
name:
146+
type: string
147+
description: Name of the metric.
148+
dimension:
149+
type: string
150+
description: Dimension of the metric, for example, in case of a revenue metric it could be USD, EUR...
151+
required:
152+
- name
153+
example:
154+
- name: revenue
155+
dimension: USD
156+
- name: conversionRate
157+
- name: clickThroughRate
158+
- name: trackedSearchCount

‎specs/abtesting-v3/common/schemas/ABTest.yml‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
ABTests:
2+
oneOf:
3+
- type: array
4+
description: A/B tests.
5+
items:
6+
$ref: '#/ABTest'
7+
- type: 'null'
8+
9+
ABTest:
10+
type: object
11+
additionalProperties: false
12+
properties:
13+
abTestID:
14+
$ref: '../parameters.yml#/abTestID'
15+
updatedAt:
16+
$ref: '../parameters.yml#/updatedAt'
17+
createdAt:
18+
$ref: '../parameters.yml#/createdAt'
19+
endAt:
20+
$ref: '../parameters.yml#/endAt'
21+
name:
22+
$ref: '../parameters.yml#/name'
23+
status:
24+
$ref: '#/Status'
25+
variants:
26+
$ref: 'Variant.yml#/variants'
27+
configuration:
28+
$ref: '#/ABTestConfiguration'
29+
migratedAbTestID:
30+
$ref: '#/MigratedABTestId'
31+
required:
32+
- status
33+
- name
34+
- createdAt
35+
- endAt
36+
- updatedAt
37+
- abTestID
38+
- variants
39+
40+
Status:
41+
type: string
42+
description: |
43+
A/B test status.
44+
45+
- `active`. The A/B test is live and search traffic is split between the two variants.
46+
- `stopped`. You stopped the A/B test. The A/B test data is still available for analysis.
47+
- `expired`. The A/B test was automatically stopped after reaching its end date.
48+
- `failed`. Creating the A/B test failed.
49+
example: active
50+
enum:
51+
- active
52+
- stopped
53+
- expired
54+
- failed
55+
56+
ABTestConfiguration:
57+
title: configuration
58+
type: object
59+
description: A/B test configuration.
60+
properties:
61+
minimumDetectableEffect:
62+
$ref: '#/MinimumDetectableEffect'
63+
filters:
64+
$ref: '#/MetricsFilters'
65+
errorCorrection:
66+
$ref: '#/ErrorCorrectionType'
67+
68+
MinimumDetectableEffect:
69+
type: object
70+
description: Configuration for the smallest difference between test variants you want to detect.
71+
properties:
72+
size:
73+
type: number
74+
format: double
75+
minimum: 0
76+
maximum: 1
77+
description: |
78+
Smallest difference in an observable metric between variants.
79+
For example, to detect a 10% difference between variants, set this value to 0.1.
80+
metric:
81+
$ref: '#/EffectMetric'
82+
required:
83+
- size
84+
- metric
85+
86+
EffectMetric:
87+
type: string
88+
description: Metric for which you want to detect the smallest relative difference.
89+
enum:
90+
- addToCartRate
91+
- clickThroughRate
92+
- conversionRate
93+
- purchaseRate
94+
- noResultsRate
95+
96+
MigratedABTestId:
97+
type: integer
98+
description: Unique migrated A/B test identifier.
99+
example: 224
100+
101+
MetricsFilter:
102+
type: object
103+
additionalProperties: false
104+
description: |
105+
Boolean filter applied to the A/B test population. Each filter targets a boolean metric
106+
and decides whether to include (true) or exclude (false) matching records.
107+
properties:
108+
domain:
109+
type: string
110+
description: Metric domain (for example `abtesting`, `personalization`).
111+
example: abtesting
112+
name:
113+
type: string
114+
description: Public metric name.
115+
example: isOutlier
116+
trackEffects:
117+
type: boolean
118+
description: Whether the experiment should record the effects of this filter.
119+
includes:
120+
type: boolean
121+
description: If true, keep items that match the filter; if false, exclude them.
122+
required:
123+
- domain
124+
- name
125+
126+
MetricsFilters:
127+
type: array
128+
description: List of metric filters applied to the test population.
129+
items:
130+
$ref: '#/MetricsFilter'
131+
132+
ErrorCorrectionType:
133+
type: string
134+
description: Multiple-testing correction method applied when evaluating metric significance.
135+
enum:
136+
- bonferroni
137+
- benjamini-hochberg

‎specs/abtesting-v3/common/schemas/ABTestResponse.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
ABTestResponse:
2+
type: object
3+
additionalProperties: false
4+
properties:
5+
index:
6+
$ref: '../parameters.yml#/index'
7+
abTestID:
8+
$ref: '../parameters.yml#/abTestID'
9+
taskID:
10+
$ref: '../../../common/responses/common.yml#/taskID'
11+
required:
12+
- abTestID
13+
- index
14+
- taskID

‎specs/abtesting-v3/common/schemas/AddABTestsVariant.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
AddABTestsVariant:
2+
oneOf:
3+
- $ref: '#/abTestsVariant'
4+
- $ref: '#/abTestsVariantSearchParams'
5+
6+
abTestsVariantSearchParams:
7+
allOf:
8+
- $ref: '#/abTestsVariant'
9+
- $ref: '#/customSearchParams'
10+
11+
abTestsVariant:
12+
type: object
13+
additionalProperties: false
14+
properties:
15+
index:
16+
$ref: '../parameters.yml#/index'
17+
trafficPercentage:
18+
$ref: '../parameters.yml#/trafficPercentage'
19+
description:
20+
$ref: '../parameters.yml#/description'
21+
required:
22+
- index
23+
- trafficPercentage
24+
25+
customSearchParams:
26+
type: object
27+
description: |
28+
Search parameters to add to the test variant.
29+
Only use this parameter if the two variants use the same index.
30+
example: {'typoTolerance': false, 'synonyms': false}
31+
additionalProperties: false
32+
properties:
33+
customSearchParameters:
34+
type: object
35+
required:
36+
- customSearchParameters
37+
x-discriminator-fields:
38+
- customSearchParameters

‎specs/abtesting-v3/common/schemas/EstimateABTestResponse.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
EstimateABTestResponse:
2+
type: object
3+
properties:
4+
durationDays:
5+
type: integer
6+
format: int64
7+
description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic.
8+
example: 21
9+
sampleSizes:
10+
type: array
11+
description: |
12+
Sample size estimates for each variant. The first element is the control variant.
13+
Each element is the estimated number of searches required to achieve the desired statistical significance.
14+
items:
15+
type: integer
16+
format: int64
17+
description: Number of tracked searches needed to be able to detect the configured effect for the control variant.
18+
example: 23415

‎specs/abtesting-v3/common/schemas/ScheduleABTestResponse.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
ScheduleABTestResponse:
2+
type: object
3+
additionalProperties: false
4+
properties:
5+
abTestScheduleID:
6+
$ref: '../parameters.yml#/abTestScheduleID'
7+
required:
8+
- abTestScheduleID

0 commit comments

Comments
 (0)