From 90054e074afd893a54d3e0b39e98803c55361161 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Fri, 24 Jan 2025 12:10:37 +0100 Subject: [PATCH 01/21] docs(abtests) v3 api --- specs/abtesting-v3/common/parameters.yml | 158 ++++++++++++++++ specs/abtesting-v3/common/schemas/ABTest.yml | 170 ++++++++++++++++++ .../common/schemas/ABTestResponse.yml | 14 ++ .../common/schemas/AddABTestsVariant.yml | 38 ++++ .../common/schemas/EstimateABTestResponse.yml | 18 ++ .../common/schemas/ScheduleABTestResponse.yml | 8 + specs/abtesting-v3/common/schemas/Variant.yml | 94 ++++++++++ specs/abtesting-v3/paths/abtest.yml | 65 +++++++ specs/abtesting-v3/paths/abtests.yml | 137 ++++++++++++++ specs/abtesting-v3/paths/estimate.yml | 62 +++++++ specs/abtesting-v3/paths/scheduleABTest.yml | 58 ++++++ specs/abtesting-v3/paths/stopABTest.yml | 35 ++++ specs/abtesting-v3/spec.yml | 109 +++++++++++ 13 files changed, 966 insertions(+) create mode 100644 specs/abtesting-v3/common/parameters.yml create mode 100644 specs/abtesting-v3/common/schemas/ABTest.yml create mode 100644 specs/abtesting-v3/common/schemas/ABTestResponse.yml create mode 100644 specs/abtesting-v3/common/schemas/AddABTestsVariant.yml create mode 100644 specs/abtesting-v3/common/schemas/EstimateABTestResponse.yml create mode 100644 specs/abtesting-v3/common/schemas/ScheduleABTestResponse.yml create mode 100644 specs/abtesting-v3/common/schemas/Variant.yml create mode 100644 specs/abtesting-v3/paths/abtest.yml create mode 100644 specs/abtesting-v3/paths/abtests.yml create mode 100644 specs/abtesting-v3/paths/estimate.yml create mode 100644 specs/abtesting-v3/paths/scheduleABTest.yml create mode 100644 specs/abtesting-v3/paths/stopABTest.yml create mode 100644 specs/abtesting-v3/spec.yml diff --git a/specs/abtesting-v3/common/parameters.yml b/specs/abtesting-v3/common/parameters.yml new file mode 100644 index 00000000000..1be5a7b9764 --- /dev/null +++ b/specs/abtesting-v3/common/parameters.yml @@ -0,0 +1,158 @@ +# path +ID: + in: path + name: id + description: Unique A/B test identifier. + required: true + schema: + $ref: "#/abTestID" + +# misc +index: + type: string + description: Index name of the A/B test variant (case-sensitive). + example: "delcourt_production" + +abTestID: + type: integer + description: Unique A/B test identifier. + example: 224 + +abTestScheduleID: + type: integer + description: Unique scheduled A/B test identifier. + example: 224 + +endAt: + type: string + description: End date and time of the A/B test, in RFC 3339 format. + example: 2023-06-17T00:00:00Z + +createdAt: + type: string + description: Date and time when the A/B test was created, in RFC 3339 format. + example: 2023-06-15T15:06:04.249906Z + +updatedAt: + type: string + description: Date and time when the A/B test was last updated, in RFC 3339 format. + example: 2023-06-15T15:06:44.400601Z + +scheduledAt: + type: string + description: Date and time when the A/B test is scheduled to start, in RFC 3339 format. + example: 2023-06-15T15:06:44.400601Z + +name: + type: string + description: A/B test name. + example: Custom ranking sales rank test + +description: + type: string + description: Description for this variant. + example: Current production index + +trafficPercentage: + type: integer + description: Percentage of search requests each variant receives. + minimum: 0 + maximum: 100 + example: 60 + +currencies: + type: object + description: A/B test currencies. + example: + USD: + currency: USD + revenue: 120.0 + mean: 53.7 + standardDeviation: 12.3 + EUR: + currency: EUR + revenue: 100.0 + mean: 43.7 + standardDeviation: 10.3 + additionalProperties: + $ref: "#/currency" + x-additionalPropertiesName: currency code + +currency: + type: object + properties: + currency: + type: string + description: Currency code. + example: "USD" + revenue: + type: number + format: double + description: Revenue for this currency. + example: 120.0 + mean: + type: number + format: double + description: Mean for this currency. + example: 53.7 + standardDeviation: + type: number + format: double + description: Standard deviation for this currency. + example: 12.3 + +filterEffects: + type: object + description: A/B test filter effects resulting from configuration settings. + properties: + outliers: + title: outliersFilter + type: object + description: Outliers removed from the A/B test as a result of configuration settings. + example: + usersCount: 1 + trackedSearchesCount: 237 + properties: + usersCount: + type: integer + description: Number of users removed from the A/B test. + example: 1 + trackedSearchesCount: + type: integer + description: Number of tracked searches removed from the A/B test. + example: 237 + emptySearch: + title: emptySearchFilter + type: object + description: Empty searches removed from the A/B test as a result of configuration settings. + example: + usersCount: 1 + trackedSearchesCount: 237 + properties: + usersCount: + type: integer + description: Number of users removed from the A/B test. + example: 1 + trackedSearchesCount: + type: integer + description: Number of tracked searches removed from the A/B test. + example: 237 + +metric: + type: object + description: Defines a metric to be retrieved during an A/B test. + properties: + name: + type: string + description: Name of the metric. + dimension: + type: string + description: Dimension of the metric, for example, in case of a revenue metric it could be USD, EUR... + required: + - name + example: + - name: revenue + dimension: USD + - name: conversionRate + - name: clickThroughRate + - name: trackedSearchCount diff --git a/specs/abtesting-v3/common/schemas/ABTest.yml b/specs/abtesting-v3/common/schemas/ABTest.yml new file mode 100644 index 00000000000..a769d098a06 --- /dev/null +++ b/specs/abtesting-v3/common/schemas/ABTest.yml @@ -0,0 +1,170 @@ +ABTests: + oneOf: + - type: array + description: A/B tests. + items: + $ref: '#/ABTest' + - type: 'null' + description: No A/B tests are configured for this application. + +ABTest: + type: object + additionalProperties: false + properties: + abTestID: + $ref: '../parameters.yml#/abTestID' + clickSignificance: + description: | + A/B test significance calculated from click events. + + Values of 0.95 or higher can be considered significant, + that is, the difference between A and B variants is _not_ due to random variations. + Lower values have a. + oneOf: + - type: number + format: double + example: 1 + - type: 'null' + conversionSignificance: + description: | + A/B test significance calculated from conversion events. + + Values of 0.95 or higher can be considered significant, + that is, the difference between A and B variants is _not_ due to random variations. + oneOf: + - type: number + format: double + example: 1 + - type: 'null' + addToCartSignificance: + description: | + A/B test significance calculated from add-to-cart events. + + Values of 0.95 or higher can be considered significant, + that is, the difference between A and B variants is _not_ due to random variations. + oneOf: + - type: number + format: double + example: 1 + - type: 'null' + purchaseSignificance: + description: | + A/B test significance calculated from purchase events. + + Values of 0.95 or higher can be considered significant, + that is, the difference between A and B variants is _not_ due to random variations. + oneOf: + - type: number + format: double + example: 1 + - type: 'null' + revenueSignificance: + description: | + A/B test significance calculated from revenue data. + + Values of 0.95 or higher can be considered significant, + that is, the difference between A and B variants is _not_ due to random variations. + oneOf: + - type: object + additionalProperties: + type: number + format: double + x-additionalPropertiesName: currency code + example: + USD: 1 + EUR: 0.87 + - type: 'null' + updatedAt: + $ref: '../parameters.yml#/updatedAt' + createdAt: + $ref: '../parameters.yml#/createdAt' + endAt: + $ref: '../parameters.yml#/endAt' + name: + $ref: '../parameters.yml#/name' + status: + $ref: '#/Status' + variants: + $ref: 'Variant.yml#/variants' + configuration: + $ref: '#/ABTestConfiguration' + required: + - status + - name + - createdAt + - endAt + - updatedAt + - abTestID + - variants + +Status: + type: string + description: | + A/B test status. + + - `active`. The A/B test is live and search traffic is split between the two variants. + - `stopped`. You stopped the A/B test. The A/B test data is still available for analysis. + - `expired`. The A/B test was automatically stopped after reaching its end date. + - `failed`. Creating the A/B test failed. + example: active + enum: + - active + - stopped + - expired + - failed + +ABTestConfiguration: + title: configuration + type: object + description: A/B test configuration. + properties: + outliers: + $ref: '#/Outliers' + emptySearch: + $ref: '#/EmptySearch' + minimumDetectableEffect: + $ref: '#/MinimumDetectableEffect' + +Outliers: + type: object + description: Configuration for handling outliers. + properties: + exclude: + type: boolean + description: Whether to exclude outliers when calculating A/B test results. + default: true + +EmptySearch: + type: object + description: Configuration for handling empty searches. + properties: + exclude: + type: boolean + description: Whether to exclude empty searches when calculating A/B test results. + +MinimumDetectableEffect: + type: object + description: Configuration for the smallest difference between test variants you want to detect. + properties: + size: + type: number + format: double + minimum: 0 + maximum: 1 + description: | + Smallest difference in an observable metric between variants. + For example, to detect a 10% difference between variants, set this value to 0.1. + metric: + $ref: '#/EffectMetric' + required: + - size + - metric + +EffectMetric: + type: string + description: Metric for which you want to detect the smallest relative difference. + enum: + - addToCartRate + - clickThroughRate + - conversionRate + - purchaseRate diff --git a/specs/abtesting-v3/common/schemas/ABTestResponse.yml b/specs/abtesting-v3/common/schemas/ABTestResponse.yml new file mode 100644 index 00000000000..ed3f5e6d901 --- /dev/null +++ b/specs/abtesting-v3/common/schemas/ABTestResponse.yml @@ -0,0 +1,14 @@ +ABTestResponse: + type: object + additionalProperties: false + properties: + index: + $ref: '../parameters.yml#/index' + abTestID: + $ref: '../parameters.yml#/abTestID' + taskID: + $ref: '../../../common/responses/common.yml#/taskID' + required: + - abTestID + - index + - taskID diff --git a/specs/abtesting-v3/common/schemas/AddABTestsVariant.yml b/specs/abtesting-v3/common/schemas/AddABTestsVariant.yml new file mode 100644 index 00000000000..fc14f380240 --- /dev/null +++ b/specs/abtesting-v3/common/schemas/AddABTestsVariant.yml @@ -0,0 +1,38 @@ +AddABTestsVariant: + oneOf: + - $ref: '#/abTestsVariant' + - $ref: '#/abTestsVariantSearchParams' + +abTestsVariantSearchParams: + allOf: + - $ref: '#/abTestsVariant' + - $ref: '#/customSearchParams' + +abTestsVariant: + type: object + additionalProperties: false + properties: + index: + $ref: '../parameters.yml#/index' + trafficPercentage: + $ref: '../parameters.yml#/trafficPercentage' + description: + $ref: '../parameters.yml#/description' + required: + - index + - trafficPercentage + +customSearchParams: + type: object + description: | + Search parameters to add to the test variant. + Only use this parameter if the two variants use the same index. + example: {'typoTolerance': false, 'synonyms': false} + additionalProperties: false + properties: + customSearchParameters: + type: object + required: + - customSearchParameters + x-discriminator-fields: + - customSearchParameters diff --git a/specs/abtesting-v3/common/schemas/EstimateABTestResponse.yml b/specs/abtesting-v3/common/schemas/EstimateABTestResponse.yml new file mode 100644 index 00000000000..6d491f24cc6 --- /dev/null +++ b/specs/abtesting-v3/common/schemas/EstimateABTestResponse.yml @@ -0,0 +1,18 @@ +EstimateABTestResponse: + type: object + properties: + durationDays: + type: integer + format: int64 + description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic. + example: 21 + sampleSizes: + type: array + description: | + Sample size estimates for each variant. The first element is the control variant. + Each element is the estimated number of searches required to achieve the desired statistical significance. + items: + type: integer + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the control variant. + example: 23415 diff --git a/specs/abtesting-v3/common/schemas/ScheduleABTestResponse.yml b/specs/abtesting-v3/common/schemas/ScheduleABTestResponse.yml new file mode 100644 index 00000000000..a1278e7b77f --- /dev/null +++ b/specs/abtesting-v3/common/schemas/ScheduleABTestResponse.yml @@ -0,0 +1,8 @@ +ScheduleABTestResponse: + type: object + additionalProperties: false + properties: + abTestScheduleID: + $ref: '../parameters.yml#/abTestScheduleID' + required: + - abTestScheduleID diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml new file mode 100644 index 00000000000..b4c07a412dc --- /dev/null +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -0,0 +1,94 @@ +variants: + type: array + description: | + A/B test variants. + + The first variant is your _control_ index, typically your production index. + All of the additionnal variants are indexes with changed settings that you want to test against the control. + items: + $ref: "#/variant" + +variant: + type: object + additionalProperties: false + properties: + description: + $ref: "../parameters.yml#/description" + estimatedSampleSize: + type: integer + description: | + Estimated number of searches required to achieve the desired statistical significance. + + The A/B test configuration must include a `mininmumDetectableEffect` setting for this number to be included in the response. + example: 0 + filterEffects: + $ref: "../parameters.yml#/filterEffects" + index: + $ref: "../parameters.yml#/index" + trafficPercentage: + $ref: "../parameters.yml#/trafficPercentage" + metrics: + $ref: "#/metrics" + required: + - index + - description + - trafficPercentage + - metrics + +metrics: + type: array + description: All ABTest metrics that were defined during test creation + items: + $ref: "#/metric" + +metric: + type: object + properties: + name: + type: string + updatedAt: + type: string + description: Date and time when the metric was last updated, in RFC 3339 format. + value: + type: number + format: double + pvalue: + type: number + format: double + dimension: + type: string + description: Dimension defined during test creation + winsorizedValue: + type: number + format: double + description: | + Only present in case the metric is a revenue. + It is the normalized value of it. + It is tied to a per revenue-currency pair contrary to other + global filter effects (such as outliers and empty search count). + required: + - name + - updatedAt + - value + - pvalue + example: + - name: addToCartCount + updatedAt: 2025-06-15T15:06:44.400601Z + value: 5 + pvalue: 0.01 + - name: clickThroughRate + updatedAt: 2025-05-15T17:52:15.644906Z + value: 0.20869847452125934 + pvalue: 0.004 + - name: revenue + dimension: USD + updatedAt": 2025-05-15T17:52:15.644906Z + value: 1200.50 + pValue: 0.04 + winsorizedValue: 1123.45 + - name: revenue + dimension: EUR + updatedAt: 2025-05-15T17:52:15.644906Z + value: 999.66 + pValue: 0.04 + winsorizedValue: 888.44 diff --git a/specs/abtesting-v3/paths/abtest.yml b/specs/abtesting-v3/paths/abtest.yml new file mode 100644 index 00000000000..1e2bf896112 --- /dev/null +++ b/specs/abtesting-v3/paths/abtest.yml @@ -0,0 +1,65 @@ +get: + tags: + - abtest + operationId: getABTest + x-acl: + - analytics + summary: Retrieve A/B test details + description: Retrieves the details for an A/B test by its ID. + parameters: + - $ref: '../common/parameters.yml#/ID' + responses: + '200': + description: OK + headers: + x-ratelimit-limit: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' + x-ratelimit-remaining: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' + x-ratelimit-reset: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' + content: + application/json: + schema: + $ref: '../common/schemas/ABTest.yml#/ABTest' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' + +delete: + tags: + - abtest + operationId: deleteABTest + x-acl: + - editSettings + summary: Delete an A/B test + description: Deletes an A/B test by its ID. + parameters: + - $ref: '../common/parameters.yml#/ID' + responses: + '200': + description: OK + headers: + x-ratelimit-limit: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' + x-ratelimit-remaining: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' + x-ratelimit-reset: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' + content: + application/json: + schema: + $ref: '../common/schemas/ABTestResponse.yml#/ABTestResponse' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' diff --git a/specs/abtesting-v3/paths/abtests.yml b/specs/abtesting-v3/paths/abtests.yml new file mode 100644 index 00000000000..c10258234a2 --- /dev/null +++ b/specs/abtesting-v3/paths/abtests.yml @@ -0,0 +1,137 @@ +post: + tags: + - abtest + operationId: addABTests + x-acl: + - editSettings + summary: Create an A/B test + description: Creates a new A/B test. + requestBody: + required: true + content: + application/json: + schema: + title: addABTestsRequest + type: object + additionalProperties: false + properties: + name: + $ref: "../common/parameters.yml#/name" + variants: + type: array + description: A/B test variants. + minItems: 2 + items: + $ref: "../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant" + metrics: + type: array + description: A/B test metrics to retrieve. + items: + $ref: "../common/parameters.yml#/metric" + configuration: + $ref: "../common/schemas/ABTest.yml#/ABTestConfiguration" + endAt: + $ref: "../common/parameters.yml#/endAt" + required: + - name + - variants + - metrics + - endAt + responses: + "200": + description: OK + headers: + x-ratelimit-limit: + $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-limit" + x-ratelimit-remaining: + $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-remaining" + x-ratelimit-reset: + $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-reset" + content: + application/json: + schema: + $ref: "../common/schemas/ABTestResponse.yml#/ABTestResponse" + "400": + $ref: "../../common/responses/BadRequest.yml" + "402": + $ref: "../../common/responses/FeatureNotEnabled.yml" + "403": + $ref: "../../common/responses/MethodNotAllowed.yml" + "404": + $ref: "../../common/responses/IndexNotFound.yml" + +get: + tags: + - abtest + operationId: listABTests + x-acl: + - analytics + summary: List all A/B tests + description: Lists all A/B tests you configured for this application. + parameters: + - name: offset + in: query + description: Position of the first item to return. + required: false + schema: + type: integer + default: 0 + minimum: 0 + - name: limit + in: query + description: Number of items to return. + required: false + schema: + type: integer + default: 10 + - name: indexPrefix + in: query + description: Index name prefix. Only A/B tests for indices starting with this string are included in the response. + example: "dev_" + schema: + type: string + - name: indexSuffix + in: query + description: Index name suffix. Only A/B tests for indices ending with this string are included in the response. + example: "_development" + schema: + type: string + responses: + "200": + description: OK + headers: + x-ratelimit-limit: + $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-limit" + x-ratelimit-remaining: + $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-remaining" + x-ratelimit-reset: + $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-reset" + content: + application/json: + schema: + title: listABTestsResponse + type: object + additionalProperties: false + properties: + abtests: + $ref: "../common/schemas/ABTest.yml#/ABTests" + count: + type: integer + description: Number of A/B tests. + example: 10 + total: + type: integer + description: Number of retrievable A/B tests. + example: 12 + required: + - abtests + - count + - total + "400": + $ref: "../../common/responses/BadRequest.yml" + "402": + $ref: "../../common/responses/FeatureNotEnabled.yml" + "403": + $ref: "../../common/responses/MethodNotAllowed.yml" + "404": + $ref: "../../common/responses/IndexNotFound.yml" diff --git a/specs/abtesting-v3/paths/estimate.yml b/specs/abtesting-v3/paths/estimate.yml new file mode 100644 index 00000000000..fa29a90a080 --- /dev/null +++ b/specs/abtesting-v3/paths/estimate.yml @@ -0,0 +1,62 @@ +post: + tags: + - abtest + operationId: estimateABTest + x-acl: + - analytics + summary: Estimate the sample size and duration of an A/B test + description: Given the traffic percentage and the expected effect size, this endpoint estimates the sample size and duration of an A/B test based on historical traffic. + requestBody: + required: true + content: + application/json: + schema: + title: estimateABTestRequest + type: object + additionalProperties: false + properties: + configuration: + title: estimateConfiguration + type: object + description: A/B test configuration for estimating the sample size and duration using minimum detectable effect. + properties: + outliers: + $ref: '../common/schemas/ABTest.yml#/Outliers' + emptySearch: + $ref: '../common/schemas/ABTest.yml#/EmptySearch' + minimumDetectableEffect: + $ref: '../common/schemas/ABTest.yml#/MinimumDetectableEffect' + required: + - minimumDetectableEffect + variants: + type: array + description: A/B test variants. + minItems: 2 + maxItems: 2 + items: + $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant' + required: + - configuration + - variants + responses: + '200': + description: OK + headers: + x-ratelimit-limit: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' + x-ratelimit-remaining: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' + x-ratelimit-reset: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' + content: + application/json: + schema: + $ref: '../common/schemas/EstimateABTestResponse.yml#/EstimateABTestResponse' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' diff --git a/specs/abtesting-v3/paths/scheduleABTest.yml b/specs/abtesting-v3/paths/scheduleABTest.yml new file mode 100644 index 00000000000..3939d4314c4 --- /dev/null +++ b/specs/abtesting-v3/paths/scheduleABTest.yml @@ -0,0 +1,58 @@ +post: + tags: + - abtest + operationId: scheduleABTest + x-acl: + - editSettings + summary: Schedule an A/B test + description: | + Schedule an A/B test to be started at a later time. + requestBody: + required: true + content: + application/json: + schema: + title: scheduleABTestsRequest + type: object + additionalProperties: false + properties: + name: + $ref: '../common/parameters.yml#/name' + variants: + type: array + description: A/B test variants. + minItems: 2 + maxItems: 2 + items: + $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant' + scheduledAt: + $ref: '../common/parameters.yml#/scheduledAt' + endAt: + $ref: '../common/parameters.yml#/endAt' + required: + - name + - variants + - scheduledAt + - endAt + responses: + '200': + description: OK + headers: + x-ratelimit-limit: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' + x-ratelimit-remaining: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' + x-ratelimit-reset: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' + content: + application/json: + schema: + $ref: '../common/schemas/ScheduleABTestResponse.yml#/ScheduleABTestResponse' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' diff --git a/specs/abtesting-v3/paths/stopABTest.yml b/specs/abtesting-v3/paths/stopABTest.yml new file mode 100644 index 00000000000..1542f053397 --- /dev/null +++ b/specs/abtesting-v3/paths/stopABTest.yml @@ -0,0 +1,35 @@ +post: + tags: + - abtest + operationId: stopABTest + x-acl: + - editSettings + summary: Stop an A/B test + description: | + Stops an A/B test by its ID. + + You can't restart stopped A/B tests. + parameters: + - $ref: '../common/parameters.yml#/ID' + responses: + '200': + description: OK + headers: + x-ratelimit-limit: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' + x-ratelimit-remaining: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' + x-ratelimit-reset: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' + content: + application/json: + schema: + $ref: '../common/schemas/ABTestResponse.yml#/ABTestResponse' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' diff --git a/specs/abtesting-v3/spec.yml b/specs/abtesting-v3/spec.yml new file mode 100644 index 00000000000..31a92a8bff2 --- /dev/null +++ b/specs/abtesting-v3/spec.yml @@ -0,0 +1,109 @@ +openapi: 3.0.2 +externalDocs: + url: https://www.algolia.com/doc/guides/ab-testing/what-is-ab-testing/ + description: | + Related guide: A/B testing. +info: + title: A/B Testing API + description: | + The Algolia A/B Testing API lets you manage your Algolia A/B tests to optimize your search experience. + + ## Base URLs + + The base URLs for requests to the A/B testing API are: + + - `https://analytics.us.algolia.com` + - `https://analytics.de.algolia.com` + - `https://analytics.algolia.com` (routes requests to the closest of the above servers, based on your geographical location) + + Use the URL that matches your [analytics region](https://dashboard.algolia.com/account/infrastructure/analytics). + + **All requests must use HTTPS.** + + ## Availability and authentication + + Access to the A/B testing API is available as part of the [Premium or Elevate plans](https://www.algolia.com/pricing). + + To authenticate your API requests, add these headers: + + - `x-algolia-application-id`. Your Algolia application ID. + - `x-algolia-api-key`. An API key with the necessary permissions to make the request. + The required access control list (ACL) to make a request is listed in each endpoint's reference. + + You can find your application ID and API key in the [Algolia dashboard](https://dashboard.algolia.com/account). + + ## Rate limits + + You can make up to **100 requests per minute per app** to the A/B testing API. + The response includes headers with information about the limits. + + ## Parameters + + Query parameters must be [URL-encoded](https://developer.mozilla.org/en-US/docs/Glossary/Percent-encoding). + Non-ASCII characters must be UTF-8 encoded. + Plus characters (`+`) are interpreted as spaces. + + ## Response status and errors + + The A/B testing API returns JSON responses. + Since JSON doesn't guarantee any specific ordering, don't rely on the order of attributes in the API response. + + Successful responses return a `2xx` status. Client errors return a `4xx` status. Server errors are indicated by a `5xx` status. + Error responses have a `message` property with more information. + + ## Version + + The current version of the A/B Testing API is version 3, as indicated by the `/3/` in each endpoint's URL. + version: 3.0.0 +components: + securitySchemes: + appId: + $ref: "../common/securitySchemes.yml#/appId" + apiKey: + $ref: "../common/securitySchemes.yml#/apiKey" +servers: + - url: https://analytics.{region}.algolia.com + variables: + region: + enum: + - us + - de + default: us + - url: https://analytics.algolia.com +security: + - appId: [] + apiKey: [] +tags: + - name: abtest + x-displayName: A/B testing + description: | + Manage A/B tests. + + A/B tests are configurations one or more indices, usually your production index and an index with different settings that you want to test. +x-tagGroups: + - name: General + tags: + - abtest +paths: + # ###################### + # ### Custom request ### + # ###################### + /{path}: + $ref: "../common/paths/customRequest.yml" + + /3/abtests: + $ref: "paths/abtests.yml" + /3/abtests/{id}: + $ref: "paths/abtest.yml" + /3/abtests/{id}/stop: + $ref: "paths/stopABTest.yml" + /3/abtests/schedule: + $ref: "paths/scheduleABTest.yml" + /3/abtests/estimate: + $ref: "paths/estimate.yml" + + # ############### + # ### Helpers ### + # ############### + /setClientApiKey: + $ref: "../common/helpers/setClientApiKey.yml#/method" From f09e83e402a46d6e8a65d12ad46cee91f6c75beb Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 14:51:26 +0100 Subject: [PATCH 02/21] Update specs/abtesting-v3/common/schemas/Variant.yml Co-authored-by: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> --- specs/abtesting-v3/common/schemas/Variant.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml index b4c07a412dc..474339e6133 100644 --- a/specs/abtesting-v3/common/schemas/Variant.yml +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -4,7 +4,7 @@ variants: A/B test variants. The first variant is your _control_ index, typically your production index. - All of the additionnal variants are indexes with changed settings that you want to test against the control. + All of the additional variants are indexes with changed settings that you want to test against the control. items: $ref: "#/variant" From af3d013f32d93cce793f073a1fa36e07183c4176 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 14:51:40 +0100 Subject: [PATCH 03/21] Update specs/abtesting-v3/common/schemas/Variant.yml Co-authored-by: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> --- specs/abtesting-v3/common/schemas/Variant.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml index 474339e6133..f1552633546 100644 --- a/specs/abtesting-v3/common/schemas/Variant.yml +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -62,7 +62,7 @@ metric: type: number format: double description: | - Only present in case the metric is a revenue. + Only present in case the metric is 'revenue'. It is the normalized value of it. It is tied to a per revenue-currency pair contrary to other global filter effects (such as outliers and empty search count). From 9f413349adc7d392afe9acc8726a5a2bbec3fd11 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 14:51:50 +0100 Subject: [PATCH 04/21] Update specs/abtesting-v3/common/schemas/Variant.yml Co-authored-by: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> --- specs/abtesting-v3/common/schemas/Variant.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml index f1552633546..87774d27f9c 100644 --- a/specs/abtesting-v3/common/schemas/Variant.yml +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -63,7 +63,7 @@ metric: format: double description: | Only present in case the metric is 'revenue'. - It is the normalized value of it. + It is the amount exceeding the 95th percentile of global revenue transactions involved in the AB Test. This amount is not considered when calculating statistical significance. It is tied to a per revenue-currency pair contrary to other global filter effects (such as outliers and empty search count). required: From ebb0a580e5b481698f2af705ef7b06878d285d92 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 14:52:15 +0100 Subject: [PATCH 05/21] Update specs/abtesting-v3/paths/abtests.yml Co-authored-by: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> --- specs/abtesting-v3/paths/abtests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/abtesting-v3/paths/abtests.yml b/specs/abtesting-v3/paths/abtests.yml index c10258234a2..dcf14bd1df1 100644 --- a/specs/abtesting-v3/paths/abtests.yml +++ b/specs/abtesting-v3/paths/abtests.yml @@ -25,7 +25,7 @@ post: $ref: "../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant" metrics: type: array - description: A/B test metrics to retrieve. + description: A/B test metrics involved in the test. Only these metrics will be considered when calculating results. items: $ref: "../common/parameters.yml#/metric" configuration: From 0a7faeb9f61745b24efef3c0e73ea0c7e91ef025 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 14:52:35 +0100 Subject: [PATCH 06/21] Update specs/abtesting-v3/paths/scheduleABTest.yml Co-authored-by: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> --- specs/abtesting-v3/paths/scheduleABTest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/abtesting-v3/paths/scheduleABTest.yml b/specs/abtesting-v3/paths/scheduleABTest.yml index 3939d4314c4..dbceed987ba 100644 --- a/specs/abtesting-v3/paths/scheduleABTest.yml +++ b/specs/abtesting-v3/paths/scheduleABTest.yml @@ -22,7 +22,6 @@ post: type: array description: A/B test variants. minItems: 2 - maxItems: 2 items: $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant' scheduledAt: From b75ccfe45ba69053944b7180f0ec2bdb84c70e26 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 14:52:44 +0100 Subject: [PATCH 07/21] Update specs/abtesting-v3/paths/estimate.yml Co-authored-by: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> --- specs/abtesting-v3/paths/estimate.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/abtesting-v3/paths/estimate.yml b/specs/abtesting-v3/paths/estimate.yml index fa29a90a080..1869ee5a569 100644 --- a/specs/abtesting-v3/paths/estimate.yml +++ b/specs/abtesting-v3/paths/estimate.yml @@ -32,7 +32,6 @@ post: type: array description: A/B test variants. minItems: 2 - maxItems: 2 items: $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant' required: From 99a31a78f86fdb15926fd3b740e64c231100b1c5 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 15:49:11 +0100 Subject: [PATCH 08/21] fix(abtests) remove significance from abtest --- specs/abtesting-v3/common/schemas/ABTest.yml | 89 +++----------------- 1 file changed, 14 insertions(+), 75 deletions(-) diff --git a/specs/abtesting-v3/common/schemas/ABTest.yml b/specs/abtesting-v3/common/schemas/ABTest.yml index a769d098a06..a967d23e981 100644 --- a/specs/abtesting-v3/common/schemas/ABTest.yml +++ b/specs/abtesting-v3/common/schemas/ABTest.yml @@ -3,8 +3,8 @@ ABTests: - type: array description: A/B tests. items: - $ref: '#/ABTest' - - type: 'null' + $ref: "#/ABTest" + - type: "null" description: No A/B tests are configured for this application. ABTest: @@ -12,82 +12,21 @@ ABTest: additionalProperties: false properties: abTestID: - $ref: '../parameters.yml#/abTestID' - clickSignificance: - description: | - A/B test significance calculated from click events. - - Values of 0.95 or higher can be considered significant, - that is, the difference between A and B variants is _not_ due to random variations. - Lower values have a. - oneOf: - - type: number - format: double - example: 1 - - type: 'null' - conversionSignificance: - description: | - A/B test significance calculated from conversion events. - - Values of 0.95 or higher can be considered significant, - that is, the difference between A and B variants is _not_ due to random variations. - oneOf: - - type: number - format: double - example: 1 - - type: 'null' - addToCartSignificance: - description: | - A/B test significance calculated from add-to-cart events. - - Values of 0.95 or higher can be considered significant, - that is, the difference between A and B variants is _not_ due to random variations. - oneOf: - - type: number - format: double - example: 1 - - type: 'null' - purchaseSignificance: - description: | - A/B test significance calculated from purchase events. - - Values of 0.95 or higher can be considered significant, - that is, the difference between A and B variants is _not_ due to random variations. - oneOf: - - type: number - format: double - example: 1 - - type: 'null' - revenueSignificance: - description: | - A/B test significance calculated from revenue data. - - Values of 0.95 or higher can be considered significant, - that is, the difference between A and B variants is _not_ due to random variations. - oneOf: - - type: object - additionalProperties: - type: number - format: double - x-additionalPropertiesName: currency code - example: - USD: 1 - EUR: 0.87 - - type: 'null' + $ref: "../parameters.yml#/abTestID" updatedAt: - $ref: '../parameters.yml#/updatedAt' + $ref: "../parameters.yml#/updatedAt" createdAt: - $ref: '../parameters.yml#/createdAt' + $ref: "../parameters.yml#/createdAt" endAt: - $ref: '../parameters.yml#/endAt' + $ref: "../parameters.yml#/endAt" name: - $ref: '../parameters.yml#/name' + $ref: "../parameters.yml#/name" status: - $ref: '#/Status' + $ref: "#/Status" variants: - $ref: 'Variant.yml#/variants' + $ref: "Variant.yml#/variants" configuration: - $ref: '#/ABTestConfiguration' + $ref: "#/ABTestConfiguration" required: - status - name @@ -119,11 +58,11 @@ ABTestConfiguration: description: A/B test configuration. properties: outliers: - $ref: '#/Outliers' + $ref: "#/Outliers" emptySearch: - $ref: '#/EmptySearch' + $ref: "#/EmptySearch" minimumDetectableEffect: - $ref: '#/MinimumDetectableEffect' + $ref: "#/MinimumDetectableEffect" Outliers: type: object @@ -155,7 +94,7 @@ MinimumDetectableEffect: Smallest difference in an observable metric between variants. For example, to detect a 10% difference between variants, set this value to 0.1. metric: - $ref: '#/EffectMetric' + $ref: "#/EffectMetric" required: - size - metric From 92286d8f9c5ce7261d0cee64c687a35ed92bf1c2 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 16:31:38 +0100 Subject: [PATCH 09/21] feat(abtests) move filterEffects in metadata + add description to pValue --- specs/abtesting-v3/common/schemas/Variant.yml | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml index 87774d27f9c..ef6dc6d4601 100644 --- a/specs/abtesting-v3/common/schemas/Variant.yml +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -21,14 +21,15 @@ variant: The A/B test configuration must include a `mininmumDetectableEffect` setting for this number to be included in the response. example: 0 - filterEffects: - $ref: "../parameters.yml#/filterEffects" index: $ref: "../parameters.yml#/index" trafficPercentage: $ref: "../parameters.yml#/trafficPercentage" metrics: $ref: "#/metrics" + metadata: + $ref: "#/metadata" + required: - index - description @@ -52,9 +53,10 @@ metric: value: type: number format: double - pvalue: + pValue: type: number format: double + description: PValue for the first variant (control) will always be 0. For the other variants, pValue is calculated for the current variant based on the control. dimension: type: string description: Dimension defined during test creation @@ -70,25 +72,31 @@ metric: - name - updatedAt - value - - pvalue + - pValue example: - name: addToCartCount updatedAt: 2025-06-15T15:06:44.400601Z value: 5 - pvalue: 0.01 + pValue: 0.01 - name: clickThroughRate updatedAt: 2025-05-15T17:52:15.644906Z value: 0.20869847452125934 - pvalue: 0.004 + pValue: 0.004 - name: revenue dimension: USD updatedAt": 2025-05-15T17:52:15.644906Z value: 1200.50 pValue: 0.04 - winsorizedValue: 1123.45 + example: 20.2 - name: revenue dimension: EUR updatedAt: 2025-05-15T17:52:15.644906Z value: 999.66 pValue: 0.04 - winsorizedValue: 888.44 + example: 888.44 + +metadata: + type: object + properties: + filterEffects: + $ref: "../parameters.yml#/filterEffects" From a04ae8c27e0c394f8a2f35dedd7d2adfda66b8a6 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Tue, 28 Jan 2025 17:00:43 +0100 Subject: [PATCH 10/21] feat(abtests) add metadata into metric --- specs/abtesting-v3/common/schemas/Variant.yml | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml index ef6dc6d4601..9f5202fc7fc 100644 --- a/specs/abtesting-v3/common/schemas/Variant.yml +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -28,7 +28,7 @@ variant: metrics: $ref: "#/metrics" metadata: - $ref: "#/metadata" + $ref: "#/variantMetadata" required: - index @@ -60,14 +60,8 @@ metric: dimension: type: string description: Dimension defined during test creation - winsorizedValue: - type: number - format: double - description: | - Only present in case the metric is 'revenue'. - It is the amount exceeding the 95th percentile of global revenue transactions involved in the AB Test. This amount is not considered when calculating statistical significance. - It is tied to a per revenue-currency pair contrary to other - global filter effects (such as outliers and empty search count). + metadata: + $ref: "#/metricMetadata" required: - name - updatedAt @@ -84,19 +78,35 @@ metric: pValue: 0.004 - name: revenue dimension: USD - updatedAt": 2025-05-15T17:52:15.644906Z + updatedAt: 2025-05-15T17:52:15.644906Z value: 1200.50 pValue: 0.04 - example: 20.2 + metadata: { "winsorizedValue": 80.2 } - name: revenue dimension: EUR updatedAt: 2025-05-15T17:52:15.644906Z value: 999.66 pValue: 0.04 - example: 888.44 + metadata: { "winsorizedValue": 888.8 } -metadata: +variantMetadata: type: object + description: Variant specific metadata properties: filterEffects: $ref: "../parameters.yml#/filterEffects" + +metricMetadata: + type: object + description: Metric specific metadata + properties: + winsorizedValue: + type: number + format: double + description: | + Only present in case the metric is 'revenue'. + It is the amount exceeding the 95th percentile of global revenue transactions involved in the AB Test. This amount is not considered when calculating statistical significance. + It is tied to a per revenue-currency pair contrary to other + global filter effects (such as outliers and empty search count). + example: + winsorizedValue: 888.80 From 9ca8fad46178fa384265be4813f1b11871a61905 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Thu, 10 Apr 2025 14:57:36 +0200 Subject: [PATCH 11/21] doc(abtests) add timeseries endpoint for v3 + add confidence interval values in variant schema --- .../common/schemas/Timeseries.yml | 43 +++++++++++++++++++ specs/abtesting-v3/common/schemas/Variant.yml | 12 ++++++ specs/abtesting-v3/paths/timeseries.yml | 32 ++++++++++++++ specs/abtesting-v3/spec.yml | 2 + 4 files changed, 89 insertions(+) create mode 100644 specs/abtesting-v3/common/schemas/Timeseries.yml create mode 100644 specs/abtesting-v3/paths/timeseries.yml diff --git a/specs/abtesting-v3/common/schemas/Timeseries.yml b/specs/abtesting-v3/common/schemas/Timeseries.yml new file mode 100644 index 00000000000..2f2f3c00ca9 --- /dev/null +++ b/specs/abtesting-v3/common/schemas/Timeseries.yml @@ -0,0 +1,43 @@ +Timeseries: + type: object + additionalProperties: false + properties: + abTestID: + $ref: "../parameters.yml#/abTestID" + variants: + $ref: "#/timeseriesVariants" + required: + - abTestID + - variants + +timeseriesVariants: + type: array + description: | + A/B test timeseries variants. + + The first variant is your _control_ index, typically your production index. + All of the additional variants are indexes with changed settings that you want to test against the control. + items: + $ref: "#/timeseriesVariant" + +timeseriesVariant: + type: object + properties: + dates: + $ref: "#/metricDates" + +metricDates: + type: array + items: + $ref: "#/metricDate" + +metricDate: + type: object + properties: + date: + type: string + description: Date where the metric was updated, in RFC 3339 format. + format: date + example: 2025-06-15 + metrics: + $ref: "Variant.yml#/metrics" diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml index 9f5202fc7fc..70764bbe113 100644 --- a/specs/abtesting-v3/common/schemas/Variant.yml +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -53,10 +53,22 @@ metric: value: type: number format: double + valueCIHigh: + type: number + format: double + valueCILow: + type: number + format: double pValue: type: number format: double description: PValue for the first variant (control) will always be 0. For the other variants, pValue is calculated for the current variant based on the control. + pValueCIHigh: + type: number + format: double + pValueCILow: + type: number + format: double dimension: type: string description: Dimension defined during test creation diff --git a/specs/abtesting-v3/paths/timeseries.yml b/specs/abtesting-v3/paths/timeseries.yml new file mode 100644 index 00000000000..dd95cf72d94 --- /dev/null +++ b/specs/abtesting-v3/paths/timeseries.yml @@ -0,0 +1,32 @@ +get: + tags: + - abtest + operationId: getTimeseries + x-acl: + - analytics + summary: Retrieve timeseries of an A/B test + description: Retrieves timeseries for an A/B test by its ID. + parameters: + - $ref: '../common/parameters.yml#/ID' + responses: + '200': + description: OK + headers: + x-ratelimit-limit: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' + x-ratelimit-remaining: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' + x-ratelimit-reset: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' + content: + application/json: + schema: + $ref: '../common/schemas/Timeseries.yml#/Timeseries' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' diff --git a/specs/abtesting-v3/spec.yml b/specs/abtesting-v3/spec.yml index 31a92a8bff2..5d39a52e194 100644 --- a/specs/abtesting-v3/spec.yml +++ b/specs/abtesting-v3/spec.yml @@ -101,6 +101,8 @@ paths: $ref: "paths/scheduleABTest.yml" /3/abtests/estimate: $ref: "paths/estimate.yml" + /3/abtests/{id}/timeseries: + $ref: "paths/timeseries.yml" # ############### # ### Helpers ### From f2b99f0400fc52b39389b33b3e1bd9a3071f1b8b Mon Sep 17 00:00:00 2001 From: Steven Evans Date: Mon, 14 Apr 2025 18:34:43 -0400 Subject: [PATCH 12/21] add confidence interval field descriptions --- specs/abtesting-v3/common/schemas/Variant.yml | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml index 70764bbe113..1106e69fb26 100644 --- a/specs/abtesting-v3/common/schemas/Variant.yml +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -6,29 +6,29 @@ variants: The first variant is your _control_ index, typically your production index. All of the additional variants are indexes with changed settings that you want to test against the control. items: - $ref: "#/variant" + $ref: '#/variant' variant: type: object additionalProperties: false properties: description: - $ref: "../parameters.yml#/description" + $ref: '../parameters.yml#/description' estimatedSampleSize: type: integer description: | Estimated number of searches required to achieve the desired statistical significance. - The A/B test configuration must include a `mininmumDetectableEffect` setting for this number to be included in the response. + The A/B test configuration must include a `minimumDetectableEffect` setting for this number to be included in the response. example: 0 index: - $ref: "../parameters.yml#/index" + $ref: '../parameters.yml#/index' trafficPercentage: - $ref: "../parameters.yml#/trafficPercentage" + $ref: '../parameters.yml#/trafficPercentage' metrics: - $ref: "#/metrics" + $ref: '#/metrics' metadata: - $ref: "#/variantMetadata" + $ref: '#/variantMetadata' required: - index @@ -38,9 +38,9 @@ variant: metrics: type: array - description: All ABTest metrics that were defined during test creation + description: All ABTest metrics that were defined during test creation. items: - $ref: "#/metric" + $ref: '#/metric' metric: type: object @@ -56,24 +56,28 @@ metric: valueCIHigh: type: number format: double + description: | + The upper bound of the 95% confidence interval for the metric value. The confidence interval is calculated using + either the relative ratio or relative difference between the metric values for the control and the variant. + Relative ratio is used for metrics that are ratios (e.g., click-through rate, conversion rate), + while relative difference is used for continuous metrics (e.g., revenue). valueCILow: type: number format: double + description: | + The lower bound of the 95% confidence interval for the metric value. The confidence interval is calculated using + either the relative ratio or relative difference between the metric values for the control and the variant. + Relative ratio is used for metrics that are ratios (e.g., click-through rate, conversion rate), + while relative difference is used for continuous metrics (e.g., revenue). pValue: type: number format: double description: PValue for the first variant (control) will always be 0. For the other variants, pValue is calculated for the current variant based on the control. - pValueCIHigh: - type: number - format: double - pValueCILow: - type: number - format: double dimension: type: string - description: Dimension defined during test creation + description: Dimension defined during test creation. metadata: - $ref: "#/metricMetadata" + $ref: '#/metricMetadata' required: - name - updatedAt @@ -93,24 +97,24 @@ metric: updatedAt: 2025-05-15T17:52:15.644906Z value: 1200.50 pValue: 0.04 - metadata: { "winsorizedValue": 80.2 } + metadata: {'winsorizedValue': 80.2} - name: revenue dimension: EUR updatedAt: 2025-05-15T17:52:15.644906Z value: 999.66 pValue: 0.04 - metadata: { "winsorizedValue": 888.8 } + metadata: {'winsorizedValue': 888.8} variantMetadata: type: object - description: Variant specific metadata + description: Variant specific metadata. properties: filterEffects: - $ref: "../parameters.yml#/filterEffects" + $ref: '../parameters.yml#/filterEffects' metricMetadata: type: object - description: Metric specific metadata + description: Metric specific metadata. properties: winsorizedValue: type: number From 6c2f5e27b566721c5c677a5a08c18e3707f677f3 Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Wed, 16 Apr 2025 10:53:04 +0200 Subject: [PATCH 13/21] fix(abtests) traffic percentage min and max + add migratedAbTestID to get/list responses --- specs/abtesting-v3/common/parameters.yml | 4 ++-- specs/abtesting-v3/common/schemas/ABTest.yml | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/specs/abtesting-v3/common/parameters.yml b/specs/abtesting-v3/common/parameters.yml index 1be5a7b9764..77d924b6f60 100644 --- a/specs/abtesting-v3/common/parameters.yml +++ b/specs/abtesting-v3/common/parameters.yml @@ -56,8 +56,8 @@ description: trafficPercentage: type: integer description: Percentage of search requests each variant receives. - minimum: 0 - maximum: 100 + minimum: 1 + maximum: 99 example: 60 currencies: diff --git a/specs/abtesting-v3/common/schemas/ABTest.yml b/specs/abtesting-v3/common/schemas/ABTest.yml index a967d23e981..4818799f987 100644 --- a/specs/abtesting-v3/common/schemas/ABTest.yml +++ b/specs/abtesting-v3/common/schemas/ABTest.yml @@ -27,6 +27,8 @@ ABTest: $ref: "Variant.yml#/variants" configuration: $ref: "#/ABTestConfiguration" + migratedAbTestID: + $ref: "#/MigratedABTestId" required: - status - name @@ -107,3 +109,8 @@ EffectMetric: - clickThroughRate - conversionRate - purchaseRate + +MigratedABTestId: + type: integer + description: Unique migrated A/B test identifier. + example: 224 From b83e6de14b4ebe4485ce533924b712766eeb358a Mon Sep 17 00:00:00 2001 From: My Lan Aragon Date: Fri, 25 Apr 2025 10:39:11 +0200 Subject: [PATCH 14/21] feat(abtests) add error correction to get/list endpoint for v3 --- specs/abtesting-v3/common/schemas/Variant.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/specs/abtesting-v3/common/schemas/Variant.yml b/specs/abtesting-v3/common/schemas/Variant.yml index 1106e69fb26..d7c69e7ebed 100644 --- a/specs/abtesting-v3/common/schemas/Variant.yml +++ b/specs/abtesting-v3/common/schemas/Variant.yml @@ -78,6 +78,17 @@ metric: description: Dimension defined during test creation. metadata: $ref: '#/metricMetadata' + criticalValue: + type: number + format: double + description: | + The value that was computed during error correction. It is used to determine significance of the metric pValue. + The critical value is calculated using Bonferroni or Benjamini-Hochberg corrections, based on the given + configuration during the A/B test creation. + significant: + type: boolean + description: | + Whether the pValue is significant or not based on the critical value and the error correction algorithm used. required: - name - updatedAt From 0bf629d2a9e7abe92e30d63cb3c2eee18c2e3874 Mon Sep 17 00:00:00 2001 From: Leonardo Gavaudan Date: Mon, 26 May 2025 10:38:52 -0400 Subject: [PATCH 15/21] feat(abtests): Add query params for GET /3/abtests/{id}/timeseries (#4881) --- specs/abtesting-v3/paths/timeseries.yml | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/specs/abtesting-v3/paths/timeseries.yml b/specs/abtesting-v3/paths/timeseries.yml index dd95cf72d94..03408dce60a 100644 --- a/specs/abtesting-v3/paths/timeseries.yml +++ b/specs/abtesting-v3/paths/timeseries.yml @@ -8,6 +8,32 @@ get: description: Retrieves timeseries for an A/B test by its ID. parameters: - $ref: '../common/parameters.yml#/ID' + - $ref: '../../common/parameters.yml#/StartDate' + - $ref: '../../common/parameters.yml#/EndDate' + - in: query + name: metric + description: List of metrics to retrieve. If not specified, all metrics are returned. + schema: + type: array + items: + type: string + enum: + - search_count + - tracked_search_count + - user_count + - tracked_user_count + - no_result_count + - add_to_cart_count + - purchase_count + - clicked_search_count + - converted_search_count + - click_through_rate + - conversion_rate + - add_to_cart_rate + - purchase_rate + - average_click_position + - revenue + example: ["search_count", "click_through_rate", "conversion_rate"] responses: '200': description: OK @@ -30,3 +56,5 @@ get: $ref: '../../common/responses/MethodNotAllowed.yml' '404': $ref: '../../common/responses/IndexNotFound.yml' + '422': + $ref: '../../common/responses/UnprocessableEntity.yml' From 4cebf3f9d89cfaa34e39dae7fd2344894e32bfd9 Mon Sep 17 00:00:00 2001 From: Leonardo Gavaudan Date: Fri, 4 Jul 2025 11:52:23 +0200 Subject: [PATCH 16/21] Update specs/abtesting-v3/common/schemas/ABTest.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Vannicatte --- specs/abtesting-v3/common/schemas/ABTest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/abtesting-v3/common/schemas/ABTest.yml b/specs/abtesting-v3/common/schemas/ABTest.yml index 4818799f987..a8368e677ae 100644 --- a/specs/abtesting-v3/common/schemas/ABTest.yml +++ b/specs/abtesting-v3/common/schemas/ABTest.yml @@ -5,7 +5,6 @@ ABTests: items: $ref: "#/ABTest" - type: "null" - description: No A/B tests are configured for this application. ABTest: type: object From 5beb7b195b90424983ddef6abdee6d8810bd1f49 Mon Sep 17 00:00:00 2001 From: Leonardo Gavaudan Date: Fri, 4 Jul 2025 12:16:42 +0200 Subject: [PATCH 17/21] Add filters and errorCorrection to AB test config, remove legacy outliers and empty search --- specs/abtesting-v3/common/schemas/ABTest.yml | 63 +++++++++++++------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/specs/abtesting-v3/common/schemas/ABTest.yml b/specs/abtesting-v3/common/schemas/ABTest.yml index a8368e677ae..c717585d782 100644 --- a/specs/abtesting-v3/common/schemas/ABTest.yml +++ b/specs/abtesting-v3/common/schemas/ABTest.yml @@ -58,29 +58,12 @@ ABTestConfiguration: type: object description: A/B test configuration. properties: - outliers: - $ref: "#/Outliers" - emptySearch: - $ref: "#/EmptySearch" minimumDetectableEffect: $ref: "#/MinimumDetectableEffect" - -Outliers: - type: object - description: Configuration for handling outliers. - properties: - exclude: - type: boolean - description: Whether to exclude outliers when calculating A/B test results. - default: true - -EmptySearch: - type: object - description: Configuration for handling empty searches. - properties: - exclude: - type: boolean - description: Whether to exclude empty searches when calculating A/B test results. + filters: + $ref: "#/MetricsFilters" + errorCorrection: + $ref: "#/ErrorCorrectionType" MinimumDetectableEffect: type: object @@ -113,3 +96,41 @@ MigratedABTestId: type: integer description: Unique migrated A/B test identifier. example: 224 + +MetricsFilter: + type: object + additionalProperties: false + description: | + Boolean filter applied to the A/B test population. Each filter targets a boolean metric + and decides whether to include (true) or exclude (false) matching records. + properties: + domain: + type: string + description: Metric domain (for example `abtesting`, `personalization`). + example: abtesting + name: + type: string + description: Public metric name. + example: isOutlier + trackEffects: + type: boolean + description: Whether the experiment should record the effects of this filter. + includes: + type: boolean + description: If true, keep items that match the filter; if false, exclude them. + required: + - domain + - name + +MetricsFilters: + type: array + description: List of metric filters applied to the test population. + items: + $ref: "#/MetricsFilter" + +ErrorCorrectionType: + type: string + description: Multiple-testing correction method applied when evaluating metric significance. + enum: + - bonferroni + - benjamini-hochberg From 77b77ef321ebc68929b7ac571e47e136629f2949 Mon Sep 17 00:00:00 2001 From: Leonardo Gavaudan Date: Fri, 4 Jul 2025 12:17:33 +0200 Subject: [PATCH 18/21] Add noResults to list of metrics supported --- specs/abtesting-v3/common/schemas/ABTest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/abtesting-v3/common/schemas/ABTest.yml b/specs/abtesting-v3/common/schemas/ABTest.yml index c717585d782..dca2644bc50 100644 --- a/specs/abtesting-v3/common/schemas/ABTest.yml +++ b/specs/abtesting-v3/common/schemas/ABTest.yml @@ -91,6 +91,7 @@ EffectMetric: - clickThroughRate - conversionRate - purchaseRate + - noResultsRate MigratedABTestId: type: integer From 2dd55040b842e87a6451284d6b91d1a30ce70c65 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Fri, 4 Jul 2025 14:27:25 +0200 Subject: [PATCH 19/21] chore: validate specs on ci/cli --- scripts/common.ts | 2 +- specs/abtesting-v3/common/parameters.yml | 8 +-- specs/abtesting-v3/common/schemas/ABTest.yml | 40 ++++++----- .../common/schemas/Timeseries.yml | 31 +++++++-- specs/abtesting-v3/paths/abtests.yml | 66 +++++++++---------- specs/abtesting-v3/paths/timeseries.yml | 19 +----- specs/abtesting-v3/spec.yml | 20 +++--- 7 files changed, 98 insertions(+), 88 deletions(-) diff --git a/scripts/common.ts b/scripts/common.ts index 58cc7e20b41..ff4b7549bb4 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -73,7 +73,7 @@ export const LANGUAGES = [...new Set(Object.values(GENERATORS).map((gen) => gen. // `crawler` is manually added so we can still bundled and validate the specs // the entry can be removed once at least one client is generated -export const CLIENTS = [...new Set(Object.values(GENERATORS).map((gen) => gen.client)), 'crawler']; +export const CLIENTS = [...new Set(Object.values(GENERATORS).map((gen) => gen.client)), 'crawler', 'abtesting-v3']; export async function run(command: string, { errorMessage, cwd, language }: RunOptions = {}): Promise { const realCwd = path.resolve(ROOT_DIR, cwd ?? '.'); diff --git a/specs/abtesting-v3/common/parameters.yml b/specs/abtesting-v3/common/parameters.yml index 77d924b6f60..e88c606ebf5 100644 --- a/specs/abtesting-v3/common/parameters.yml +++ b/specs/abtesting-v3/common/parameters.yml @@ -5,13 +5,13 @@ ID: description: Unique A/B test identifier. required: true schema: - $ref: "#/abTestID" + $ref: '#/abTestID' # misc index: type: string description: Index name of the A/B test variant (case-sensitive). - example: "delcourt_production" + example: 'delcourt_production' abTestID: type: integer @@ -75,7 +75,7 @@ currencies: mean: 43.7 standardDeviation: 10.3 additionalProperties: - $ref: "#/currency" + $ref: '#/currency' x-additionalPropertiesName: currency code currency: @@ -84,7 +84,7 @@ currency: currency: type: string description: Currency code. - example: "USD" + example: 'USD' revenue: type: number format: double diff --git a/specs/abtesting-v3/common/schemas/ABTest.yml b/specs/abtesting-v3/common/schemas/ABTest.yml index dca2644bc50..0dd8d6f06df 100644 --- a/specs/abtesting-v3/common/schemas/ABTest.yml +++ b/specs/abtesting-v3/common/schemas/ABTest.yml @@ -3,31 +3,39 @@ ABTests: - type: array description: A/B tests. items: - $ref: "#/ABTest" - - type: "null" + $ref: '#/ABTest' + - type: 'null' + +EmptySearch: + type: object + description: Configuration for handling empty searches. + properties: + exclude: + type: boolean + description: Whether to exclude empty searches when calculating A/B test results. ABTest: type: object additionalProperties: false properties: abTestID: - $ref: "../parameters.yml#/abTestID" + $ref: '../parameters.yml#/abTestID' updatedAt: - $ref: "../parameters.yml#/updatedAt" + $ref: '../parameters.yml#/updatedAt' createdAt: - $ref: "../parameters.yml#/createdAt" + $ref: '../parameters.yml#/createdAt' endAt: - $ref: "../parameters.yml#/endAt" + $ref: '../parameters.yml#/endAt' name: - $ref: "../parameters.yml#/name" + $ref: '../parameters.yml#/name' status: - $ref: "#/Status" + $ref: '#/Status' variants: - $ref: "Variant.yml#/variants" + $ref: 'Variant.yml#/variants' configuration: - $ref: "#/ABTestConfiguration" + $ref: '#/ABTestConfiguration' migratedAbTestID: - $ref: "#/MigratedABTestId" + $ref: '#/MigratedABTestId' required: - status - name @@ -59,11 +67,11 @@ ABTestConfiguration: description: A/B test configuration. properties: minimumDetectableEffect: - $ref: "#/MinimumDetectableEffect" + $ref: '#/MinimumDetectableEffect' filters: - $ref: "#/MetricsFilters" + $ref: '#/MetricsFilters' errorCorrection: - $ref: "#/ErrorCorrectionType" + $ref: '#/ErrorCorrectionType' MinimumDetectableEffect: type: object @@ -78,7 +86,7 @@ MinimumDetectableEffect: Smallest difference in an observable metric between variants. For example, to detect a 10% difference between variants, set this value to 0.1. metric: - $ref: "#/EffectMetric" + $ref: '#/EffectMetric' required: - size - metric @@ -127,7 +135,7 @@ MetricsFilters: type: array description: List of metric filters applied to the test population. items: - $ref: "#/MetricsFilter" + $ref: '#/MetricsFilter' ErrorCorrectionType: type: string diff --git a/specs/abtesting-v3/common/schemas/Timeseries.yml b/specs/abtesting-v3/common/schemas/Timeseries.yml index 2f2f3c00ca9..377e10abcbf 100644 --- a/specs/abtesting-v3/common/schemas/Timeseries.yml +++ b/specs/abtesting-v3/common/schemas/Timeseries.yml @@ -3,9 +3,9 @@ Timeseries: additionalProperties: false properties: abTestID: - $ref: "../parameters.yml#/abTestID" + $ref: '../parameters.yml#/abTestID' variants: - $ref: "#/timeseriesVariants" + $ref: '#/timeseriesVariants' required: - abTestID - variants @@ -18,18 +18,18 @@ timeseriesVariants: The first variant is your _control_ index, typically your production index. All of the additional variants are indexes with changed settings that you want to test against the control. items: - $ref: "#/timeseriesVariant" + $ref: '#/timeseriesVariant' timeseriesVariant: type: object properties: dates: - $ref: "#/metricDates" + $ref: '#/metricDates' metricDates: type: array items: - $ref: "#/metricDate" + $ref: '#/metricDate' metricDate: type: object @@ -40,4 +40,23 @@ metricDate: format: date example: 2025-06-15 metrics: - $ref: "Variant.yml#/metrics" + $ref: 'Variant.yml#/metrics' + +Metric: + type: string + enum: + - search_count + - tracked_search_count + - user_count + - tracked_user_count + - no_result_count + - add_to_cart_count + - purchase_count + - clicked_search_count + - converted_search_count + - click_through_rate + - conversion_rate + - add_to_cart_rate + - purchase_rate + - average_click_position + - revenue diff --git a/specs/abtesting-v3/paths/abtests.yml b/specs/abtesting-v3/paths/abtests.yml index dcf14bd1df1..e31ce03e6c1 100644 --- a/specs/abtesting-v3/paths/abtests.yml +++ b/specs/abtesting-v3/paths/abtests.yml @@ -16,49 +16,49 @@ post: additionalProperties: false properties: name: - $ref: "../common/parameters.yml#/name" + $ref: '../common/parameters.yml#/name' variants: type: array description: A/B test variants. minItems: 2 items: - $ref: "../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant" + $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant' metrics: type: array description: A/B test metrics involved in the test. Only these metrics will be considered when calculating results. items: - $ref: "../common/parameters.yml#/metric" + $ref: '../common/parameters.yml#/metric' configuration: - $ref: "../common/schemas/ABTest.yml#/ABTestConfiguration" + $ref: '../common/schemas/ABTest.yml#/ABTestConfiguration' endAt: - $ref: "../common/parameters.yml#/endAt" + $ref: '../common/parameters.yml#/endAt' required: - name - variants - metrics - endAt responses: - "200": + '200': description: OK headers: x-ratelimit-limit: - $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-limit" + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' x-ratelimit-remaining: - $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-remaining" + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' x-ratelimit-reset: - $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-reset" + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' content: application/json: schema: - $ref: "../common/schemas/ABTestResponse.yml#/ABTestResponse" - "400": - $ref: "../../common/responses/BadRequest.yml" - "402": - $ref: "../../common/responses/FeatureNotEnabled.yml" - "403": - $ref: "../../common/responses/MethodNotAllowed.yml" - "404": - $ref: "../../common/responses/IndexNotFound.yml" + $ref: '../common/schemas/ABTestResponse.yml#/ABTestResponse' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' get: tags: @@ -87,25 +87,25 @@ get: - name: indexPrefix in: query description: Index name prefix. Only A/B tests for indices starting with this string are included in the response. - example: "dev_" + example: 'dev_' schema: type: string - name: indexSuffix in: query description: Index name suffix. Only A/B tests for indices ending with this string are included in the response. - example: "_development" + example: '_development' schema: type: string responses: - "200": + '200': description: OK headers: x-ratelimit-limit: - $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-limit" + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' x-ratelimit-remaining: - $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-remaining" + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' x-ratelimit-reset: - $ref: "../../common/responses/rateLimit.yml#/x-ratelimit-reset" + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' content: application/json: schema: @@ -114,7 +114,7 @@ get: additionalProperties: false properties: abtests: - $ref: "../common/schemas/ABTest.yml#/ABTests" + $ref: '../common/schemas/ABTest.yml#/ABTests' count: type: integer description: Number of A/B tests. @@ -127,11 +127,11 @@ get: - abtests - count - total - "400": - $ref: "../../common/responses/BadRequest.yml" - "402": - $ref: "../../common/responses/FeatureNotEnabled.yml" - "403": - $ref: "../../common/responses/MethodNotAllowed.yml" - "404": - $ref: "../../common/responses/IndexNotFound.yml" + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' diff --git a/specs/abtesting-v3/paths/timeseries.yml b/specs/abtesting-v3/paths/timeseries.yml index 03408dce60a..de530f062fd 100644 --- a/specs/abtesting-v3/paths/timeseries.yml +++ b/specs/abtesting-v3/paths/timeseries.yml @@ -16,24 +16,7 @@ get: schema: type: array items: - type: string - enum: - - search_count - - tracked_search_count - - user_count - - tracked_user_count - - no_result_count - - add_to_cart_count - - purchase_count - - clicked_search_count - - converted_search_count - - click_through_rate - - conversion_rate - - add_to_cart_rate - - purchase_rate - - average_click_position - - revenue - example: ["search_count", "click_through_rate", "conversion_rate"] + $ref: '../common/schemas/Timeseries.yml#/Metric' responses: '200': description: OK diff --git a/specs/abtesting-v3/spec.yml b/specs/abtesting-v3/spec.yml index 5d39a52e194..d25d3cf3871 100644 --- a/specs/abtesting-v3/spec.yml +++ b/specs/abtesting-v3/spec.yml @@ -58,9 +58,9 @@ info: components: securitySchemes: appId: - $ref: "../common/securitySchemes.yml#/appId" + $ref: '../common/securitySchemes.yml#/appId' apiKey: - $ref: "../common/securitySchemes.yml#/apiKey" + $ref: '../common/securitySchemes.yml#/apiKey' servers: - url: https://analytics.{region}.algolia.com variables: @@ -89,23 +89,23 @@ paths: # ### Custom request ### # ###################### /{path}: - $ref: "../common/paths/customRequest.yml" + $ref: '../common/paths/customRequest.yml' /3/abtests: - $ref: "paths/abtests.yml" + $ref: 'paths/abtests.yml' /3/abtests/{id}: - $ref: "paths/abtest.yml" + $ref: 'paths/abtest.yml' /3/abtests/{id}/stop: - $ref: "paths/stopABTest.yml" + $ref: 'paths/stopABTest.yml' /3/abtests/schedule: - $ref: "paths/scheduleABTest.yml" + $ref: 'paths/scheduleABTest.yml' /3/abtests/estimate: - $ref: "paths/estimate.yml" + $ref: 'paths/estimate.yml' /3/abtests/{id}/timeseries: - $ref: "paths/timeseries.yml" + $ref: 'paths/timeseries.yml' # ############### # ### Helpers ### # ############### /setClientApiKey: - $ref: "../common/helpers/setClientApiKey.yml#/method" + $ref: '../common/helpers/setClientApiKey.yml#/method' From c20a3296df0a3c24817cb00c48efd936f985c102 Mon Sep 17 00:00:00 2001 From: Leonardo Gavaudan Date: Fri, 4 Jul 2025 15:00:04 +0200 Subject: [PATCH 20/21] add metrics and configuration to the schedule endpoint --- specs/abtesting-v3/paths/scheduleABTest.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/specs/abtesting-v3/paths/scheduleABTest.yml b/specs/abtesting-v3/paths/scheduleABTest.yml index dbceed987ba..c8d4f2d4865 100644 --- a/specs/abtesting-v3/paths/scheduleABTest.yml +++ b/specs/abtesting-v3/paths/scheduleABTest.yml @@ -24,6 +24,13 @@ post: minItems: 2 items: $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant' + metrics: + type: array + description: A/B test metrics involved in the test. Only these metrics will be considered when calculating results. + items: + $ref: '../common/parameters.yml#/metric' + configuration: + $ref: '../common/schemas/ABTest.yml#/ABTestConfiguration' scheduledAt: $ref: '../common/parameters.yml#/scheduledAt' endAt: @@ -31,6 +38,7 @@ post: required: - name - variants + - metrics - scheduledAt - endAt responses: From 0ce4b0c10267ac3747fc0597931a9ff813b0a03a Mon Sep 17 00:00:00 2001 From: Leonardo Gavaudan Date: Fri, 4 Jul 2025 15:21:55 +0200 Subject: [PATCH 21/21] Fix tests: replace outliers and empty search with metric filters in estimate endpoint --- specs/abtesting-v3/common/schemas/ABTest.yml | 8 -------- specs/abtesting-v3/paths/estimate.yml | 6 ++---- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/specs/abtesting-v3/common/schemas/ABTest.yml b/specs/abtesting-v3/common/schemas/ABTest.yml index 0dd8d6f06df..0245da7a15f 100644 --- a/specs/abtesting-v3/common/schemas/ABTest.yml +++ b/specs/abtesting-v3/common/schemas/ABTest.yml @@ -6,14 +6,6 @@ ABTests: $ref: '#/ABTest' - type: 'null' -EmptySearch: - type: object - description: Configuration for handling empty searches. - properties: - exclude: - type: boolean - description: Whether to exclude empty searches when calculating A/B test results. - ABTest: type: object additionalProperties: false diff --git a/specs/abtesting-v3/paths/estimate.yml b/specs/abtesting-v3/paths/estimate.yml index 1869ee5a569..fa2bbf5663b 100644 --- a/specs/abtesting-v3/paths/estimate.yml +++ b/specs/abtesting-v3/paths/estimate.yml @@ -20,10 +20,8 @@ post: type: object description: A/B test configuration for estimating the sample size and duration using minimum detectable effect. properties: - outliers: - $ref: '../common/schemas/ABTest.yml#/Outliers' - emptySearch: - $ref: '../common/schemas/ABTest.yml#/EmptySearch' + filters: + $ref: '../common/schemas/ABTest.yml#/MetricsFilters' minimumDetectableEffect: $ref: '../common/schemas/ABTest.yml#/MinimumDetectableEffect' required: