feat(specs): extend ABTest data model with Bayesian fields

clients/algoliasearch-client-javascript/package.json

@@ -27,7 +27,7 @@
     "files": [
       {
         "path": "packages/algoliasearch/dist/algoliasearch.umd.js",
-        "maxSize": "22KB"
+        "maxSize": "22.25KB"
       },
       {
         "path": "packages/algoliasearch/dist/lite/builds/browser.umd.js",

specs/abtesting-v3/common/parameters.yml

@@ -7,6 +7,22 @@ ID:
   schema:
     $ref: '#/abTestID'
 
+Methods:
+  in: query
+  name: methods
+  description: |
+    Statistical analysis results to include, as a comma-separated list.
+    When omitted, each test uses its configured method, or `frequentist` if no method is configured.
+    Request both methods to include both sets of available results. This doesn't change the test configuration or compute missing results.
+    Duplicate values aren't allowed.
+  style: form
+  explode: false
+  schema:
+    type: array
+    minItems: 1
+    items:
+      $ref: 'schemas/ABTest.yml#/AnalysisMethod'
+
 # misc
 index:
   type: string

specs/abtesting-v3/common/schemas/ABTest.yml

@@ -88,10 +88,14 @@ ABTestConfiguration:
       $ref: '#/MetricsFilters'
     errorCorrection:
       $ref: '#/ErrorCorrectionType'
+    method:
+      $ref: '#/AnalysisMethod'
+    primaryMetric:
+      $ref: '#/PrimaryMetric'
 
 MinimumDetectableEffect:
   type: object
-  description: Configuration for the smallest difference between test variants you want to detect.
+  description: Configuration for the smallest difference between test variants you want to detect, used to estimate the required sample size.
   properties:
     size:
       type: number
@@ -117,6 +121,20 @@ EffectMetric:
     - purchaseRate
     - noResultsRate
 
+PrimaryMetric:
+  type: string
+  description: |
+    Primary metric for Bayesian analysis. Required when `method` is `bayesian`.
+    If the request includes a non-empty `metrics` list, this metric must be in that list.
+    Revenue per search requires access to revenue analytics.
+  example: conversion_rate
+  enum:
+    - add_to_cart_rate
+    - click_through_rate
+    - conversion_rate
+    - purchase_rate
+    - revenue_per_search
+
 MetricsFilter:
   type: object
   additionalProperties: false
@@ -154,3 +172,12 @@ ErrorCorrectionType:
   enum:
     - bonferroni
     - benjamini-hochberg
+
+AnalysisMethod:
+  type: string
+  description: |
+    A/B test statistical analysis method.
+    When omitted, the test is treated as `frequentist`. The server doesn't write a default value back to the configuration.
+  enum:
+    - bayesian
+    - frequentist

specs/abtesting-v3/common/schemas/ABTestSettings.yml

@@ -6,7 +6,7 @@ FeatureSettings:
     name:
       type: string
       description: Name of the feature the settings belong to.
-      example: 'reranking'
+      example: 'drr'
     settings:
       description: Feature settings payload. The shape depends on the feature.
   required:

specs/abtesting-v3/common/schemas/Timeseries.yml

@@ -49,6 +49,7 @@ MetricName:
     - user_count
     - tracked_user_count
     - no_result_count
+    - tracked_no_result_count
     - add_to_cart_count
     - purchase_count
     - clicked_search_count
@@ -59,3 +60,5 @@ MetricName:
     - purchase_rate
     - average_click_position
     - revenue
+    - revenue_per_search
+    - average_order_value

specs/abtesting-v3/common/schemas/Variant.yml

@@ -53,12 +53,14 @@ metricResult:
   properties:
     name:
       type: string
+      description: Metric name. Revenue per search results use `revenue_per_search`.
     updatedAt:
       type: string
       description: Date and time when the metric was last updated, in RFC 3339 format.
     value:
       type: number
       format: double
+      description: Metric value. For `revenue_per_search`, this is the winsorized mean revenue per search in the specified currency.
     valueCIHigh:
       type: number
       format: double
@@ -83,7 +85,7 @@ metricResult:
         Omitted when no p-value is available for this metric.
     dimension:
       type: string
-      description: Dimension defined during test creation.
+      description: Dimension defined during test creation. For revenue metrics, including `revenue_per_search`, this is the currency.
     metadata:
       $ref: '#/metricMetadata'
     criticalValue:
@@ -97,6 +99,8 @@ metricResult:
       type: boolean
       description: |
         Whether the pValue is significant or not based on the critical value and the error correction algorithm used.
+    bayesian:
+      $ref: '#/bayesianMetricResult'
   required:
     - name
     - updatedAt
@@ -109,6 +113,7 @@ metricResult:
       updatedAt: 2025-05-15T17:52:15.644906Z
       value: 0.20869847452125934
       pValue: 0.004
+      bayesian: {'probabilityToBeBetter': 0.991, 'evidence': {'status': 'enough'}}
     - name: revenue
       dimension: USD
       updatedAt: 2025-05-15T17:52:15.644906Z
@@ -121,6 +126,71 @@ metricResult:
       value: 999.66
       pValue: 0.04
       metadata: {'winsorizedValue': 888.8}
+    - name: revenue_per_search
+      dimension: USD
+      updatedAt: 2025-05-15T17:52:15.644906Z
+      value: 12.45
+      metadata: {'mean': 12.45}
+      bayesian:
+        probabilityToBeBetter: 0.991
+        relativeEffectCILow: 0.04
+        relativeEffectCIHigh: 0.39
+        evidence: {'status': 'unavailable'}
+
+bayesianMetricResult:
+  type: object
+  description: |
+    Bayesian inference results for this variant metric.
+    Omitted when Bayesian results aren't requested or no Bayesian result is available for this metric.
+    Individual inference fields can be omitted when their values aren't available.
+  properties:
+    probabilityToBeBetter:
+      type: number
+      format: double
+      description: Probability that this variant is better than the control.
+      example: 0.991
+    relativeEffectCILow:
+      type: number
+      format: double
+      description: Lower bound of the 95% credible interval for the relative effect (variant/control minus 1).
+      example: 0.04
+    relativeEffectCIHigh:
+      type: number
+      format: double
+      description: Upper bound of the 95% credible interval for the relative effect (variant/control minus 1).
+      example: 0.39
+    evidence:
+      $ref: '#/metricEvidence'
+
+metricEvidence:
+  type: object
+  description: |
+    Evidence status for this variant metric's comparison with the control.
+    Only present when the API has an evidence status for the comparison. Omitted when evidence isn't applicable or no status exists.
+  properties:
+    status:
+      $ref: '#/evidenceStatus'
+  required:
+    - status
+
+evidenceStatus:
+  type: string
+  description: |
+    Whether this variant metric's result is trustworthy yet.
+    This doesn't mean the A/B test has ended, and it doesn't declare a winning variant.
+
+    - `enough`. Enough evidence has accumulated to trust this metric's result for this variant.
+    - `not_enough`. Evidence was computed but hasn't reached the threshold yet.
+    - `collecting`. Evidence is still accumulating, so it's too early to report a verdict.
+    - `no_data`. No usable data for this variant metric, and none is expected for it.
+    - `unavailable`. Evidence can't be computed for this variant metric's comparison.
+  example: not_enough
+  enum:
+    - enough
+    - not_enough
+    - collecting
+    - no_data
+    - unavailable
 
 variantMetadata:
   type: object
@@ -137,7 +207,7 @@ metricMetadata:
       type: number
       format: double
       description: |
-        Only present in case the metric is 'revenue'.
+        Only present for `revenue` metrics.
         It is the amount exceeding the 95th percentile of global revenue transactions involved in the AB Test. This amount is not considered when calculating statistical significance.
         It is tied to a per revenue-currency pair contrary to other
         global filter effects (such as outliers and empty search count).

specs/abtesting-v3/paths/abtest.yml

@@ -8,6 +8,7 @@ get:
   description: Retrieves the details for an A/B test by its ID.
   parameters:
     - $ref: '../common/parameters.yml#/ID'
+    - $ref: '../common/parameters.yml#/Methods'
   responses:
     '200':
       description: OK
@@ -30,6 +31,8 @@ get:
       $ref: '../../common/responses/MethodNotAllowed.yml'
     '404':
       $ref: '../../common/responses/IndexNotFound.yml'
+    '422':
+      $ref: '../../common/responses/UnprocessableEntity.yml'
 
 delete:
   tags:

specs/abtesting-v3/paths/abtests.yml

@@ -61,6 +61,8 @@ post:
       $ref: '../../common/responses/MethodNotAllowed.yml'
     '404':
       $ref: '../../common/responses/IndexNotFound.yml'
+    '422':
+      $ref: '../../common/responses/UnprocessableEntity.yml'
 
 get:
   tags:
@@ -106,6 +108,7 @@ get:
       example: desc
       schema:
         $ref: '../common/parameters.yml#/direction'
+    - $ref: '../common/parameters.yml#/Methods'
   responses:
     '200':
       description: OK
@@ -145,3 +148,5 @@ get:
       $ref: '../../common/responses/MethodNotAllowed.yml'
     '404':
       $ref: '../../common/responses/IndexNotFound.yml'
+    '422':
+      $ref: '../../common/responses/UnprocessableEntity.yml'

specs/abtesting-v3/paths/applyVariantSettings.yml

@@ -36,6 +36,10 @@ post:
     '403':
       $ref: '../../common/responses/MethodNotAllowed.yml'
     '404':
-      $ref: '../../common/responses/IndexNotFound.yml'
+      description: A/B test, variant, or index not found.
+      content:
+        application/json:
+          schema:
+            $ref: '../../common/schemas/ErrorBase.yml'
     '422':
       $ref: '../../common/responses/UnprocessableEntity.yml'

specs/abtesting-v3/paths/timeseries.yml

@@ -17,6 +17,7 @@ get:
         type: array
         items:
           $ref: '../common/schemas/Timeseries.yml#/MetricName'
+    - $ref: '../common/parameters.yml#/Methods'
   responses:
     '200':
       description: OK

specs/abtesting-v3/paths/variantSettings.yml

@@ -42,7 +42,11 @@ post:
     '403':
       $ref: '../../common/responses/MethodNotAllowed.yml'
     '404':
-      $ref: '../../common/responses/IndexNotFound.yml'
+      description: A/B test, variant, or index not found.
+      content:
+        application/json:
+          schema:
+            $ref: '../../common/schemas/ErrorBase.yml'
     '409':
       $ref: '../../common/responses/Conflict.yml'
     '422':

tests/CTS/requests/abtesting-v3/addABTests.json

@@ -35,5 +35,50 @@
         ]
       }
     }
+  },
+  {
+    "testName": "addABTests with Bayesian configuration",
+    "parameters": {
+      "endAt": "2022-12-31T00:00:00.000Z",
+      "name": "myABTest",
+      "metrics": [{"name": "conversionRate"}],
+      "variants": [
+        {
+          "index": "AB_TEST_1",
+          "trafficPercentage": 30
+        },
+        {
+          "index": "AB_TEST_2",
+          "trafficPercentage": 50
+        }
+      ],
+      "configuration": {
+        "method": "bayesian",
+        "primaryMetric": "conversion_rate"
+      }
+    },
+    "request": {
+      "path": "/3/abtests",
+      "method": "POST",
+      "body": {
+        "endAt": "2022-12-31T00:00:00.000Z",
+        "name": "myABTest",
+        "metrics": [{"name": "conversionRate"}],
+        "variants": [
+          {
+            "index": "AB_TEST_1",
+            "trafficPercentage": 30
+          },
+          {
+            "index": "AB_TEST_2",
+            "trafficPercentage": 50
+          }
+        ],
+        "configuration": {
+          "method": "bayesian",
+          "primaryMetric": "conversion_rate"
+        }
+      }
+    }
   }
 ]

tests/CTS/requests/abtesting-v3/getABTest.json

@@ -8,5 +8,14 @@
       "path": "/3/abtests/42",
       "method": "GET"
     }
+  },
+  {
+    "testName": "getABTest with both inference methods",
+    "parameters": { "id": 42, "methods": ["frequentist", "bayesian"] },
+    "request": {
+      "path": "/3/abtests/42",
+      "method": "GET",
+      "queryParameters": { "methods": "frequentist%2Cbayesian" }
+    }
   }
 ]

tests/CTS/requests/abtesting-v3/getTimeseries.json

@@ -7,5 +7,25 @@
       "path": "/3/abtests/42/timeseries",
       "method": "GET"
     }
+  },
+  {
+    "testName": "getTimeseries with Bayesian revenue per search",
+    "parameters": {
+      "id": 42,
+      "startDate": "1999-09-19",
+      "endDate": "2001-01-01",
+      "metric": ["revenue_per_search"],
+      "methods": ["bayesian"]
+    },
+    "request": {
+      "path": "/3/abtests/42/timeseries",
+      "method": "GET",
+      "queryParameters": {
+        "startDate": "1999-09-19",
+        "endDate": "2001-01-01",
+        "metric": "revenue_per_search",
+        "methods": "bayesian"
+      }
+    }
   }
 ]