Merge branch 'master' into srest2021/ENG-6194-frontend

Author: srest2021

.claude/settings.json

@@ -9,6 +9,7 @@
       "Bash(grep:*)",
       "Bash(pnpm run test:*)",
       "Bash(pytest:*)",
+      "Bash(mypy:*)",
       "Bash(pre-commit run:*)",
       "Bash(gh pr checks:*)",
       "Bash(gh pr view:*)",

.github/actions/setup-sentry/action.yml

@@ -15,6 +15,10 @@ inputs:
     description: 'Mode to bring up by new devservices'
     required: false
     default: 'default'
+  skip-devservices:
+    description: 'Skip bringing up devservices'
+    required: false
+    default: 'false'
 
 outputs:
   matrix-instance-number:
@@ -38,7 +42,7 @@ runs:
         # isn't right because job-total will be 3x larger and you'd never run 2/3 of the tests.
         # MATRIX_INSTANCE_TOTAL: ${{ strategy.job-total }}
       run: |
-        echo "SENTRY_SKIP_BACKEND_VALIDATION=1" >> $GITHUB_ENV
+        echo "SENTRY_SKIP_SERVICE_VALIDATION=1" >> $GITHUB_ENV
 
         # for actions/setup-python which always runs pip install
         echo "PIP_DISABLE_PIP_VERSION_CHECK=on" >> $GITHUB_ENV
@@ -122,6 +126,7 @@ runs:
         python3 -m tools.fast_editable --path .
 
     - name: Start new devservices
+      if: ${{ inputs.skip-devservices != 'true' }}
       shell: bash --noprofile --norc -eo pipefail -ux {0}
       env:
         WORKDIR: ${{ inputs.workdir }}

.github/workflows/backend.yml

@@ -73,9 +73,34 @@ jobs:
             devservices logs
           fi
 
-  backend-test:
+  calculate-shards:
     if: needs.files-changed.outputs.backend == 'true'
     needs: files-changed
+    name: calculate test shards
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    outputs:
+      shard-count: ${{ steps.calculate-shards.outputs.shard-count }}
+      shard-indices: ${{ steps.calculate-shards.outputs.shard-indices }}
+
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Setup sentry env
+        uses: ./.github/actions/setup-sentry
+        id: setup
+        with:
+          mode: backend-ci
+          skip-devservices: true
+
+      - name: Calculate test shards
+        id: calculate-shards
+        run: |
+          python3 .github/workflows/scripts/calculate-backend-test-shards.py
+
+  backend-test:
+    if: needs.files-changed.outputs.backend == 'true'
+    needs: [files-changed, calculate-shards]
     name: backend test
     runs-on: ubuntu-24.04
     timeout-minutes: 60
@@ -88,14 +113,12 @@ jobs:
       # and reducing the risk that one of many runs would turn red again (read: intermittent tests)
       fail-fast: false
       matrix:
-        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
-        instance:
-          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]
+        # Dynamic matrix from calculate-shards
+        instance: ${{ fromJSON(needs.calculate-shards.outputs.shard-indices) }}
 
     env:
-      # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`.
-      # If this increases, make sure to also increase `flags.backend.after_n_builds` in `codecov.yml`.
-      MATRIX_INSTANCE_TOTAL: 22
+      # Dynamic total from calculate-shards
+      MATRIX_INSTANCE_TOTAL: ${{ needs.calculate-shards.outputs.shard-count }}
       TEST_GROUP_STRATEGY: roundrobin
 
     steps:
@@ -406,6 +429,7 @@ jobs:
         api-docs,
         backend-test,
         backend-migration-tests,
+        calculate-shards,
         cli,
         files-changed,
         requirements,

.github/workflows/frontend.yml

@@ -209,8 +209,7 @@ jobs:
           GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }}
           # XXX: CI_NODE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
           #      Otherwise, if there are other things in the matrix, using strategy.job-total
-          #      wouldn't be correct. Also, if this increases, make sure to also increase
-          #      `flags.frontend.after_n_builds` in `codecov.yml`.
+          #      wouldn't be correct.
           CI_NODE_TOTAL: 4
           CI_NODE_INDEX: ${{ matrix.instance }}
           # Disable testing-library from printing out any of of the DOM to
@@ -219,12 +218,7 @@ jobs:
           #
           # This quiets up the logs quite a bit.
           DEBUG_PRINT_LIMIT: 0
-        run: |
-          if [ ${{ github.ref }} = 'refs/heads/master' ]; then
-            pnpm run test-ci --forceExit --coverage
-          else
-            pnpm run test-ci --forceExit
-          fi
+        run: pnpm run test-ci --forceExit
 
   # This check runs once all dependant jobs have passed
   # It symbolizes that all required Frontend checks have succesfully passed (Or skipped)

.github/workflows/scripts/calculate-backend-test-shards.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+import json
+import math
+import os
+import re
+import subprocess
+import sys
+
+TESTS_PER_SHARD = 1200
+MIN_SHARDS = 1
+MAX_SHARDS = 22
+DEFAULT_SHARDS = 22
+
+PYTEST_ARGS = [
+    "pytest",
+    "--collect-only",
+    "--quiet",
+    "tests",
+    "--ignore=tests/acceptance",
+    "--ignore=tests/apidocs",
+    "--ignore=tests/js",
+    "--ignore=tests/tools",
+]
+
+
+def collect_test_count():
+    try:
+        result = subprocess.run(
+            PYTEST_ARGS,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+        # Parse output for "N tests collected"
+        # Format: "27000 tests collected in 18.53s"
+        match = re.search(r"(\d+) tests? collected", result.stdout + result.stderr)
+        if match:
+            count = int(match.group(1))
+            print(f"Collected {count} tests", file=sys.stderr)
+            return count
+
+        # If no match, check if pytest failed
+        if result.returncode != 0:
+            print(
+                f"Pytest collection failed (exit {result.returncode})",
+                file=sys.stderr,
+            )
+            print(result.stderr, file=sys.stderr)
+            return None
+
+        print("No tests collected", file=sys.stderr)
+        return 0
+    except Exception as e:
+        print(f"Error collecting tests: {e}", file=sys.stderr)
+        return None
+
+
+def calculate_shards(test_count):
+    if test_count is None:
+        print(f"Using default shard count: {DEFAULT_SHARDS}", file=sys.stderr)
+        return DEFAULT_SHARDS
+
+    if test_count == 0:
+        print(f"No tests to run, using minimum: {MIN_SHARDS}", file=sys.stderr)
+        return MIN_SHARDS
+
+    calculated = math.ceil(test_count / TESTS_PER_SHARD)
+    bounded = max(MIN_SHARDS, min(calculated, MAX_SHARDS))
+
+    if bounded != calculated:
+        print(
+            f"Calculated {calculated} shards, bounded to {bounded}",
+            file=sys.stderr,
+        )
+    else:
+        print(
+            f"Calculated {bounded} shards ({test_count} tests ÷ {TESTS_PER_SHARD})",
+            file=sys.stderr,
+        )
+
+    return bounded
+
+
+def main():
+    test_count = collect_test_count()
+    shard_count = calculate_shards(test_count)
+    # Generate a JSON array of shard indices [0, 1, 2, ..., shard_count-1]
+    shard_indices = json.dumps(list(range(shard_count)))
+
+    github_output = os.getenv("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a") as f:
+            f.write("\n")
+            f.write(f"shard-count={shard_count}\n")
+            f.write(f"shard-indices={shard_indices}\n")
+
+    print(f"shard-count={shard_count}")
+    print(f"shard-indices={shard_indices}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/shuffle-tests.yml

@@ -40,7 +40,6 @@ jobs:
 
     env:
       # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`.
-      # If this increases, make sure to also increase `flags.backend.after_n_builds` in `codecov.yml`.
       MATRIX_INSTANCE_TOTAL: 11
 
     steps:

build-utils/ts-extract-gettext.ts

@@ -2,6 +2,7 @@ import fs from 'node:fs/promises';
 import path, {resolve} from 'node:path'; // Added for module check
 import {fileURLToPath} from 'node:url';
 
+import {ATTRIBUTE_METADATA} from '@sentry/conventions';
 import {po} from 'gettext-parser';
 import type {GetTextTranslation, GetTextTranslations} from 'gettext-parser';
 import {glob} from 'tinyglobby';
@@ -185,6 +186,11 @@ function extractTranslationsFromFileContent(
                 ? existingEntry.comments.translator + `\n`
                 : '') + translate.comments.translator;
           }
+          // Preserve msgid_plural if the new entry has one
+          if (finalTranslateEntry.msgid_plural && !existingEntry.msgid_plural) {
+            existingEntry.msgid_plural = finalTranslateEntry.msgid_plural;
+            existingEntry.msgstr = finalTranslateEntry.msgstr;
+          }
         } else {
           currentContext[finalTranslateEntry.msgid] = finalTranslateEntry;
         }
@@ -197,6 +203,38 @@ function extractTranslationsFromFileContent(
   visit(sourceFile);
 }
 
+function extractBriefsFromAttributeMetadata(
+  gettextData: GetTextTranslations,
+  nplurals: number
+): void {
+  const context = gettextData.translations[''] ?? {};
+
+  for (const [attributeName, metadata] of Object.entries(ATTRIBUTE_METADATA)) {
+    const brief = metadata.brief;
+
+    if (!brief || typeof brief !== 'string') {
+      continue;
+    }
+
+    // Skip if already exists
+    if (context[brief]) {
+      continue;
+    }
+
+    const translate: GetTextTranslation = {
+      msgid: brief,
+      msgstr: [''],
+      comments: {
+        extracted: `Attribute \`${attributeName}\` metadata description from @sentry/conventions`,
+      },
+    };
+
+    context[brief] = translate;
+  }
+
+  gettextData.translations[''] = context;
+}
+
 const OUTPUT_FILE = 'build/javascript.po';
 const BASE_DIRECTORY = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
 
@@ -248,6 +286,9 @@ async function main() {
 
   await Promise.all(files.map(processFile));
 
+  // Extract briefs from ATTRIBUTE_METADATA
+  extractBriefsFromAttributeMetadata(gettextData, nplurals);
+
   const outputFilePath = path.resolve(BASE_DIRECTORY, OUTPUT_FILE);
   const compiledPoBuffer = po.compile(gettextData, {sort: true});
   await fs.writeFile(outputFilePath, compiledPoBuffer, 'utf8');