Extend gencode compatibility support back to 3.20.0 (#22424)

* Extend gencode compatibility support back to 3.20.0

This adds 2 shims for some previous breakages, and compatibility tests to avoid further accidents.  This will be backported to 25.x and 29.x.

PiperOrigin-RevId: 776414924

* Remove 26.x and 30.x

* Remove continuous testing tags which didn't exist in 25.x

* Fix requirements install

* pin to bazel 6.3.0

* Desynchronize upb/pure tests

* Remove missing test proto from hardcoded list

* Use python 11 for 25.x

* Run tests independently to avoid contamination of global state between generator/descriptor tests

* Update comment

Author: mkruskal-google

.github/workflows/test_upb.yml

@@ -277,3 +277,56 @@ jobs:
           for test in $TESTS; do
             python -m unittest -v $test
           done
+
+  test_python_compatibility:
+    name: Test Python Compatibility (${{ matrix.implementation }} - ${{ matrix.version }})
+    needs: build_wheels
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        # Test the first release of each python major version, plus some extra coverage for 3.x.y
+        # since it predates our breaking change policies.
+        # Major versions: 4.21.0
+        version: ["3.18.0", "3.20.0", "21.0"]
+        implementation: ["Pure", "upb"]
+        include:
+          # We don't support 3.18.0 anymore, so our infra should show a failure.
+          - version: "3.18.0"
+            fail: true
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request_target' }}
+    steps:
+      - name: Checkout pending changes
+        uses: protocolbuffers/protobuf-ci/checkout@v4
+        with:
+          ref: ${{ inputs.safe-checkout }}
+      - name: Download Wheels
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #4.1.8
+        with:
+          name: python-wheels
+          path: wheels
+      - name: Delete Binary Wheels (pure python only)
+        if: ${{ matrix.implementation == 'Pure' }}
+        run: find wheels -type f | grep -v none-any | xargs rm
+      - name: Setup Python
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+        with:
+          python-version: 3.11
+      - name: Setup Python venv
+        run: |
+          python -m pip install --upgrade pip
+          python -m venv env
+          source env/bin/activate
+          echo "VIRTUAL ENV:" $VIRTUAL_ENV
+      - name: Install numpy
+        run: pip install numpy
+      - name: Install Protobuf Wheels
+        run: pip install -vvv --no-index --find-links wheels protobuf protobuftests
+      - name: Make the test script executable
+        run: chmod +x ci/python_compatibility.sh
+      - name: Run compatibility tests
+        if: ${{ !matrix.fail }}
+        run: ci/python_compatibility.sh ${{ matrix.version }} ${{ matrix.implementation == 'upb' && 'pb_unit_tests.*py$' || '_test.py' }}
+      - name: Run failing compatibility tests
+        if: ${{ matrix.fail }}
+        run: (! ci/python_compatibility.sh ${{ matrix.version }} ${{ matrix.implementation == 'upb' && 'pb_unit_tests.*py$' || '_test.py' }})

ci/python_compatibility.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -ex
+
+PROTOC_VERSION=$1
+TEST_FILTER=$2
+PROTOC_DOWNLOAD=https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-linux-x86_64.zip
+PY_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])')
+
+rm -rf protoc-old
+mkdir protoc-old
+pushd protoc-old
+wget $PROTOC_DOWNLOAD
+unzip *.zip
+PROTOC=$(pwd)/bin/protoc
+popd
+
+# protoc prior to 28.0 doesn't support inf/nan option values.
+sed -i 's/\(inf\|nan\)/0/g' src/google/protobuf/unittest_custom_options.proto
+
+USE_BAZEL_VERSION=6.3.0 bazelisk build //python:copied_test_proto_files //python:copied_wkt_proto_files
+
+COMPAT_COPIED_PROTOS=(
+  # Well-known types give good build coverage
+  any
+  api
+  duration
+  empty
+  field_mask
+  source_context
+  struct
+  timestamp
+  type
+  wrappers
+  # These protos are used in tests of custom options handling.
+  unittest_custom_options
+  unittest_import
+)
+
+for proto in ${COMPAT_COPIED_PROTOS[@]}; do
+  $PROTOC --python_out=$PY_SITE_PACKAGES \
+    bazel-bin/python/google/protobuf/$proto.proto \
+    -Ibazel-bin/python
+done
+
+COMPAT_PROTOS=(
+  # All protos without transitive dependencies on edition 2023+.
+  descriptor_pool_test1
+  descriptor_pool_test2
+  factory_test1
+  factory_test2
+  file_options_test
+  import_test_package/import_public
+  import_test_package/import_public_nested
+  import_test_package/inner
+  import_test_package/outer
+  message_set_extensions
+  missing_enum_values
+  more_extensions
+  more_extensions_dynamic
+  more_messages
+  no_package
+  packed_field_test
+  test_bad_identifiers
+  test_proto3_optional
+)
+
+for proto in ${COMPAT_PROTOS[@]}; do
+  $PROTOC --python_out=$PY_SITE_PACKAGES \
+    python/google/protobuf/internal/$proto.proto \
+    -Ipython
+done
+
+# Exclude pybind11 tests because they require C++ code that doesn't ship with
+# our test wheels.
+TEST_EXCLUSIONS="_pybind11_test.py"
+TESTS=$(pip show -f protobuftests | grep -i $TEST_FILTER | grep --invert-match $TEST_EXCLUSIONS | sed 's,/,.,g' | sed -E 's,.py$,,g')
+for test in $TESTS; do
+  python -m unittest -v $test
+done