Merge branch 'main' into fix-property-reflection-values

.bcr/metadata.template.json

@@ -22,9 +22,9 @@
       "name": "Jerry Berg"
     },
     {
-      "email": "acozette@google.com",
-      "github": "acozette",
-      "name": "Adam Cozette",
+      "email": "acozzette@google.com",
+      "github": "acozzette",
+      "name": "Adam Cozzette",
       "do_not_notify": true
     },
     {

.bcr/presubmit.yml

@@ -8,8 +8,8 @@ tasks:
     platform: ${{ platform }}
     bazel: ${{ bazel }}
     build_flags:
-    - '--host_cxxopt=-std=c++14'
-    - '--cxxopt=-std=c++14'
+    - '--host_cxxopt=-std=c++17'
+    - '--cxxopt=-std=c++17'
     build_targets:
     - '@protobuf//:protobuf'
     - '@protobuf//:protobuf_lite'
@@ -30,7 +30,7 @@ bcr_test_module:
       platform: ${{ platform }}
       bazel: ${{ bazel }}
       build_flags:
-      - '--host_cxxopt=-std=c++14'
-      - '--cxxopt=-std=c++14'
+      - '--host_cxxopt=-std=c++17'
+      - '--cxxopt=-std=c++17'
       build_targets:
       - "//..."

.github/scripts/validate_yaml.py

@@ -58,16 +58,18 @@
         elif 'with' in step and 'name' in step['with']:
           name = step['with']['name']
         else:
-          raise ValueError('Step in job %s does not have a name.' % job)
+          raise ValueError(
+              'Step in job %s from file %s does not have a name.' % (job, file)
+          )
         if continuous_condition and 'continuous-run' not in step.get('if', ''):
           raise ValueError(
-              'Step %s in job %s does not check the continuous-run condition'
-              % (name, job)
+              'Step %s in job %s from file %s does not check the continuous-run'
+              ' condition' % (name, job, file)
           )
         if not continuous_condition and 'continuous-run' in step.get('if', ''):
           raise ValueError(
-              'Step %s in job %s checks the continuous-run condition but '
-              'the job does not contain the continuous-prefix'
-              % (name, job)
+              'Step %s in job %s from file %s checks the continuous-run'
+              ' condition but the job does not contain the continuous-prefix'
+              % (name, job, file)
           )
 print('PASSED: All steps in all jobs check the continuous-run condition.')

.github/workflows/test_cpp.yml

@@ -141,17 +141,19 @@ jobs:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         include:
-          - flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=17  -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_FETCH_DEPENDENCIES=OFF
+          - flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=17
           - name: Ninja
-            flags: -G Ninja -DCMAKE_CXX_STANDARD=17 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_FETCH_DEPENDENCIES=OFF
+            flags: -G Ninja -DCMAKE_CXX_STANDARD=17
             continuous-only: true
           - name: Shared
-            flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=17 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_FETCH_DEPENDENCIES=OFF
+            flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=17
             continuous-only: true
           - name: C++20
-            flags: -DCMAKE_CXX_STANDARD=20 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_FETCH_DEPENDENCIES=OFF
+            flags: -DCMAKE_CXX_STANDARD=20
+          - name: Package
+            flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_LOCAL_DEPENDENCIES_ONLY=ON
           - name: Fetch
-            flags: -DCMAKE_CXX_STANDARD=17
+            flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_FORCE_FETCH_DEPENDENCIES=ON
 
     name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux CMake ${{ matrix.name}}
     runs-on: ubuntu-latest
@@ -188,9 +190,10 @@ jobs:
           # Set defaults
           - type: package
             name: Install
-            flags: -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_FETCH_DEPENDENCIES=OFF
+            flags: -Dprotobuf_LOCAL_DEPENDENCIES_ONLY=ON
           - type: fetch
             name: Install (Fetch)
+            flags: -Dprotobuf_FORCE_FETCH_DEPENDENCIES=ON
             continuous-only: true
     name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }}Linux CMake ${{ matrix.name }}
     runs-on: ubuntu-latest
@@ -252,7 +255,7 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
             /install.sh -DCMAKE_CXX_STANDARD=17 ${{ env.SCCACHE_CMAKE_FLAGS }}
-            -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_FETCH_DEPENDENCIES=OFF
+            -Dprotobuf_LOCAL_DEPENDENCIES_ONLY=OFF
             -Dprotobuf_BUILD_EXAMPLES=OFF \&\&
             mkdir examples/build \&\&
             cd examples/build \&\&
@@ -320,7 +323,7 @@ jobs:
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/docker@v3
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:56548bef786201330017eae685cc3d2fdb564fd2ca3b88e30e28d84572e4c5dd
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:d6028ab408c49932836cdc514116f06886d7f6868a4d430630aa52adc5aee2fc
           platform: linux/386
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
@@ -426,8 +429,7 @@ jobs:
           - name: Windows CMake Install
             os: windows-2022
             install-flags: >-
-              -G Ninja -Dprotobuf_WITH_ZLIB=OFF
-              -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF
+              -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
             flags: >-
               -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
               -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON

.github/workflows/test_python.yml

@@ -28,7 +28,7 @@ jobs:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         type: [ Pure, C++]
-        version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
@@ -42,12 +42,14 @@ jobs:
             # TODO Enable this once conformance tests are fixed.
             flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17
-          - version: "3.8"
           - version: "3.9"
-            continuous-only: true
           - version: "3.10"
             continuous-only: true
           - version: "3.11"
+            continuous-only: true
+          - version: "3.12"
+            continuous-only: true
+          - version: "3.13"
 
     name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux ${{ matrix.type }} ${{ matrix.version }}
     runs-on: ubuntu-latest
@@ -61,7 +63,7 @@ jobs:
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         uses: protocolbuffers/protobuf-ci/bazel-docker@v3
         with:
-          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:6.4.0-{0}-27cf7b86212020d7e552bc13b1e084abb971da75', matrix.version) }}
+          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:7.1.2-{0}-d9624f2aa83cba3eaf906f751d75b36aacb9aa82', matrix.version) }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
           bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION
@@ -73,14 +75,15 @@ jobs:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         type: [ Pure, C++]
-        # TODO Consider expanding this set of versions.
-        version: [ "3.12" ]
+        version: [ "3.12", "3.13" ]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
           - type: C++
             targets: //python/... //python:python_version_test
             flags: --define=use_fast_cpp_protos=true
+          - version: "3.13"
+            continuous-only: true
 
     name: MacOS ${{ matrix.type }} ${{ matrix.version }}
     runs-on: macos-13
@@ -91,7 +94,7 @@ jobs:
           ref: ${{ inputs.safe-checkout }}
 
       - name: Pin Python version
-        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f #v5.1.1
         with:
           python-version: ${{ matrix.version }}
           cache: pip

.github/workflows/test_runner.yml

@@ -48,7 +48,7 @@ permissions:
 
 concurrency:
   group: ${{ github.event_name }}-${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: ${{ contains(fromJSON('["pull_request", "pull_request_target", "workflow_dispatch"]'), github.event_name) }}
+  cancel-in-progress: ${{ contains(fromJSON('["pull_request", "pull_request_target", "workflow_dispatch", "schedule"]'), github.event_name) }}
 
 jobs:
   set-vars:

.github/workflows/test_rust.yml

@@ -14,7 +14,27 @@ permissions:
 jobs:
   # This job should be run on presubmit, if any continuous-only tests are added we will need to input test-type above
   linux:
-    name: Linux
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        config:
+          - { name: "No bzlmod", flags: --noenable_bzlmod }
+          # TODO: b/379846319 - Fix the tests with bzlmod and enable this.
+          # - { name: "bzlmod", flags: --enable_bzlmod --enable_workspace }
+          - { name: Optimized, flags: --config=opt }
+          - { name: ASAN, flags: --config=asan }
+
+        include:
+          - targets: "//rust/... //src/google/protobuf/compiler/rust/..."
+          - image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.1-97f82260fd504923d8af642d567afb2d83a1959d"
+          - bazel_cmd: "test"
+
+          # Override cases with custom images
+          - config: { name: Cargo }
+            image: "us-docker.pkg.dev/protobuf-build/containers/release/linux/rust:6.3.0-1.74.0-8858126dd9480abf91e6ce8d6e41a5cd3c03882c"
+            bazel_cmd: "run"
+            targets: "//rust:cargo_test"
+    name: Linux ${{ matrix.config.name }}
     runs-on: ubuntu-22-4core
     steps:
       - name: Checkout pending changes
@@ -24,20 +44,10 @@ jobs:
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/bazel-docker@v3
         with:
-          image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.1-97f82260fd504923d8af642d567afb2d83a1959d"
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          bazel-cache: rust_linux
-          bazel: >-
-            test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --@rules_rust//rust/settings:experimental_use_cc_common_link=True
-            //rust:all
-            //rust/test/cpp/interop/...
-            //rust/test/rust_proto_library_unit_test:rust_upb_aspect_test
-            //src/google/protobuf/compiler/rust/...
-      - name: Run Cargo tests
-        uses: protocolbuffers/protobuf-ci/bazel-docker@v3
-        with:
-          image: "us-docker.pkg.dev/protobuf-build/containers/release/linux/rust:6.3.0-1.74.0-8858126dd9480abf91e6ce8d6e41a5cd3c03882c"
+          image: ${{ matrix.image }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: rust_linux
           bazel: >-
-            run --crosstool_top=//toolchain:clang_suite --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //rust:cargo_test
+            ${{ matrix.bazel_cmd }} --crosstool_top=//toolchain:clang_suite  --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --@rules_rust//rust/settings:experimental_use_cc_common_link=True
+            ${{ matrix.targets }} ${{ matrix.config.flags }}
+

.github/workflows/test_upb.yml

@@ -87,7 +87,7 @@ jobs:
         uses: protocolbuffers/protobuf-ci/checkout@v3
         with:
           ref: ${{ inputs.safe-checkout }}
-      - name: "Setup Python"
+      - name: Setup Python
         uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           cache: pip
@@ -115,9 +115,10 @@ jobs:
         uses: protocolbuffers/protobuf-ci/checkout@v3
         with:
           ref: ${{ inputs.safe-checkout }}
-      - name: "Setup Python"
+      - name: Setup Python
         uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
+          python-version: 3.12
           cache: pip
           cache-dependency-path: 'python/requirements.txt'
       - name: Run tests
@@ -187,26 +188,26 @@ jobs:
           # a single wheel. As a result we can just test the oldest and newest
           # supported Python versions and assume this gives us sufficient test
           # coverage.
-          - { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'binary' }
-          - { os: macos-13, python-version: "3.8", architecture: x64, type: 'binary' }
+          - { os: ubuntu-latest, python-version: "3.9", architecture: x64, type: 'binary' }
+          - { os: macos-13, python-version: "3.9", architecture: x64, type: 'binary' }
           - { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'binary' }
           - { os: macos-13, python-version: "3.12", architecture: x64, type: 'binary' }
-          - { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'source', continuous-only: true }
-          - { os: macos-13, python-version: "3.8", architecture: x64, type: 'source', continuous-only: true }
           - { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'source', continuous-only: true }
           - { os: macos-13, python-version: "3.12", architecture: x64, type: 'source', continuous-only: true }
+          - { os: ubuntu-latest, python-version: "3.13", architecture: x64, type: 'source', continuous-only: true }
+          - { os: macos-13, python-version: "3.13", architecture: x64, type: 'source', continuous-only: true }
 
           # Windows uses the full API up until Python 3.10.
-          - { os: windows-2019, python-version: "3.8", architecture: x86, type: 'binary', continuous-only: true }
           - { os: windows-2019, python-version: "3.9", architecture: x86, type: 'binary', continuous-only: true }
           - { os: windows-2019, python-version: "3.10", architecture: x86, type: 'binary', continuous-only: true }
           - { os: windows-2019, python-version: "3.11", architecture: x86, type: 'binary', continuous-only: true }
           - { os: windows-2019, python-version: "3.12", architecture: x86, type: 'binary', continuous-only: true }
-          - { os: windows-2019, python-version: "3.8", architecture: x64, type: 'binary' }
-          - { os: windows-2019, python-version: "3.9", architecture: x64, type: 'binary', continuous-only: true }
+          - { os: windows-2019, python-version: "3.13", architecture: x86, type: 'binary', continuous-only: true }
+          - { os: windows-2019, python-version: "3.9", architecture: x64, type: 'binary' }
           - { os: windows-2019, python-version: "3.10", architecture: x64, type: 'binary', continuous-only: true }
           - { os: windows-2019, python-version: "3.11", architecture: x64, type: 'binary', continuous-only: true }
-          - { os: windows-2019, python-version: "3.12", architecture: x64, type: 'binary' }
+          - { os: windows-2019, python-version: "3.12", architecture: x64, type: 'binary', continuous-only: true }
+          - { os: windows-2019, python-version: "3.13", architecture: x64, type: 'binary' }
     name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Test Wheels Python ${{ matrix.python-version }} ${{ matrix.os }} ${{ matrix.architecture }} ${{ matrix.type }}
     needs: build_wheels
     runs-on: ${{ matrix.os }}
@@ -278,7 +279,7 @@ jobs:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
-        python-version: ["3.8", "3.12"]
+        python-version: ["3.9", "3.13"]
     runs-on: ubuntu-latest
     if: ${{ github.event_name != 'pull_request_target' }}
     steps:
@@ -289,7 +290,7 @@ jobs:
           path: wheels
       - name: Delete Binary Wheels
         run: find wheels -type f | grep -v none-any | xargs rm
-      - name: "Setup Python"
+      - name: Setup Python
         uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: ${{ matrix.python-version }}

CMakeLists.txt

@@ -35,6 +35,8 @@ option(protobuf_BUILD_LIBUPB "Build libupb" ON)
 option(protobuf_DISABLE_RTTI "Remove runtime type information in the binaries" OFF)
 option(protobuf_TEST_XML_OUTDIR "Output directory for XML logs from tests." "")
 option(protobuf_ALLOW_CCACHE "Adjust build flags to allow for ccache support." OFF)
+option(protobuf_FORCE_FETCH_DEPENDENCIES "Force all dependencies to be downloaded from GitHub.  Local installations will be ignored." OFF)
+option(protobuf_LOCAL_DEPENDENCIES_ONLY "Prevent downloading any dependencies from GitHub. If this option is set, the dependency must be available locally as an installed package." OFF)
 
 # We support Unity (Jumbo) builds best-effort.
 option(protobuf_USE_UNITY_BUILD "Enable Unity (Jumbo) build for" OFF)
@@ -106,18 +108,21 @@ string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\3"
 string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\5"
   protobuf_VERSION_PRERELEASE "${protobuf_VERSION_STRING}")
 
-message(STATUS "${protobuf_VERSION_PRERELEASE}")
+if (protobuf_FORCE_FETCH_DEPENDENCIES AND protobuf_LOCAL_DEPENDENCIES_ONLY)
+  message(FATAL_ERROR "Conflicting options protobuf_FORCE_FETCH_DEPENDENCIES and protobuf_LOCAL_DEPENDENCIES_ONLY both set")
+endif()
 
 # Package version
 set(protobuf_VERSION
   "${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
 
 if(protobuf_VERSION_PRERELEASE)
+  message(STATUS "${protobuf_VERSION_PRERELEASE}")
   set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}")
 else()
   set(protobuf_VERSION "${protobuf_VERSION}.0")
 endif()
-message(STATUS "${protobuf_VERSION}")
+message(STATUS "protobuf version: ${protobuf_VERSION}")
 
 if(protobuf_VERBOSE)
   message(STATUS "Configuration script parsing status [")
@@ -261,14 +266,6 @@ include_directories(
   ${protobuf_BINARY_DIR}/src
   ${protobuf_SOURCE_DIR}/src)
 
-set(protobuf_FETCH_DEPENDENCIES ON CACHE BOOL "Allow downloading dependencies from GitHub. If this option is not set, the dependency must be available locally as an installed package.")
-
-set(protobuf_ABSL_PROVIDER "fetch" CACHE STRING "Provider of absl library.  `fetch` downloads from GitHub and `package` searches for a local installation")
-set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "package" "fetch")
-
-set(protobuf_JSONCPP_PROVIDER "fetch" CACHE STRING "Provider of jsoncpp library.  `fetch` downloads from GitHub and `package` searches for a local installation")
-set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "package" "fetch")
-
 if (protobuf_BUILD_TESTS)
   include(${protobuf_SOURCE_DIR}/cmake/gtest.cmake)
 endif (protobuf_BUILD_TESTS)

MODULE.bazel

@@ -15,9 +15,9 @@ module(
 bazel_dep(name = "abseil-cpp", version = "20240722.0", repo_name = "com_google_absl")
 bazel_dep(name = "bazel_skylib", version = "1.7.0")
 bazel_dep(name = "jsoncpp", version = "1.9.6")
-bazel_dep(name = "rules_cc", version = "0.0.13")
+bazel_dep(name = "rules_cc", version = "0.0.16")
 bazel_dep(name = "rules_fuzzing", version = "0.5.2")
-bazel_dep(name = "rules_java", version = "7.12.2")
+bazel_dep(name = "rules_java", version = "8.3.2")
 bazel_dep(name = "rules_jvm_external", version = "6.3")
 bazel_dep(name = "rules_kotlin", version = "1.9.6")
 bazel_dep(name = "rules_license", version = "1.0.0")
@@ -36,7 +36,6 @@ bazel_dep(
 register_toolchains("//bazel/private/toolchains:all")
 
 SUPPORTED_PYTHON_VERSIONS = [
-    "3.8",
     "3.9",
     "3.10",
     "3.11",