Support fetching CMake dependencies via `-Dprotobuf_FETCH_DEPENDENCIE…

…S=ON` This ports upb's WORKSPACE scraping logic to protobuf, and allows us to dynamically fetch our dependencies at the exact same pinned version as in Bazel via protobuf_FETCH_DEPENDENCIES=ON. This is mostly for development purposes, and is preferable to git submodules. In a later cl we will flip the default behavior to "package" #test-continuous PiperOrigin-RevId: 686265348
protocolbuffers · Oct 15, 2024 · d3735bc · d3735bc
1 parent 2b0a414
commit d3735bc
Show file tree

Hide file tree

Showing 14 changed files with 266 additions and 603 deletions.
diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml
@@ -107,7 +107,6 @@ jobs:
         uses: protocolbuffers/protobuf-ci/checkout@v3
         with:
           ref: ${{ inputs.safe-checkout }}
-          submodules: recursive
       - name: Cross compile protoc for ${{ matrix.arch }}
         id: cross-compile
         uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3
@@ -131,7 +130,7 @@ jobs:
             sccache -z;
             cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
             -Dprotobuf_BUILD_LIBUPB=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14
-            -Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }};
+            -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_FETCH_DEPENDENCIES=ON ${{ env.SCCACHE_CMAKE_FLAGS }};
             cmake --build . --parallel 20;
             ctest --parallel 20;
             sccache -s"
@@ -141,18 +140,19 @@ jobs:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         include:
-          - flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14
+          - flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14  -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
           - name: Ninja
-            flags: -G Ninja -DCMAKE_CXX_STANDARD=14
+            flags: -G Ninja -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
             continuous-only: true
           - name: Shared
-            flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14
+            flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
             continuous-only: true
           - name: C++17
-            flags: -DCMAKE_CXX_STANDARD=17
-          # TODO Re-enable this.
-          #- name: C++20
-          #  flags: -DCMAKE_CXX_STANDARD=20
+            flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
+          - name: C++20
+            flags: -DCMAKE_CXX_STANDARD=20 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
+          - name: Fetch
+            flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_FETCH_DEPENDENCIES=ON
 
     name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux CMake ${{ matrix.name}}
     runs-on: ubuntu-latest
@@ -174,45 +174,59 @@ jobs:
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         uses: protocolbuffers/protobuf-ci/docker@v3
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
             /test.sh ${{ matrix.flags}} ${{ env.SCCACHE_CMAKE_FLAGS }}
-            -Dprotobuf_BUILD_TESTS=ON -Dprotobuf_USE_EXTERNAL_GTEST=ON
-            -Dprotobuf_ABSL_PROVIDER=package
+            -Dprotobuf_BUILD_TESTS=ON ${{ matrix.package_flags }}
 
   linux-cmake-install:
-    name: Linux CMake Install
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        type: [package, fetch]
+        include:
+          # Set defaults
+          - type: package
+            name: Install
+            flags: -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
+          - type: fetch
+            name: Install (Fetch)
+            flags: -Dprotobuf_FETCH_DEPENDENCIES=ON
+            continuous-only: true
+    name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }}Linux CMake ${{ matrix.name }})
     runs-on: ubuntu-latest
     steps:
       - name: Checkout pending changes
         uses: protocolbuffers/protobuf-ci/checkout@v3
+        if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         with:
           ref: ${{ inputs.safe-checkout }}
-          submodules: recursive
 
       - name: Setup sccache
         uses: protocolbuffers/protobuf-ci/sccache@v3
+        if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         with:
           cache-prefix: linux-cmake-install
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
 
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/docker@v3
+        if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
             /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}
-            -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package
+            ${{ matrix.flags }}
             -Dprotobuf_BUILD_SHARED_LIBS=ON \&\&
             /test.sh
             ${{ env.SCCACHE_CMAKE_FLAGS }}
             -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON
             -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF
             -Dprotobuf_BUILD_CONFORMANCE=ON
             -DCMAKE_CXX_STANDARD=14
-            -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package
+            ${{ matrix.flags }}
 
   # This test should always be skipped on presubmit
   linux-cmake-examples:
@@ -236,7 +250,7 @@ jobs:
         if: ${{ inputs.continuous-run }}
         uses: protocolbuffers/protobuf-ci/docker@v3
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
             /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}
@@ -268,7 +282,6 @@ jobs:
         uses: protocolbuffers/protobuf-ci/checkout@v3
         with:
           ref: ${{ inputs.safe-checkout }}
-          submodules: recursive
 
       - name: Setup sccache
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}
@@ -288,7 +301,7 @@ jobs:
               -c 'set -ex;
               cd /workspace;
               sccache -z;
-              cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }};
+              cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_FETCH_DEPENDENCIES=ON;
               cmake --build . --parallel 20;
               ctest --verbose --parallel 20;
               sccache -s'
@@ -312,7 +325,7 @@ jobs:
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/docker@v3
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
             /test.sh ${{ env.SCCACHE_CMAKE_FLAGS }}
@@ -326,7 +339,6 @@ jobs:
         uses: protocolbuffers/protobuf-ci/checkout@v3
         with:
           ref: ${{ inputs.safe-checkout }}
-          submodules: recursive
 
       - name: Setup sccache
         uses: protocolbuffers/protobuf-ci/sccache@v3
@@ -337,14 +349,14 @@ jobs:
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/docker@v3
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:429f924aec315704b4233adcbe4b29006116f27769db98acd176b9eb69c31299
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:56548bef786201330017eae685cc3d2fdb564fd2ca3b88e30e28d84572e4c5dd
           platform: linux/386
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: >-
               /bin/bash -cex '
               cd /workspace;
               sccache -z;
-              cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }};
+              cmake . -DCMAKE_CXX_STANDARD=14  -Dprotobuf_FETCH_DEPENDENCIES=ON ${{ env.SCCACHE_CMAKE_FLAGS }};
               cmake --build . --parallel 20;
               ctest --verbose --parallel 20;
               sccache -s'
@@ -460,7 +472,6 @@ jobs:
         uses: protocolbuffers/protobuf-ci/checkout@v3
         with:
           ref: ${{ inputs.safe-checkout }}
-          submodules: recursive
 
       - name: Setup MSVC
         if: ${{ runner.os == 'Windows' && (!matrix.continuous-only || inputs.continuous-run) }}
@@ -499,7 +510,7 @@ jobs:
         uses: protocolbuffers/protobuf-ci/bash@v3
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
+          command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON -Dprotobuf_FETCH_DEPENDENCIES=ON
       - name: Build for install
         if: ${{ matrix.install-flags && (!matrix.continuous-only || inputs.continuous-run) }}
         shell: bash
@@ -522,7 +533,7 @@ jobs:
         uses: protocolbuffers/protobuf-ci/bash@v3
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON 
+          command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON -Dprotobuf_FETCH_DEPENDENCIES=ON 
 
       - name: Build
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -24,6 +24,11 @@ license(
     license_text = ":LICENSE",
 )
 
+exports_files(
+    ["MODULE.bazel"],
+    visibility = ["//cmake:__pkg__"],
+)
+
 ################################################################################
 # Well Known Types Proto Library Rules
 #
@@ -642,53 +647,3 @@ filegroup(
     srcs = glob(["**/*.bzl"]),
     visibility = ["//visibility:public"],
 )
-
-################################################################################
-# Packaging rules
-################################################################################
-
-# Files included in all source distributions
-pkg_files(
-    name = "common_dist_files",
-    srcs = glob(
-        [
-            "*.bzl",
-            "cmake/*.cmake",
-            "cmake/*.in",
-            "editors/*",
-        ],
-        allow_empty = True,
-    ) + [
-        "BUILD.bazel",
-        "CMakeLists.txt",
-        "CONTRIBUTORS.txt",
-        "LICENSE",
-        "README.md",
-        "WORKSPACE",
-        "cmake/README.md",
-        "generate_descriptor_proto.sh",
-        "maven_install.json",
-        "//third_party:BUILD.bazel",
-        "//third_party:zlib.BUILD",
-    ],
-    strip_prefix = strip_prefix.from_root(""),
-    visibility = ["//pkg:__pkg__"],
-)
-
-# Additional files for C#
-pkg_files(
-    name = "csharp_dist_files",
-    srcs = [
-        "global.json",
-    ],
-    visibility = ["//pkg:__pkg__"],
-)
-
-# Additional files for ObjC
-pkg_files(
-    name = "objectivec_dist_files",
-    srcs = [
-        "Protobuf.podspec",
-    ],
-    visibility = ["//pkg:__pkg__"],
-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -153,14 +153,6 @@ file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/cmaketest.map)
 
 find_package(Threads REQUIRED)
 
-# We can install dependencies from submodules if we're running
-# CMake v3.13 or newer.
-if(CMAKE_VERSION VERSION_LESS 3.13)
-  set(_protobuf_INSTALL_SUPPORTED_FROM_MODULE OFF)
-else()
-  set(_protobuf_INSTALL_SUPPORTED_FROM_MODULE ON)
-endif()
-
 set(_protobuf_FIND_ZLIB)
 if (protobuf_WITH_ZLIB)
   find_package(ZLIB)
@@ -289,11 +281,13 @@ include_directories(
   ${protobuf_BINARY_DIR}/src
   ${protobuf_SOURCE_DIR}/src)
 
-set(protobuf_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library")
-set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "module" "package")
+set(protobuf_FETCH_DEPENDENCIES OFF CACHE BOOL "Download dependencies from GitHub. If this option is not set, the dependency must be available locally, either as a sub-module or an installed package.")
+
+set(protobuf_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library.  `module` uses sub-modules, `package` searches for a local installation, and `fetch` downloads from GitHub")
+set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "module" "package" "fetch")
 
-set(protobuf_JSONCPP_PROVIDER "module" CACHE STRING "Provider of jsoncpp library")
-set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "module" "package")
+set(protobuf_JSONCPP_PROVIDER "module" CACHE STRING "Provider of jsoncpp library.  `module` uses sub-modules, `package` searches for a local installation, and `fetch` downloads from GitHub")
+set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "module" "package" "fetch")
 
 if (protobuf_BUILD_TESTS)
   include(${protobuf_SOURCE_DIR}/cmake/gtest.cmake)

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -12,7 +12,7 @@ module(
 # Bzlmod follows MVS:
 # https://bazel.build/versions/6.0.0/build/bzlmod#version-resolution
 # Thus the highest version in their module graph is resolved.
-bazel_dep(name = "abseil-cpp", version = "20230802.0.bcr.1", repo_name = "com_google_absl")
+bazel_dep(name = "abseil-cpp", version = "20230802.1", repo_name = "com_google_absl")
 bazel_dep(name = "bazel_skylib", version = "1.7.0")
 bazel_dep(name = "jsoncpp", version = "1.9.5")
 bazel_dep(name = "rules_cc", version = "0.0.13")
@@ -27,10 +27,9 @@ bazel_dep(name = "rules_rust", version = "0.51.0")
 bazel_dep(name = "platforms", version = "0.0.8")
 bazel_dep(name = "zlib", version = "1.3.1")
 bazel_dep(name = "bazel_features", version = "1.17.0", repo_name = "proto_bazel_features")
-
 bazel_dep(
     name = "rules_shell",
-    version = "0.2.0"
+    version = "0.2.0",
 )
 
 # Proto toolchains

diff --git a/cmake/BUILD.bazel b/cmake/BUILD.bazel
@@ -0,0 +1,25 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+load("//upb/cmake:build_defs.bzl", "staleness_test")
+
+py_binary(
+    name = "dependencies_generator",
+    srcs = ["dependencies_generator.py"],
+)
+
+genrule(
+    name = "generate_dependencies",
+    srcs = ["//:MODULE.bazel"],
+    outs = ["generated-in/dependencies.cmake"],
+    cmd = "$(location :dependencies_generator) " +
+          "$(location //:MODULE.bazel) $@",
+    tools = [":dependencies_generator"],
+)
+
+staleness_test(
+    name = "test_dependencies_staleness",
+    outs = [
+        "dependencies.cmake",
+    ],
+    generated_pattern = "generated-in/%s",
+    tags = ["manual"],
+)
diff --git a/cmake/abseil-cpp.cmake b/cmake/abseil-cpp.cmake
@@ -13,6 +13,20 @@ endif()
 if(TARGET absl::strings)
   # If Abseil is included already, skip including it.
   # (https://github.com/protocolbuffers/protobuf/issues/10435)
+elseif (protobuf_FETCH_DEPENDENCIES OR protobuf_ABSL_PROVIDER STREQUAL "fetch")
+  include(${protobuf_SOURCE_DIR}/cmake/dependencies.cmake)
+  include(FetchContent)
+  FetchContent_Declare(
+    absl
+    GIT_REPOSITORY "https://github.com/abseil/abseil-cpp.git"
+    GIT_TAG "${abseil-cpp-version}"
+  )
+  if(protobuf_INSTALL)
+    # When protobuf_INSTALL is enabled and Abseil will be built as a module,
+    # Abseil will be installed along with protobuf for convenience.
+    set(ABSL_ENABLE_INSTALL ON)
+  endif()
+  FetchContent_MakeAvailable(absl)
 elseif(protobuf_ABSL_PROVIDER STREQUAL "module")
   if(NOT ABSL_ROOT_DIR)
     set(ABSL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp)
@@ -27,10 +41,6 @@ elseif(protobuf_ABSL_PROVIDER STREQUAL "module")
   else()
     message(WARNING "protobuf_ABSL_PROVIDER is \"module\" but ABSL_ROOT_DIR is wrong")
   endif()
-  if(protobuf_INSTALL AND NOT _protobuf_INSTALL_SUPPORTED_FROM_MODULE)
-    message(WARNING "protobuf_INSTALL will be forced to FALSE because protobuf_ABSL_PROVIDER is \"module\" and CMake version (${CMAKE_VERSION}) is less than 3.13.")
-    set(protobuf_INSTALL FALSE)
-  endif()
 elseif(protobuf_ABSL_PROVIDER STREQUAL "package")
   # Use "CONFIG" as there is no built-in cmake module for absl.
   find_package(absl REQUIRED CONFIG)