diff --git a/MODULE.bazel b/MODULE.bazel index b9f9fbf24dbee..fcf2eeaa958a7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,7 @@ module( # https://bazel.build/versions/6.0.0/build/bzlmod#version-resolution # Thus the highest version in their module graph is resolved. bazel_dep(name = "abseil-cpp", version = "20230802.0.bcr.1", repo_name = "com_google_absl") -bazel_dep(name = "bazel_skylib", version = "1.4.1") +bazel_dep(name = "bazel_skylib", version = "1.7.0") bazel_dep(name = "jsoncpp", version = "1.9.5") bazel_dep(name = "rules_cc", version = "0.0.9") bazel_dep(name = "rules_fuzzing", version = "0.5.2") diff --git a/bazel/private/BUILD.bazel b/bazel/private/BUILD.bazel index 1ff703b53d73b..407c325916129 100644 --- a/bazel/private/BUILD.bazel +++ b/bazel/private/BUILD.bazel @@ -6,6 +6,7 @@ # https://developers.google.com/open-source/licenses/bsd load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//bazel/private:native_bool_flag.bzl", "native_bool_flag") licenses(["notice"]) @@ -52,3 +53,26 @@ bzl_library( "//bazel/common:proto_lang_toolchain_info_bzl", ], ) + +native_bool_flag( + name = "experimental_proto_descriptor_sets_include_source_info", + flag = "experimental_proto_descriptor_sets_include_source_info", + match_value = "true", + visibility = ["//visibility:public"], +) + +native_bool_flag( + name = "strict_proto_deps", + flag = "strict_proto_deps", + match_value = "off", + result = False, + visibility = ["//visibility:public"], +) + +native_bool_flag( + name = "strict_public_imports", + flag = "strict_public_imports", + match_value = "off", + result = False, + visibility = ["//visibility:public"], +) diff --git a/bazel/private/native_bool_flag.bzl b/bazel/private/native_bool_flag.bzl new file mode 100644 index 0000000000000..960fbed5128e7 --- /dev/null +++ b/bazel/private/native_bool_flag.bzl @@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +A helper rule that reads a native boolean flag. +""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") + +def _impl(ctx): + return [BuildSettingInfo(value = ctx.attr.value)] + +_native_bool_flag_rule = rule( + implementation = _impl, + attrs = {"value": attr.bool()}, +) + +def native_bool_flag(*, name, flag, match_value = "true", result = True, **kwargs): + _native_bool_flag_rule( + name = name, + value = select({ + name + "_setting": result, + "//conditions:default": not result, + }), + **kwargs + ) + + native.config_setting( + name = name + "_setting", + values = {flag: match_value}, + visibility = ["//visibility:private"], + ) diff --git a/bazel/private/proto_library_rule.bzl b/bazel/private/proto_library_rule.bzl new file mode 100644 index 0000000000000..b02f691f45e37 --- /dev/null +++ b/bazel/private/proto_library_rule.bzl @@ -0,0 +1,357 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +Implementation of proto_library rule. +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@proto_bazel_features//:features.bzl", "bazel_features") +load("//bazel/common:proto_common.bzl", "proto_common") +load("//bazel/common:proto_info.bzl", "ProtoInfo") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") + +STRICT_DEPS_FLAG_TEMPLATE = ( + # + "--direct_dependencies_violation_msg=" + + "%%s is imported, but %s doesn't directly depend on a proto_library that 'srcs' it." +) + +def _check_srcs_package(target_package, srcs): + """Check that .proto files in sources are from the same package. + + This is done to avoid clashes with the generated sources.""" + + #TODO: this does not work with filegroups that contain files that are not in the package + for src in srcs: + if target_package != src.label.package: + fail("Proto source with label '%s' must be in same package as consuming rule." % src.label) + +def _get_import_prefix(ctx): + """Gets and verifies import_prefix attribute if it is declared.""" + + import_prefix = ctx.attr.import_prefix + + if not paths.is_normalized(import_prefix): + fail("should be normalized (without uplevel references or '.' path segments)", attr = "import_prefix") + if paths.is_absolute(import_prefix): + fail("should be a relative path", attr = "import_prefix") + + return import_prefix + +def _get_strip_import_prefix(ctx): + """Gets and verifies strip_import_prefix.""" + + strip_import_prefix = ctx.attr.strip_import_prefix + + if not paths.is_normalized(strip_import_prefix): + fail("should be normalized (without uplevel references or '.' path segments)", attr = "strip_import_prefix") + + if paths.is_absolute(strip_import_prefix): + strip_import_prefix = strip_import_prefix[1:] + else: # Relative to current package + strip_import_prefix = _join(ctx.label.package, strip_import_prefix) + + return strip_import_prefix.removesuffix("/") + +def _proto_library_impl(ctx): + # Verifies attributes. + _check_srcs_package(ctx.label.package, ctx.attr.srcs) + srcs = ctx.files.srcs + deps = [dep[ProtoInfo] for dep in ctx.attr.deps] + exports = [dep[ProtoInfo] for dep in ctx.attr.exports] + import_prefix = _get_import_prefix(ctx) + strip_import_prefix = _get_strip_import_prefix(ctx) + check_for_reexport = deps + exports if not srcs else exports + _PackageSpecificationInfo = bazel_features.globals.PackageSpecificationInfo + for proto in check_for_reexport: + if getattr(proto, "allow_exports", None): + if not _PackageSpecificationInfo: + fail("Allowlist checks not supported before Bazel 6.4.0") + if not proto.allow_exports[_PackageSpecificationInfo].contains(ctx.label): + fail("proto_library '%s' can't be reexported in package '//%s'" % (proto.direct_descriptor_set.owner, ctx.label.package)) + + proto_path, virtual_srcs = _process_srcs(ctx, srcs, import_prefix, strip_import_prefix) + descriptor_set = ctx.actions.declare_file(ctx.label.name + "-descriptor-set.proto.bin") + proto_info = ProtoInfo( + srcs = virtual_srcs, + deps = deps, + descriptor_set = descriptor_set, + proto_path = proto_path, + workspace_root = ctx.label.workspace_root, + bin_dir = ctx.bin_dir.path, + allow_exports = ctx.attr.allow_exports, + ) + + _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set) + + # We assume that the proto sources will not have conflicting artifacts + # with the same root relative path + data_runfiles = ctx.runfiles( + files = [proto_info.direct_descriptor_set], + transitive_files = depset(transitive = [proto_info.transitive_sources]), + ) + return [ + proto_info, + DefaultInfo( + files = depset([proto_info.direct_descriptor_set]), + default_runfiles = ctx.runfiles(), # empty + data_runfiles = data_runfiles, + ), + ] + +def _process_srcs(ctx, srcs, import_prefix, strip_import_prefix): + """Returns proto_path and sources, optionally symlinking them to _virtual_imports. + + Returns: + (str, [File]) A pair of proto_path and virtual_sources. + """ + if import_prefix != "" or strip_import_prefix != "": + # Use virtual source roots + return _symlink_to_virtual_imports(ctx, srcs, import_prefix, strip_import_prefix) + else: + # No virtual source roots + return "", srcs + +def _join(*path): + return "/".join([p for p in path if p != ""]) + +def _symlink_to_virtual_imports(ctx, srcs, import_prefix, strip_import_prefix): + """Symlinks srcs to _virtual_imports. + + Returns: + A pair proto_path, directs_sources. + """ + virtual_imports = _join("_virtual_imports", ctx.label.name) + proto_path = _join(ctx.label.package, virtual_imports) + + if ctx.label.workspace_name == "": + full_strip_import_prefix = strip_import_prefix + else: + full_strip_import_prefix = _join("..", ctx.label.workspace_name, strip_import_prefix) + if full_strip_import_prefix: + full_strip_import_prefix += "/" + + virtual_srcs = [] + for src in srcs: + # Remove strip_import_prefix + if not src.short_path.startswith(full_strip_import_prefix): + fail(".proto file '%s' is not under the specified strip prefix '%s'" % + (src.short_path, full_strip_import_prefix)) + import_path = src.short_path[len(full_strip_import_prefix):] + + # Add import_prefix + virtual_src = ctx.actions.declare_file(_join(virtual_imports, import_prefix, import_path)) + ctx.actions.symlink( + output = virtual_src, + target_file = src, + progress_message = "Symlinking virtual .proto sources for %{label}", + ) + virtual_srcs.append(virtual_src) + return proto_path, virtual_srcs + +def _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set): + """Writes descriptor set.""" + if proto_info.direct_sources == []: + ctx.actions.write(descriptor_set, "") + return + + dependencies_descriptor_sets = depset(transitive = [dep.transitive_descriptor_sets for dep in deps]) + + args = ctx.actions.args() + + if ctx.attr._experimental_proto_descriptor_sets_include_source_info[BuildSettingInfo].value: + args.add("--include_source_info") + if hasattr(ctx.attr, "_retain_options") and ctx.attr._retain_options: + args.add("--retain_options") + + strict_deps = ctx.attr._strict_proto_deps[BuildSettingInfo].value + if strict_deps: + if proto_info.direct_sources: + strict_importable_sources = depset( + direct = proto_info._direct_proto_sources, + transitive = [dep._exported_sources for dep in deps], + ) + else: + strict_importable_sources = None + if strict_importable_sources: + args.add_joined( + "--direct_dependencies", + strict_importable_sources, + map_each = proto_common.get_import_path, + join_with = ":", + ) + # Example: `--direct_dependencies a.proto:b.proto` + + else: + # The proto compiler requires an empty list to turn on strict deps checking + args.add("--direct_dependencies=") + + # Set `-direct_dependencies_violation_msg=` + args.add(ctx.label, format = STRICT_DEPS_FLAG_TEMPLATE) + + strict_imports = ctx.attr._strict_public_imports[BuildSettingInfo].value + if strict_imports: + public_import_protos = depset(transitive = [export._exported_sources for export in exports]) + if not public_import_protos: + # This line is necessary to trigger the check. + args.add("--allowed_public_imports=") + else: + args.add_joined( + "--allowed_public_imports", + public_import_protos, + map_each = proto_common.get_import_path, + join_with = ":", + ) + if proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION: + toolchain = ctx.toolchains[toolchains.PROTO_TOOLCHAIN] + if not toolchain: + fail("Protocol compiler toolchain could not be resolved.") + proto_lang_toolchain_info = toolchain.proto + else: + proto_lang_toolchain_info = proto_common.ProtoLangToolchainInfo( + out_replacement_format_flag = "--descriptor_set_out=%s", + output_files = "single", + mnemonic = "GenProtoDescriptorSet", + progress_message = "Generating Descriptor Set proto_library %{label}", + proto_compiler = ctx.executable._proto_compiler, + protoc_opts = ctx.fragments.proto.experimental_protoc_opts, + plugin = None, + ) + + proto_common.compile( + ctx.actions, + proto_info, + proto_lang_toolchain_info, + generated_files = [descriptor_set], + additional_inputs = dependencies_descriptor_sets, + additional_args = args, + ) + +proto_library = rule( + _proto_library_impl, + # TODO: proto_common docs are missing + # TODO: ProtoInfo link doesn't work and docs are missing + doc = """ +

If using Bazel, please load the rule from +https://github.com/bazelbuild/rules_proto. + +

Use proto_library to define libraries of protocol buffers which +may be used from multiple languages. A proto_library may be listed +in the deps clause of supported rules, such as +java_proto_library. + +

When compiled on the command-line, a proto_library creates a file +named foo-descriptor-set.proto.bin, which is the descriptor set for +the messages the rule srcs. The file is a serialized +FileDescriptorSet, which is described in + +https://developers.google.com/protocol-buffers/docs/techniques#self-description. + +

It only contains information about the .proto files directly +mentioned by a proto_library rule; the collection of transitive +descriptor sets is available through the +[ProtoInfo].transitive_descriptor_sets Starlark provider. +See documentation in proto_info.bzl. + +

Recommended code organization: +

""", + attrs = { + "srcs": attr.label_list( + allow_files = [".proto", ".protodevel"], + flags = ["DIRECT_COMPILE_TIME_INPUT"], + # TODO: Should .protodevel be advertised or deprecated? + doc = """ +The list of .proto and .protodevel files that are +processed to create the target. This is usually a non empty list. One usecase +where srcs can be empty is an alias-library. This is a +proto_library rule having one or more other proto_library in deps. +This pattern can be used to e.g. export a public api under a persistent name.""", + ), + "deps": attr.label_list( + providers = [ProtoInfo], + doc = """ +The list of other proto_library rules that the target depends upon. +A proto_library may only depend on other proto_library +targets. It may not depend on language-specific libraries.""", + ), + "exports": attr.label_list( + providers = [ProtoInfo], + doc = """ +List of proto_library targets that can be referenced via "import public" in the +proto source. +It's an error if you use "import public" but do not list the corresponding library +in the exports attribute. +Note that you have list the library both in deps and exports since not all +lang_proto_library implementations have been changed yet.""", + ), + "strip_import_prefix": attr.string( + default = "/", + doc = """ +The prefix to strip from the paths of the .proto files in this rule. + +

When set, .proto source files in the srcs attribute of this rule are +accessible at their path with this prefix cut off. + +

If it's a relative path (not starting with a slash), it's taken as a package-relative +one. If it's an absolute one, it's understood as a repository-relative path. + +

The prefix in the import_prefix attribute is added after this prefix is +stripped.""", + ), + "import_prefix": attr.string( + doc = """ +The prefix to add to the paths of the .proto files in this rule. + +

When set, the .proto source files in the srcs attribute of this rule are +accessible at is the value of this attribute prepended to their repository-relative path. + +

The prefix in the strip_import_prefix attribute is removed before this +prefix is added.""", + ), + "allow_exports": attr.label( + cfg = "exec", + providers = [bazel_features.globals.PackageSpecificationInfo] if bazel_features.globals.PackageSpecificationInfo else [], + doc = """ +An optional allowlist that prevents proto library to be reexported or used in +lang_proto_library that is not in one of the listed packages.""", + ), + "data": attr.label_list( + allow_files = True, + flags = ["SKIP_CONSTRAINTS_OVERRIDE"], + ), + # buildifier: disable=attr-license (calling attr.license()) + "licenses": attr.license() if hasattr(attr, "license") else attr.string_list(), + "_experimental_proto_descriptor_sets_include_source_info": attr.label( + default = "//bazel/private:experimental_proto_descriptor_sets_include_source_info", + ), + "_strict_proto_deps": attr.label( + default = + "//bazel/private:strict_proto_deps", + ), + "_strict_public_imports": attr.label( + default = "//bazel/private:strict_public_imports", + ), + } | toolchains.if_legacy_toolchain({ + "_proto_compiler": attr.label( + cfg = "exec", + executable = True, + allow_files = True, + default = configuration_field("proto", "proto_compiler"), + ), + }), # buildifier: disable=attr-licenses (attribute called licenses) + fragments = ["proto"], + provides = [ProtoInfo], + toolchains = toolchains.use_toolchain(toolchains.PROTO_TOOLCHAIN), +) diff --git a/bazel/proto_library.bzl b/bazel/proto_library.bzl index 3006e3c0c29d0..f0fece1553739 100644 --- a/bazel/proto_library.bzl +++ b/bazel/proto_library.bzl @@ -1,3 +1,20 @@ -"""proto_library rule""" +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +Macro of proto_library rule. +""" -proto_library = native.proto_library +load("@proto_bazel_features//:features.bzl", "bazel_features") +load("//bazel/private:proto_library_rule.bzl", _proto_library = "proto_library") + +def proto_library(**kwattrs): + # This condition causes Starlark rules to be used only on Bazel >=7.0.0 + if bazel_features.proto.starlark_proto_info: + _proto_library(**kwattrs) + else: + # On older Bazel versions keep using native rules, so that mismatch in ProtoInfo doesn't happen + native.proto_library(**kwattrs) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 9dcf0c0428878..eea60726969fc 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -51,11 +51,11 @@ def protobuf_deps(): if not native.existing_rule("bazel_skylib"): http_archive( name = "bazel_skylib", + sha256 = "d00f1389ee20b60018e92644e0948e16e350a7707219e7a390fb0a99b6ec9262", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.7.0/bazel-skylib-1.7.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.7.0/bazel-skylib-1.7.0.tar.gz", ], - sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", ) if not native.existing_rule("com_google_absl"):