Add support for bzlmod (#198)

This is a fairly straightforward change that adds support for using `bazel-toolchain` with `bzlmod`. The `llvm.toolchain` extension is only a wrapper around the existing `llvm_toolchain` repository rule and supports the same attributes. For trying it out, add the following to your `MODULE.bazel`: ``` bazel_dep(name = "com_grail_bazel_toolchain", version = "0.8") git_override(module_name = "com_grail_bazel_toolchain", remote = "https://github.com/grailbio/bazel-toolchain", commit = "cf915e5c7cfcd19a3e71de54e385e01240b865dc") llvm = use_extension("@com_grail_bazel_toolchain//toolchain:extensions.bzl", "llvm") llvm.toolchain( llvm_version = "15.0.0" ) use_repo(llvm, "llvm_toolchain") register_toolchains("@llvm_toolchain//:all") ``` --------- Co-authored-by: James Sharpe <[email protected]> Co-authored-by: Fabian Meumertzheim <[email protected]> Co-authored-by: Gabriel Féron <[email protected]>
bazel-contrib · Sep 12, 2023 · 5c6f806 · 5c6f806
1 parent 9d8cc8a
commit 5c6f806
Show file tree

Hide file tree

Showing 19 changed files with 591 additions and 309 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -13,17 +13,30 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, ubuntu-latest]
-        script: [run_tests.sh, run_external_tests.sh]
-        bazel_version: [latest, 6.0.0]
-        exclude:
-          - script: run_external_tests.sh
-            bazel_version: 6.0.0
+        bazel_version: [latest, 6.3.0]
+        bzlmod: [true, false]
     steps:
     - uses: actions/checkout@v3
     - name: test
       env:
         USE_BAZEL_VERSION: ${{ matrix.bazel_version }}
-      run: tests/scripts/${{ matrix.script }}
+        USE_BZLMOD: ${{ matrix.bzlmod }}
+      run: tests/scripts/run_tests.sh
+  external_test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+        bazel_version: [latest, 6.3.0]
+        bzlmod: [true, false]
+    steps:
+    - uses: actions/checkout@v2
+    - name: external_test
+      env:
+        USE_BAZEL_VERSION: ${{ matrix.bazel_version }}
+        USE_BZLMOD: ${{ matrix.bzlmod }}
+      run: tests/scripts/run_external_tests.sh
   container_test:
     runs-on: ubuntu-latest
     strategy:

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -0,0 +1,9 @@
+module(
+    name = "llvm_toolchain",
+    version = "0.0.0",
+    compatibility_level = 0,
+)
+
+bazel_dep(name = "bazel_skylib", version = "1.4.2")
+bazel_dep(name = "rules_cc", version = "0.0.8")
+bazel_dep(name = "platforms", version = "0.0.6")
diff --git a/README.md b/README.md
@@ -22,7 +22,24 @@ implementation, please let me know and I can redirect people there.
 
 Minimum bazel version: **6.0.0**
 
-To use this toolchain, include this section in your WORKSPACE:
+If you're using `bzlmod`, add the following to `MODULE.bazel`:
+
+```starlark
+bazel_dep(name = "llvm_toolchain", version = "0.8.2")
+
+llvm = use_extension("@llvm_toolchain//toolchain/extensions:llvm.bzl", "llvm")
+llvm.toolchain(
+   llvm_version = "15.0.6",
+)
+
+use_repo(llvm, "llvm_toolchain")
+# use_repo(llvm, "llvm_toolchain_llvm") # if you depend on specific tools in scripts
+
+register_toolchains("@llvm_toolchain//:all")
+```
+
+To use this toolchain, include this section in your `WORKSPACE`:
+
 ```starlark
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 

diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel
@@ -119,6 +119,10 @@ cc_test(
 sh_test(
     name = "file_dependency_test",
     srcs = ["file_dependency_test.sh"],
+    args = [
+        "$(execpath @llvm_toolchain_llvm//:bin/clang-format)",
+        "$(rootpaths @llvm_toolchain_llvm//:lib)",
+    ],
     data = [
         "@llvm_toolchain_llvm//:bin/clang-format",
         "@llvm_toolchain_llvm//:lib",

diff --git a/tests/MODULE.bazel b/tests/MODULE.bazel
@@ -0,0 +1,139 @@
+module(name = "com_grail_bazel_toolchain_tests")
+
+bazel_dep(name = "llvm_toolchain", version = "0.8.2", repo_name = "com_grail_bazel_toolchain")
+local_path_override(module_name = "llvm_toolchain", path = "..")
+
+bazel_dep(name = "bazel_skylib", version = "1.4.2")
+bazel_dep(name = "platforms", version = "0.0.6")
+bazel_dep(name = "rules_cc", version = "0.0.8")
+bazel_dep(name = "rules_go", version = "0.40.1", repo_name = "io_bazel_rules_go")
+bazel_dep(name = "rules_foreign_cc", version = "0.9.0")
+bazel_dep(name = "rules_rust", version = "0.27.0")
+
+# We have to patch abseil-cpp to add a dep on googletest
+bazel_dep(name = "abseil-cpp", repo_name = "com_google_absl")
+archive_override(module_name = "abseil-cpp", urls = ["https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz"], patches = ["//patches:abseil-cpp.patch"], strip_prefix = "abseil-cpp-20230125.3")
+
+# Temporary overrides until newer versions of these rulesets are released in the central registry
+git_override(module_name = "rules_foreign_cc", remote = "https://github.com/bazelbuild/rules_foreign_cc.git", commit = "6ecc134b114f6e086537f5f0148d166467042226")
+git_override(module_name = "rules_rust", remote = "https://github.com/bazelbuild/rules_rust.git", commit = "4dbb81a008f4031c9ad65138fa2a49f32352f80f")
+
+rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
+rust.toolchain(edition = "2021")
+use_repo(
+    rust,
+    "rust_toolchains",
+)
+
+register_toolchains("@rust_toolchains//:all")
+
+llvm = use_extension("@com_grail_bazel_toolchain//toolchain/extensions:llvm.bzl", "llvm")
+
+# When updating this version, also update the versions associated with
+# llvm_toolchain below, sys_paths_test in the workflows file, and xcompile_test
+# through the `llvm_toolchain_with_sysroot` toolchain.
+LLVM_VERSION = "15.0.6"
+
+llvm.toolchain(
+    name = "llvm_toolchain",
+    llvm_versions = {
+        "": "15.0.6",
+        "darwin-aarch64": "15.0.7",
+        "darwin-x86_64": "15.0.7",
+    },
+)
+
+use_repo(llvm, "llvm_toolchain", "llvm_toolchain_llvm")
+register_toolchains("@llvm_toolchain//:all")
+
+# Example toolchain with user provided URLs.
+# TODO(siddharthab): Add test.
+llvm.toolchain(
+    name = "llvm_toolchain_with_urls",
+    llvm_versions = {
+        "": "15.0.6",
+        "darwin-aarch64": "15.0.7",
+        "darwin-x86_64": "15.0.7",
+    },
+    sha256 = {
+        "": "38bc7f5563642e73e69ac5626724e206d6d539fbef653541b34cae0ba9c3f036",
+        "darwin-aarch64": "867c6afd41158c132ef05a8f1ddaecf476a26b91c85def8e124414f9a9ba188d",
+        "darwin-x86_64": "d16b6d536364c5bec6583d12dd7e6cf841b9f508c4430d9ee886726bd9983f1c",
+    },
+    strip_prefix = {
+        "": "clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04",
+        "darwin-aarch64": "clang+llvm-15.0.7-arm64-apple-darwin22.0",
+        "darwin-x86_64": "clang+llvm-15.0.7-x86_64-apple-darwin21.0",
+    },
+    urls = {
+        "": ["https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz"],
+        "darwin-aarch64": ["https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.7/clang+llvm-15.0.7-arm64-apple-darwin22.0.tar.xz"],
+        "darwin-x86_64": ["https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.7/clang+llvm-15.0.7-x86_64-apple-darwin21.0.tar.xz"],
+    },
+)
+
+use_repo(llvm, "llvm_toolchain_with_urls")
+
+# # This is the latest version of LLVM that seems to work with rules_go; later
+# # versions cause the tests to crash.
+llvm.toolchain(
+    name = "llvm_toolchain_14_0_0",
+    llvm_version = "14.0.0",
+)
+
+use_repo(llvm, "llvm_toolchain_14_0_0")
+
+# # This is the last known LLVM version with zlib support in ld.lld. Without zlib
+# # support, if the installed gcc toolchain has compressed sections in its object
+# # files, then ld.lld won't be able to process them. Example is archlinux docker
+# # image base-devel as of the time of this writing (23 May 2022).
+llvm.toolchain(
+    name = "llvm_toolchain_13_0_0",
+    llvm_version = "13.0.0",
+)
+
+use_repo(llvm, "llvm_toolchain_13_0_0")
+
+# ## Toolchain example with absolute paths; tested in GitHub CI.
+llvm.toolchain(
+    name = "llvm_toolchain_with_absolute_paths",
+    absolute_paths = True,
+    llvm_version = LLVM_VERSION,
+    # We can share the downloaded LLVM distribution with the first configuration.
+    toolchain_roots = {
+        "": "@llvm_toolchain_llvm//",
+    },
+)
+
+use_repo(llvm, "llvm_toolchain_with_absolute_paths")
+
+# ## Toolchain example with system LLVM; tested in GitHub CI.
+llvm.toolchain(
+    name = "llvm_toolchain_with_system_llvm",
+    llvm_version = LLVM_VERSION,
+    # For this toolchain to work, the LLVM distribution archive would need to be unpacked here.
+    # A path in /tmp to be part of system tmp cleanup schedule.
+    toolchain_roots = {"": "/tmp/llvm-15"},
+)
+
+use_repo(llvm, "llvm_toolchain_with_system_llvm")
+
+# ## Toolchain example with a sysroot.
+
+llvm.toolchain(
+    name = "llvm_toolchain_with_sysroot",
+    llvm_versions = {
+        "": "15.0.6",
+        "darwin-x86_64": "15.0.7",
+        "darwin-aarch64": "15.0.7",
+    },
+    sysroot = {
+        "linux-x86_64": "@org_chromium_sysroot_linux_x64//:sysroot",
+    },
+    # We can share the downloaded LLVM distribution with the first configuration.
+    toolchain_roots = {
+        "": "@llvm_toolchain_llvm//",
+    },
+)
+
+use_repo(llvm, "llvm_toolchain_with_sysroot")
diff --git a/tests/WORKSPACE.bzlmod b/tests/WORKSPACE.bzlmod
@@ -0,0 +1,53 @@
+workspace(name = "com_grail_bazel_toolchain_tests")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+# This sysroot is used by github.com/vsco/bazel-toolchains.
+http_archive(
+    name = "org_chromium_sysroot_linux_x64",
+    build_file_content = """
+filegroup(
+  name = "sysroot",
+  srcs = glob(["*/**"]),
+  visibility = ["//visibility:public"],
+)
+""",
+    sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3",
+    urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"],
+)
+
+# Well known repos; present here only for testing.
+
+http_archive(
+    name = "com_github_google_benchmark",
+    sha256 = "6430e4092653380d9dc4ccb45a1e2dc9259d581f4866dc0759713126056bc1d7",
+    strip_prefix = "benchmark-1.7.1",
+    urls = ["https://github.com/google/benchmark/archive/v1.7.1.tar.gz"],
+)
+
+http_archive(
+    name = "openssl",
+    build_file = "//openssl:openssl.bazel",
+    sha256 = "f6fb3079ad15076154eda9413fed42877d668e7069d9b87396d0804fdb3f4c90",
+    strip_prefix = "openssl-1.1.1c",
+    urls = ["https://www.openssl.org/source/openssl-1.1.1c.tar.gz"],
+)
+
+_ALL_CONTENT = """\
+filegroup(
+    name = "all_srcs",
+    srcs = glob(["**"]),
+    visibility = ["//visibility:public"],
+)
+"""
+
+http_archive(
+    name = "pcre",
+    build_file_content = _ALL_CONTENT,
+    sha256 = "0b8e7465dc5e98c757cc3650a20a7843ee4c3edf50aaf60bb33fd879690d2c73",
+    strip_prefix = "pcre-8.43",
+    urls = [
+        "https://mirror.bazel.build/ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz",
+        "https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz",
+    ],
+)
diff --git a/tests/file_dependency_test.sh b/tests/file_dependency_test.sh
@@ -18,10 +18,13 @@ fail() {
   exit 1
 }
 
-[[ -a "external/llvm_toolchain_llvm/bin/clang-format" ]] || fail "bin/clang-format not found"
+clang_format_path=$1
+libcpp_path=$2
 
-[[ -a "external/llvm_toolchain_llvm/lib/libc++.a" ]] \
-  || compgen -G 'external/llvm_toolchain_llvm/lib/*/libc++.a' >/dev/null \
+[[ -a "${clang_format_path}" ]] || fail "bin/clang-format not found"
+
+[[ -a "${libcpp_path}" ]] \
+  || compgen -G "${libcpp_path}" >/dev/null \
   || fail "libc++.a not found"
 
 echo "SUCCESS!"
diff --git a/tests/openssl/BUILD.bazel b/tests/openssl/BUILD.bazel
@@ -1,4 +1,4 @@
 exports_files(
     srcs = glob(["**/*.h"]),
-    visibility = ["@openssl//:__pkg__"],
+    visibility = ["//visibility:public"],
 )
diff --git a/tests/rust/BUILD.bazel → tests/patches/BUILD.bazel b/tests/rust/BUILD.bazel → tests/patches/BUILD.bazel
diff --git a/tests/patches/abseil-cpp.patch b/tests/patches/abseil-cpp.patch
@@ -0,0 +1,12 @@
+--- MODULE.bazel
++++ MODULE.bazel
+@@ -0,0 +1,9 @@
++module(
++    name = "abseil-cpp",
++    version = "20230125.1",
++    compatibility_level = 1,
++)
++bazel_dep(name = "rules_cc", version = "0.0.8")
++bazel_dep(name = "platforms", version = "0.0.6")
++bazel_dep(name = "bazel_skylib", version = "1.4.2")
++bazel_dep(name = "googletest", version = "1.12.1", repo_name = "com_google_googletest")
diff --git a/tests/rust/rules_rust.patch b/tests/rust/rules_rust.patch
diff --git a/tests/scripts/run_external_tests.sh b/tests/scripts/run_external_tests.sh
@@ -23,10 +23,16 @@ cd "${scripts_dir}"
 
 # Generate some files needed for the tests.
 "${bazel}" fetch @io_bazel_rules_go//tests/core/cgo:all
-"$("${bazel}" info output_base)/external/io_bazel_rules_go/tests/core/cgo/generate_imported_dylib.sh"
+if [[ "$USE_BZLMOD" == "true" ]]; then
+  "$("${bazel}" info output_base)/external/rules_go~0.40.1/tests/core/cgo/generate_imported_dylib.sh"
+else
+  "$("${bazel}" info output_base)/external/io_bazel_rules_go/tests/core/cgo/generate_imported_dylib.sh"
+fi
 
+set -x
 test_args=(
   "${common_test_args[@]}"
+  "--enable_bzlmod=${USE_BZLMOD:-false}"
   # Fix LLVM version to be 14.0.0 because that's the last known version with
   # which the tests in rules_go pass.
   "--extra_toolchains=@llvm_toolchain_14_0_0//:all"
@@ -39,12 +45,19 @@ test_args=(
 #   link it statically on linux.
 # - external_includes_test from rules_go because it is a nested bazel test and so takes a long time
 #   to run, and it is not particularly useful to us.
+# - time_zone_format_test from abseil-cpp because it assumes TZ is set to America/Los_Angeles, but
+#   we run the tests in UTC.
+#   The rules_rust tests should be:
+# @rules_rust//test/unit/{native_deps,linkstamps,interleaved_cc_info}:all
+#   but under bzlmod the linkstamp tests fail due to the fact we are currently
+#   overriding rules_rust locally as its not yet released in the BCR
 "${bazel}" --bazelrc=/dev/null test "${test_args[@]}" -- \
   //foreign:pcre \
   @openssl//:libssl \
-  @rules_rust//test/unit/{native_deps,linkstamps,interleaved_cc_info}:all \
+  @rules_rust//test/unit/interleaved_cc_info:all \
   @io_bazel_rules_go//tests/core/cgo:all \
   -@io_bazel_rules_go//tests/core/cgo:cc_libs_test \
   -@io_bazel_rules_go//tests/core/cgo:external_includes_test \
   $("${bazel}" query 'attr(timeout, short, tests(@com_google_absl//absl/...))') \
+  -@com_google_absl//absl/time/internal/cctz:time_zone_format_test