Maintenance

- ARM support - Revised scripts for ARM support for macOS binary packages - Switch from MRAN to Posit Package Manager - Miscellaneous updates of deps - Disable container support until switch to rules_oci.
grailbio · Jan 17, 2024 · e83da58 · e83da58
1 parent 36d9f61
commit e83da58
Show file tree

Hide file tree

Showing 16 changed files with 207 additions and 126 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -24,16 +24,16 @@ jobs:
         #    r_version: release
     steps:
     - name: Check out repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Setup R
-      uses: r-lib/actions/setup-r@v1
+      uses: r-lib/actions/setup-r@v2
       with:
         r-version: ${{ matrix.r_version }}
 
     - name: Install additional packages
       if: matrix.os == 'ubuntu-latest'
-      run: sudo apt-get install libcurl4-openssl-dev
+      run: sudo apt-get install libtinfo5 libcurl4-openssl-dev
 
     - name: Create Buildbuddy certs
       shell: bash
@@ -56,7 +56,7 @@ jobs:
       run: tests/run_tests.sh
 
     - name: Archive debug artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
         name: archive-debug-artifacts

diff --git a/R/internal/coverage_deps.bzl b/R/internal/coverage_deps.bzl
@@ -31,7 +31,7 @@ def r_coverage_dependencies(**kwargs):
         package_list = "@rules_r//R/internal:coverage_deps_list.csv",
         remote_repos = {
             # CRAN does not retain binary archives for macOS.
-            "CRAN": "https://cran.microsoft.com/snapshot/2022-02-28",
+            "CRAN": "https://packagemanager.posit.co/cran/2024-01-15/",
         },
         **kwargs
     )
diff --git a/R/internal/coverage_deps_list.csv b/R/internal/coverage_deps_list.csv
@@ -1,19 +1,21 @@
-"Package","Version","sha256","mac_4_1_sha256"
-"askpass","1.1","db40827d1bdbb90c0aa2846a2961d3bf9d76ad1b392302f9dd84cc2fd18c001f","f724f7eb2388ac00895389534c92b4f6639449b21f7b9f0c3109c0e1e34f121d"
-"covr","3.5.1","a54cfc3623ea56084158ac5d7fe33f216f45191f6dcddab9c9ed4ec1d9d8ac6c","72b1efeb88ec4970129bbb7cac3aa16dd338cbcc5228bd033d88e5e04330cd8b"
-"crayon","1.5.0","44122cafddcd8a35176b2a6bcab10c5f81cd5440d9e29d20091b31a0ef647954","26063add2696edb8ed7d2db9359e59575a4278f200bae1bdab19f1d6b6d79785"
-"curl","4.3.2","90b1facb4be8b6315bb3d272ba2dd90b88973f6ea1ab7f439550230f8500a568","2aed012ebf5aa1b6d804fb655686a1323fe81c9f4adecf56a5e1bf8061e994ce"
-"digest","0.6.29","792c1f14a4c8047745152f5e45ce7351978af8d770c29d2ea39c7acd5d619cd9","23c3d2871be2f95d8891390d1f99b2f31e55e1d62bb3f555ae9014e69729f6a7"
-"httr","1.4.2","462bed6ed0d92f811d5df4d294336025f1dbff357286999d9269bfd9c20b1ef9","1494d67e7fbf91eaf76bc979e4547b2d23d24bb8baa676ade733b4542832a583"
-"jsonlite","1.8.0","7b1892efebcb4cf4628f716000accd4b43bbf82b3e6ba90b9529d4fa0e55cd4c","26b9387277396f926d68f8dd52388d287328512bf57f8a5e34aaf2d9d8fb1ff9"
-"lazyeval","0.2.2","d6904112a21056222cfcd5eb8175a78aa063afe648a562d9c42c6b960a8820d4","c91cc7b952ede831c3dadbc755c6fae9dd2a3ebe9cdd75970f698770ae68a8fa"
-"magrittr","2.0.2","7be6fd0d0da75b92d8bad0136076da96260ee84bf639ef632a24668acdc163a6","ee6b35fbb331045fd86541035f0098127e51bb7ae7e08186469eedba81e79b5b"
-"mime","0.12","a9001051d6c1e556e881910b1816b42872a1ee41ab76d0040ce66a27135e3849","16cef814356f396d13f6d554e5e0614f0ca33af8b15c2896004e25e1bb14dc04"
-"openssl","1.4.6","43b832af70e71770168b997107c52c8f406f8c33e9ef9b289610bccea2f34359","36bf31c1c3e3dd2e115ad2efbf00e85ded3b9b4a2040654b3b0fa89cc13a8486"
-"R6","2.5.1","8d92bd29c2ed7bf15f2778618ffe4a95556193d21d8431a7f75e7e5fc102bf48","7244f47de06e35030a21ed0836fb15c46461bf255ce51381b3beea14b0237317"
-"Rcpp","1.0.8","879f9296bc045ac4ed464578723bd37fcabbbdaa30aaaf070cf953e329f678ee","11b05324ae2677f100a8a4efed9cc911b1e187a620e9381648f4e075fb3ab80c"
-"rex","1.2.1","af42e649c06e4bbdba94d5a1870a7e8347903571c90cd5e5ca40f52307a3bfd6","c19029a09cfebb001ca6657cb7d8e3f72dbc9e03de712f86fce092a7a5a9c1cb"
-"sys","3.4","17f88fbaf222f1f8fd07919461093dac0e7175ae3c3b3264b88470617afd0487","6474a97353cf4deb89353af95967d67873efeba30751bdb3b30d241a1d042dcc"
-"withr","2.4.3","9bdac7459ccc6c2d599ecfd132a7f0aa68d958942d9fe7dbb0442c9eda129d4c","599291c992ec81e4c7f733abe5dd4a3e30660f02472811c6d5aefe6d443fc60c"
-"xml2","1.3.3","cb4e9c0d31618ed67d2bfa4c7b5e52680e11612ed356a8164b541d44163c1c8d","c00bbd56fc4ae6a31bfa81ac340ec77e4ee2a724e1607206288d06a6d3ae5b22"
-"yaml","2.3.5","3edf6c0554a0e184a25e8bec5721a2e66b4ab0dceb3737428e22705e52eb5140","b9e719ae3150053532a9bdeec89bdc378b7f35c5aa8e12ef46ab7e827962115e"
+"Package","Version","sha256","mac_intel_4_3_sha256","mac_arm_4_3_sha256"
+"askpass","1.2.0","4f2dae765d933a0eb327291cbc5bd34301246c7b04c009dd0fd92a9f61e55590","7672cca460ae93bfe5270a2039b2c95cae0ede4afb00d9e0fe3948428bf485ad","f16ea1fea1cd09a53df4b9e154e8b282d292062d3d5f1bb36d28bc3198f48347"
+"cli","3.6.2","f0b910b5dcf46118acb7ab5ede068473326b34c844fdf39d2a8726f606575247","f43afe8e28d8dd7d76e4cd06c489ec132a38a0d13fa3aec1adbd46c6011ffef8","c979f598d955eaf2af58e7d73767571e848bf3b5d98a98ff3881518080eb996f"
+"covr","3.6.4","d5b974664e40983ded73365fe87a3027c18664a5597a80678b89bbe251a1cc7a","3d8f5e225a3305f5f10b18ac08653ef3ebbe4748c094f537deb0d7fbc2cddcc6","c7dd1f80dac49c9e7e5659182f70c72e0980e39c55c7f022ccebe761aa9a6bc6"
+"crayon","1.5.2","caea4f7ef1ed483476ec151624e6aef3f34625ff20655da08d2222260d5ada63","286db97a3974be40499b7e68b927b82e6c22730b20e56f8e0aaa8d7bef391aca","44a5b59f1bf1799fbfbdfb4477119327788189d384fbefd69cd9af9036193e07"
+"curl","5.2.0","451ad2747addfc39cf2c7af121ac5e9c9559d45a5618703e47c6ac68fab5f848","51e1fdeddb80a9128617be9926a5754c1328489936011296bc4793c56b968196","0902237f6d2bb9a4e848a630558730ca8a6e9051f621bfc05b147ed98c060af3"
+"digest","0.6.34","e8aa5dfd841dd332ec6c2ec0fbf41597284b7a6c0b5aa5461ddb4d1aa31e6028","0579112114c088d44138c9c0e0ca21267c83a5821c80420ae8d8f1f0f3272422","2977c5af6b87dacc15b62e6050f4c64523d08e7b192ff18c6394c4e11ca0cc9e"
+"httr","1.4.7","7cd83395f99633504deb15bcf133616624fe1b52db39e66c9319c5fd057fb95b","96d86c4dc6b83e5df12aa00d627f8058a8555344cac35af17e552ea297a57f64","50a8cd1183c1f1292d7c97b2bf5f3ab1a68159b23261657a2ec70d09bf72c6a3"
+"jsonlite","1.8.8","8f854ddeb6dbeb182a12aba110a2ecc9cdc2591ed5b3ec47a97ea37a5337e0b5","3306a260856cbd334bb704be62938608d97db4213e89e069001f5eaea7827630","a77e39b8f3c045b75ab4289096e5f901a638b2a765b4aa7cb2895fb84cf6731a"
+"lazyeval","0.2.2","60bc3e9b5ea101688b76ac42d17ae3673382343d035f77dff7d59b8200b8e2c8","2100fe641230e214938479bb729f4c8edf817b0aabea840a7abe1954733f843a","18d8cbe8c095ab157c2c9ad1adb41e93108f469bf61f9dcbe7e69b828c7ea857"
+"magrittr","2.0.3","75ff93b5a783cefb34da6b0892e8ef39a4662b408af4e62f9b5a4caef35f78a3","0e3f3e7ac8927dce96991d59d61c4ae6a46d6cbb6bc54ef199e7cc99ae772898","abb3d1ce74576e5c78e0554b1cc281986697172bb7d011946b02339dbffec721"
+"mime","0.12","72e0b01a104b3a26b818372de9299a7e55cbe31ce32e03025d9706d7cbce6bea","8b8d97e72ccc9b8630179ba2f23f2b4cf89f987093db63d6315ae1ad4b8e65c7","7090b7b6e503e69ec54b3d35d23d944cbf9ddf9f31938b1646b3ee329f1be7c7"
+"openssl","2.1.1","aa838b087870eb91486714754de403c8a73c0683450df1855bf1dc8bbd74cb0e","6c6336552b0d28422cb0bb29b6fb5d870c94169e9df5db8502f964613785f03d","32f28608f547df9d2cdd354f8908a7cb9ebae0f8c2baf53151d485dfceffcf75"
+"R6","2.5.1","999ce4bf2ab85a76a687bebc27d3ffcf772d89f285d3cca4dfb7a0a39e88965d","1a2266513517845867acfeb3dc81d22987fd6b182aa35cefae71ff5f03c32953","36c2e41c2bb185df8cf64faf6f21266419327cee753cc1d781bcb11fceaaeffa"
+"Rcpp","1.0.12","fa0cfd1067e95ba155a693399532413960c3d16c4272522db90f3d704045fe0f","44ac2ae5d1d26630005b653559fe64cad22db41d379e033ff59e37cefcd46961","b16ba840c3d9b5309158a41c8c237abafdbbbd41f0407f12742543f3c1e7a1f8"
+"rex","1.2.1","8343f29fe31cb34f09cd592b30e3dfd8af3758c34013556461393762cc886a72","ea7ebdfea44d6c31457f8d207244b85425d8fc7ad79774c356a77b5fe183f4b2","cce77204c173cb429cbd350a7d5cc670baace56d300c6be67eca0e0a860550c8"
+"rlang","1.1.3","fa4794ba0632b6b6e2b03e3a0301a9e4fc4449ec697f44be5b14cd76f18c334d","6425af24c86f17ad5932a941fc547ef3d40ba8558a0194dae9d55f9d4c76ed59","769b137565baa3a158352aa76d3d06117f8c5df71533fd30561ad0a8a1dcfbad"
+"sys","3.4.2","f42ea04e877a77cbc1455a11c609bcb2f48fcbd6717fcad52022ed7e254ebcbe","f8e305fb144fac318d037ebfbc329375db684c2178ac558a4b8f36570441a799","b28b3cb923569d195490589253ecac7608d6a3a620d7670a3e119a0fb6a31e25"
+"withr","2.5.2","d841c9a2e5f6d05dede85fff7cdbf54a845fcaab7e12e713aadf4495cbeb723b","978a36de37bed7c83a8d15e4176236dd1ea4283b29885b6db896f7a900e239a3","17b73d8c6c06ccc0101cdd14b416eae2856b0a206502f87eea7f03d2b5d7e068"
+"xml2","1.3.6","872b090d1871d7e9cc9d94fb421939c069c855f889a2ccba8f89fb43cfd838b7","936889f9dbdafe323120685b97dd879a3f1a981786a29d6026be81030c0d28c0","cedd1c2245d794beef8d6a0995b65580dcbc41e3c38685b0def781ad632ffcf4"
+"yaml","2.3.8","26b8dbb04cabc262067572da9f6401a55a9f8fc81536185773952a7d7649e69b","aeab7fd17010f737b8b7485a2ae34fc23354640e710ed5611cbe78928280e63d","9c75a20a4b68c76b2fe9f6225afa4c36e2a57634200f6e4e0fb81d357299c43d"
diff --git a/R/internal/library.bzl b/R/internal/library.bzl
@@ -101,11 +101,8 @@ def _r_library_tar_impl(ctx):
             for f in provider.container_file_map[layer]:
                 args.add("--file=%s=%s" % (f.path, path_prefix))
 
-    if ctx.attr.extension:
-        dotPos = ctx.attr.extension.find(".")
-        if dotPos > 0:
-            dotPos += 1
-            args.add("--compression=%s" % ctx.attr.extension[dotPos:])
+    if ctx.attr.compression:
+        args.add("--compression=%s" % ctx.attr.compression)
 
     ctx.actions.run(
         outputs = [ctx.outputs.out],
@@ -139,17 +136,17 @@ r_library_tar = rule(
             ],
             doc = "Library layers to include in the tar.",
         ),
-        "extension": attr.string(default = "tar"),
+        "compression": attr.string(default = "gz"),
         "_build_tar": attr.label(
-            default = Label("@bazel_tools//tools/build_defs/pkg:build_tar"),
+            default = Label("@rules_pkg//pkg/private/tar:build_tar"),
             cfg = "host",
             executable = True,
             allow_files = True,
         ),
     },
     doc = "Rule to create a tar archive of the files in this library.",
     outputs = {
-        "out": "%{name}.%{extension}",
+        "out": "%{name}.tar" + ".%{compression}" if "%{compression}" else "",
     },
     implementation = _r_library_tar_impl,
 )
diff --git a/scripts/razel.R b/scripts/razel.R
@@ -355,7 +355,15 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL,
     if (pkg_type == "both") {
       r_version <- R.Version()
       minor_version <- gsub("\\..*", "", r_version$minor)
-      sha256_col <- paste0("mac_", r_version$major, "_", minor_version, "_sha256")
+      sys_info <- Sys.info()
+      stopifnot(sys_info["sysname"] == "Darwin")
+      stopifnot(as.integer(sub("\\..*", "", sys_info["release"])) >= 20)
+      if (sys_info["machine"] == "arm64") {
+        prefix <- "mac_arm_"
+      } else {
+        prefix <- "mac_intel_"
+      }
+      sha256_col <- paste0(prefix, r_version$major, "_", minor_version, "_sha256")
       if (sha256_col %in% colnames(repo_pkgs)) {
         repo_pkgs[, "binary_package_available"] <- !is.na(repo_pkgs[, sha256_col])
         repo_pkgs[repo_pkgs[, "binary_package_available"], "sha256"] <-

diff --git a/scripts/repo_management.R b/scripts/repo_management.R
@@ -23,7 +23,7 @@
 # Options to also download binary archives.
 options("BinariesMac" = TRUE)  # Binaries for Mac
 options("BinariesWin" = FALSE)  # Binaries for Win
-options("RVersions" = c("3.6", "4.0", "4.1"))  # Binaries for these R versions.
+options("RVersions" = c("4.3"))  # Binaries for these R versions.
 options("ForceDownload" = FALSE)  # Download packages even if src package already present in repo.
 
 # Factors in unexpected places create problems.
@@ -35,11 +35,20 @@ srcContribDir <- function() {
 }
 
 # Returns macOS binary archive locations of repos based on R version.
-macContribDir <- function(r_version) {
+macContribDirs <- function(r_version) {
   stopifnot(r_version %in% getOption("RVersions"))
-  contrib_prefix <- ""
   if (r_version %in% c("3.4", "3.5", "3.6")) {
     contrib_prefix <- "el-capitan/"
+  } else {
+    if (r_version %in% c("4.3")) {
+      contrib_prefix <- "big-sur-x86_64/"
+    } else {
+      # Versions 4.0, 4.1 and 4.2.
+      contrib_prefix <- ""
+    }
+    if (r_version %in% c("4.2", "4.3")) {
+      contrib_prefix <- c(contrib_prefix, "big-sur-arm64/")
+    }
   }
   return(sprintf("/bin/macosx/%scontrib/%s", contrib_prefix, r_version))
 }
@@ -83,11 +92,14 @@ downloadLatestPackages <- function(pkgs, repo_dir, repos) {
     bin_repos <- repos[sapply(repos, isValidBinRepo, r_version)]
 
     if (getOption("BinariesMac")) {
-      mac_contrib_dir <- paste0(repo_dir, macContribDir(r_version))
-      dir.create(mac_contrib_dir, recursive = TRUE, showWarnings = FALSE)
-      download.packages(pkgs, destdir = mac_contrib_dir,
-                        contriburl = paste0(bin_repos, macContribDir(r_version)),
-                        type = "mac.binary")
+      mac_contrib_dirs <- macContribDirs(r_version)
+      for (mac_contrib_dir in mac_contrib_dirs) {
+        local_dir <- paste0(repo_dir, mac_contrib_dir)
+        dir.create(local_dir, recursive = TRUE, showWarnings = FALSE)
+        download.packages(pkgs, destdir = local_dir,
+                          contriburl = paste0(bin_repos, mac_contrib_dir),
+                          type = "mac.binary")
+      }
     }
 
     if (getOption("BinariesWin")) {
@@ -147,8 +159,16 @@ packageSHAs <- function(pkgs, repo_dir=".") {
   helper(file.path(repo_dir, "src", "contrib"), "sha256", ".tar.gz")
   for (r_version in getOption("RVersions")) {
     if (getOption("BinariesMac")) {
-      mac_contrib_dir <- paste0(repo_dir, macContribDir(r_version))
-      helper(mac_contrib_dir, paste0("mac_", r_version, "_sha256"), ".tgz")
+      mac_contrib_dirs <- macContribDirs(r_version)
+      for (mac_contrib_dir in mac_contrib_dirs) {
+        local_dir <- paste0(repo_dir, mac_contrib_dir)
+        if (grepl("arm64", mac_contrib_dir)) {
+          prefix <- "mac_arm_"
+        } else {
+          prefix <- "mac_intel_"
+        }
+        helper(local_dir, paste0(prefix, r_version, "_sha256"), ".tgz")
+      }
     }
     if (getOption("BinariesWin")) {
       win_contrib_dir <- paste0(repo_dir, winContribDir(r_version))
@@ -180,10 +200,12 @@ repoPackages <- function(repo_dir) {
 updateRepoIndex <- function(repo_dir) {
   tools::write_PACKAGES(paste0(repo_dir, srcContribDir()), type = "source", latestOnly = FALSE)
   for (r_version in getOption("RVersions")) {
-    macDir <- paste0(repo_dir, macContribDir(r_version))
+    macDirs <- paste0(repo_dir, macContribDirs(r_version))
     winDir <- paste0(repo_dir, winContribDir(r_version))
-    if (file.exists(macDir)) {
-      tools::write_PACKAGES(macDir, type = "mac.binary", latestOnly = FALSE)
+    for (macDir in macDirs) {
+      if (file.exists(macDir)) {
+        tools::write_PACKAGES(macDir, type = "mac.binary", latestOnly = FALSE)
+      }
     }
     if (file.exists(winDir)) {
       tools::write_PACKAGES(winDir, type = "win.binary", latestOnly = FALSE)

diff --git a/tests/.bazelrc b/tests/.bazelrc
@@ -1,7 +1,12 @@
+common --enable_bzlmod=false
+
 build --workspace_status_command=stamping/workspace_status.sh
 build --incompatible_strict_action_env
 build --java_runtime_version=remotejdk_11
 
+# Not needed after https://github.com/grailbio/bazel-toolchain/pull/229.
+build --features=-libtool
+
 build --action_env=ZERO_AR_DATE=1 # https://github.com/bazelbuild/bazel/issues/10886
 
 try-import buildbuddy.bazelrc
diff --git a/tests/BUILD b/tests/BUILD
@@ -40,6 +40,10 @@ r_toolchain(
             "@llvm_toolchain//:compiler-components-x86_64-linux",
             "@llvm_toolchain//:linker-components-x86_64-linux",
         ],
+        "@bazel_tools//src/conditions:linux_aarch64": [
+            "@llvm_toolchain//:compiler-components-aarch64-linux",
+            "@llvm_toolchain//:linker-components-aarch64-linux",
+        ],
         "//conditions:default": None,
     }),
     makevars_site = "//cran:Makevars.gnu",
@@ -70,6 +74,10 @@ r_toolchain(
             "@llvm_toolchain//:compiler-components-x86_64-darwin",
             "@llvm_toolchain//:linker-components-x86_64-darwin",
         ],
+        "@bazel_tools//src/conditions:darwin_arm64": [
+            "@llvm_toolchain//:compiler-components-aarch64-darwin",
+            "@llvm_toolchain//:linker-components-aarch64-darwin",
+        ],
         "//conditions:default": None,
     }),
     makevars_site = "@rules_r_makevars_darwin",
@@ -79,7 +87,7 @@ r_toolchain(
         "@llvm_toolchain_llvm//:bin/clang++",
         "@llvm_toolchain_llvm//:bin/llvm-cov",
     ],
-    version = "4.1",
+    version = "4.3",
     visibility = ["//visibility:public"],
 )
 
@@ -99,19 +107,17 @@ r_toolchain(
         "@Rnix//:bin/R",
         "@Rnix//:bin/Rscript",
     ],
-    version = "4.1",
+    version = "4.3",
     visibility = ["//visibility:public"],
 )
 
 toolchain(
     name = "toolchain-linux",
     exec_compatible_with = [
         "@bazel_tools//platforms:linux",
-        "@bazel_tools//platforms:x86_64",
     ],
     target_compatible_with = [
         "@bazel_tools//platforms:linux",
-        "@bazel_tools//platforms:x86_64",
     ],
     toolchain = ":r-toolchain-linux",
     toolchain_type = "@rules_r//R:toolchain_type",
@@ -122,11 +128,9 @@ toolchain(
     name = "toolchain-darwin",
     exec_compatible_with = [
         "@bazel_tools//platforms:osx",
-        "@bazel_tools//platforms:x86_64",
     ],
     target_compatible_with = [
         "@bazel_tools//platforms:osx",
-        "@bazel_tools//platforms:x86_64",
     ],
     toolchain = ":r-toolchain-darwin",
     toolchain_type = "@rules_r//R:toolchain_type",