Skip to content

Commit

Permalink
CUS-349: CI: enable remote execution in jobs that can use it
Browse files Browse the repository at this point in the history
Added platform and C++ toolchain configurations for platforms we use
that have configured remote execution pools. This is mostly copied
from github.com/EngFlow/examples.

In .bazelrc, added platform configurations like remote_linux_x64.
One of these needs to be specified when using remote execution.
Also split the opal and opal_bes_only configurations because not all
jobs can use remote execution.

In workflows, added --config=<remote_platform> and replaced the
CLUSTER_CONFIG environment variable with either opal or opal_bes_only
as appropriate. The jobs driven with 'bazel run' cannot use remote
execution because that always runs on the host, so they're opal_bes_only.
  • Loading branch information
jayconrod committed Aug 30, 2024
1 parent cb34acd commit a5a3523
Show file tree
Hide file tree
Showing 13 changed files with 10,634 additions and 24 deletions.
36 changes: 31 additions & 5 deletions .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,49 @@ build:noninteractive --keep_going
build:release --config=noninteractive
build:release --stamp

# Platform-specific options for each supported platform.
build:remote_linux_x64 --extra_execution_platforms=//platform/linux_x64
build:remote_linux_x64 --extra_toolchains=//platform/linux_x64:cc-toolchain
build:remote_linux_x64 --host_platform=//platform/linux_x64
build:remote_linux_x64 --platforms=//platform/linux_x64

build:remote_macos_x64 --extra_execution_platforms=//platform/macos_x64
build:remote_macos_x64 --host_action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
build:remote_macos_x64 --host_platform=//platform/macos_x64
build:remote_macos_x64 --macos_minimum_os=12
build:remote_macos_x64 --platforms=//platform/macos_x64

build:remote_windows_x64 --extra_execution_platforms=//platform/windows_x64
build:remote_windows_x64 --extra_toolchains=//platform/windows_x64:cc-toolchain
build:remote_windows_x64 --host_platform=//platform/windows_x64
build:remote_windows_x64 --platforms=//platform/windows_x64

# Options for EngFlow remote execution.
build:engflow_common --jobs=40
build:engflow_common --define=EXECUTOR=remote
build:engflow_common --experimental_inmemory_dotd_files
build:engflow_common --experimental_inmemory_jdeps_files
build:engflow_common --incompatible_strict_action_env=true
build:engflow_common --remote_timeout=600
build:engflow_common --legacy_important_outputs=false
build:engflow_common --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
build:engflow_common --grpc_keepalive_time=30s
build:engflow_common --experimental_remote_cache_compression=true
build:engflow_common --remote_instance_name=auth
build:engflow_common --bes_instance_name=auth
build:engflow_common --bes_lifecycle_events

build:opal_bes --bes_backend=grpcs://opal.cluster.engflow.com
build:opal_bes --bes_results_url=https://opal.cluster.engflow.com/invocations/auth/

build:opal_bes_only --config=engflow_common
build:opal_bes_only --config=opal_auth
build:opal_bes_only --config=opal_bes

build:opal --config=engflow_common
build:opal --bes_backend=grpcs://opal.cluster.engflow.com
build:opal --bes_results_url=https://opal.cluster.engflow.com/invocations/auth/
# TODO(CUS-349): enable remote execution
build:opal --config=opal_auth
build:opal --config=opal_bes
build:opal --remote_executor=grpcs://opal.cluster.engflow.com

# Load authentication flags for the remote service, if any.
# To authenticate with the clusters above, create a .bazelrc.user file.
# This file is not checked into Git because it refers to local file system paths.
try-import .bazelrc.user
15 changes: 6 additions & 9 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@ env:
# Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467
BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CLUSTER_HOST: opal.cluster.engflow.com
CLUSTER_CONFIG: opal
CRED_HELPER_TOKEN: ${{ secrets.OPAL_CRED_HELPER_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
OPAL_RPC_CREDENTIALS: ${{ secrets.OPAL_RPC_CREDENTIALS }}

jobs:
# TODO(OND-616): use remote execution and caching for all CI Runner jobs.
Expand Down Expand Up @@ -59,8 +57,7 @@ jobs:
- name: Run all tests
if: success()
run: |
# TODO(CUS-345): Enable remote execution
bazel test --config=noninteractive --config="${CLUSTER_CONFIG}" //...
bazel test --config=noninteractive --config=opal --config=remote_linux_x64 //...
- name: Log out
run: infra/logout.sh
Expand Down Expand Up @@ -106,8 +103,8 @@ jobs:
shell: bash
run: |
# TODO(CUS-345): Enable remote execution
bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- test ./...
bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- clean -cache -modcache
bazel run --config=noninteractive --config=opal_bes_only @rules_go//go -- test ./...
bazel run --config=noninteractive --config=opal_bes_only @rules_go//go -- clean -cache -modcache
- name: Log out
shell: bash
Expand Down Expand Up @@ -138,8 +135,8 @@ jobs:
if: success()
run: |
# TODO(CUS-345): Enable remote execution
bazel run --config=noninteractive @rules_go//go -- test ./...
bazel run --config=noninteractive @rules_go//go -- clean -cache -modcache
bazel run --config=noninteractive --config=opal_bes_only @rules_go//go -- test ./...
bazel run --config=noninteractive --config=opal_bes_only @rules_go//go -- clean -cache -modcache
- name: Log out
run: infra/logout.sh
Expand Down Expand Up @@ -168,7 +165,7 @@ jobs:
- name: Check copyright headers
if: success()
run: |
bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" //infra/internal/check_copyright_headers
bazel run --config=noninteractive --config=opal_bes_only //infra/internal/check_copyright_headers
- name: Log out
run: infra/logout.sh
19 changes: 10 additions & 9 deletions .github/workflows/presubmit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@ env:
# Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467
BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CLUSTER_HOST: opal.cluster.engflow.com
CLUSTER_CONFIG: opal
CRED_HELPER_TOKEN: ${{ secrets.OPAL_CRED_HELPER_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
# TODO(OND-616): use remote execution and caching for all CI Runner jobs.
bazel-builder:
runs-on:
- self-hosted
Expand All @@ -60,8 +58,11 @@ jobs:
- name: Run all tests
if: success()
run: |
# TODO(CUS-345): Enable remote execution
bazel test --config=noninteractive --config="${CLUSTER_CONFIG}" //...
bazel test \
--config=noninteractive \
--config=opal \
--config=remote_linux_x64 \
//...
- name: Log out
run: infra/logout.sh
Expand Down Expand Up @@ -107,8 +108,8 @@ jobs:
shell: bash
run: |
# TODO(CUS-345): Enable remote execution
bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- test ./...
bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- clean -cache -modcache
bazel run --config=noninteractive --config=opal_bes_only @rules_go//go -- test ./...
bazel run --config=noninteractive --config=opal_bes_only @rules_go//go -- clean -cache -modcache
- name: Log out
shell: bash
Expand Down Expand Up @@ -139,8 +140,8 @@ jobs:
if: success()
run: |
# TODO(CUS-345): Enable remote execution
bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- test ./...
bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- clean -cache -modcache
bazel run --config=noninteractive --config=opal_bes_only @rules_go//go -- test ./...
bazel run --config=noninteractive --config=opal_bes_only @rules_go//go -- clean -cache -modcache
- name: Log out
run: infra/logout.sh
Expand Down Expand Up @@ -169,7 +170,7 @@ jobs:
- name: Check copyright headers
if: success()
run: |
bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" //infra/internal/check_copyright_headers
bazel run --config=noninteractive --config=opal_bes_only //infra/internal/check_copyright_headers
- name: Log out
run: infra/logout.sh
1 change: 0 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ permissions:
contents: write

jobs:
# TODO(OND-616): use remote execution and caching for all CI Runner jobs.
release:
runs-on:
- self-hosted
Expand Down
2 changes: 2 additions & 0 deletions infra/release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ echo "[START] Building artifacts"
BUILD_RELEASE_VERSION="${RELEASE_VERSION}" \
bazel build \
--config=release \
--config=opal \
--config=remote_linux_x64 \
-- \
//:release_artifacts
echo "[FINISH] Building artifacts"
Expand Down
143 changes: 143 additions & 0 deletions platform/linux_x64/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
load("@rules_cc//cc:defs.bzl", "cc_toolchain", "cc_toolchain_suite")
load(":cc_toolchain_config.bzl", "cc_toolchain_config")

package(default_visibility = ["//visibility:public"])

platform(
name = "linux_x64",
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
"@bazel_tools//tools/cpp:clang",
"@rules_go//go/toolchain:cgo_on",
],
exec_properties = {
"container-image": "docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-re/debian12-x64@sha256:b636ba1e2fd5da78372596e9e882a6a19e8931b52a7ec37e17587d84f1fb8e76",
"Pool": "linux_x64",
},
)

toolchain(
name = "cc-toolchain",
exec_compatible_with = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
target_compatible_with = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
toolchain = ":cc-compiler-k8",
toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
)

cc_toolchain_suite(
name = "toolchain",
tags = ["manual"],
toolchains = {
"k8|gcc": ":cc-compiler-k8",
"k8": ":cc-compiler-k8",
},
)

cc_toolchain(
name = "cc-compiler-k8",
all_files = ":linux_gnu_x86",
ar_files = ":linux_gnu_x86",
as_files = ":linux_gnu_x86",
compiler_files = ":linux_gnu_x86",
dwp_files = ":linux_gnu_x86",
linker_files = ":linux_gnu_x86",
module_map = None,
objcopy_files = ":linux_gnu_x86",
strip_files = ":linux_gnu_x86y",
supports_param_files = 1,
toolchain_config = ":linux_gnu_x86",
toolchain_identifier = "linux_gnu_x86",
)

cc_toolchain_config(
name = "linux_gnu_x86",
abi_libc_version = "glibc_2.19",
abi_version = "gcc",
compile_flags = [
"-fstack-protector",
"-Wall",
"-Wunused-but-set-parameter",
"-Wno-free-nonheap-object",
"-fno-omit-frame-pointer",
],
compiler = "gcc",
coverage_compile_flags = ["--coverage"],
coverage_link_flags = ["--coverage"],
cpu = "k8",
cxx_builtin_include_directories = [
"/usr/lib/gcc/x86_64-linux-gnu/10/include",
"/usr/local/include",
"/usr/include/x86_64-linux-gnu",
"/usr/include",
"/usr/include/c++/10",
"/usr/include/x86_64-linux-gnu/c++/10",
"/usr/include/c++/10/backward",
],
cxx_flags = ["-std=c++14"],
dbg_compile_flags = ["-g"],
host_system_name = "i686-unknown-linux-gnu",
link_flags = [
"-fuse-ld=gold",
"-Wl,-no-as-needed",
"-Wl,-z,relro,-z,now",
"-B/usr/bin",
"-pass-exit-codes",
],
link_libs = [
"-lstdc++",
"-lm",
],
opt_compile_flags = [
"-g0",
"-O2",
"-D_FORTIFY_SOURCE=1",
"-DNDEBUG",
"-ffunction-sections",
"-fdata-sections",
],
opt_link_flags = ["-Wl,--gc-sections"],
supports_start_end_lib = True,
target_libc = "glibc_2.19",
target_system_name = "x86_64-unknown-linux-gnu",
tool_paths = {
"ar": "/usr/bin/ar",
"ld": "/usr/bin/ld",
"llvm-cov": "None",
"llvm-profdata": "None",
"cpp": "/usr/bin/cpp",
"gcc": "/usr/bin/gcc",
"dwp": "/usr/bin/dwp",
"gcov": "/usr/bin/gcov",
"nm": "/usr/bin/nm",
"objcopy": "/usr/bin/objcopy",
"objdump": "/usr/bin/objdump",
"strip": "/usr/bin/strip",
},
toolchain_identifier = "linux_gnu_x86",
unfiltered_compile_flags = [
"-fno-canonical-system-headers",
"-Wno-builtin-macro-redefined",
"-D__DATE__=\"redacted\"",
"-D__TIMESTAMP__=\"redacted\"",
"-D__TIME__=\"redacted\"",
],
)

filegroup(
name = "compiler_deps",
srcs = glob(
["extra_tools/**"],
allow_empty = True,
) + [":builtin_include_directory_paths"],
)

filegroup(
name = "empty",
)
Loading

0 comments on commit a5a3523

Please sign in to comment.