From 592171ea5df346972c6e66d6116425288f55f2fd Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 14 Jun 2024 14:24:53 +0000 Subject: [PATCH 01/18] Implement emscripten_cache --- bazel/README.md | 13 ++++ bazel/WORKSPACE | 4 ++ bazel/emscripten_cache.bzl | 59 +++++++++++++++++++ bazel/emscripten_deps.bzl | 1 + bazel/emscripten_toolchain/BUILD.bazel | 4 +- .../{emscripten_config => default_config} | 0 6 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 bazel/emscripten_cache.bzl rename bazel/emscripten_toolchain/{emscripten_config => default_config} (100%) diff --git a/bazel/README.md b/bazel/README.md index 2fd291c328..f1758d9b05 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -18,6 +18,9 @@ emsdk_deps() load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps") emsdk_emscripten_deps(emscripten_version = "2.0.31") +load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache") +emsdk_emscripten_cache() + load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") register_emscripten_toolchains() ``` @@ -66,4 +69,14 @@ rules. and all of its dependencies, and does not require amending `.bazelrc`. This is the preferred way, since it also unpacks the resulting tarball. +The Emscripten cache shipped by default does not include LTO, 64-bit or PIC +builds of the system libraries and ports. If you wish to use these features you +will need to create a secondary cache as follows. Note that the flags are the +same flags that can be passed to embuilder. + +```starlark +load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache") +emsdk_emscripten_cache(flags = "--lto") +``` + See `test_external/` for an example using [embind](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html). diff --git a/bazel/WORKSPACE b/bazel/WORKSPACE index bd5cce5b0e..ab9206fcf3 100644 --- a/bazel/WORKSPACE +++ b/bazel/WORKSPACE @@ -8,6 +8,10 @@ load(":emscripten_deps.bzl", "emscripten_deps") emscripten_deps() +load(":emscripten_cache.bzl", "emscripten_cache") + +emscripten_cache() + load(":toolchains.bzl", "register_emscripten_toolchains") register_emscripten_toolchains() diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl new file mode 100644 index 0000000000..ca4a7bebcc --- /dev/null +++ b/bazel/emscripten_cache.bzl @@ -0,0 +1,59 @@ +BUILD_FILE_CONTENT_TEMPLATE = """ +package(default_visibility = ['//visibility:public']) +exports_files(['emscripten_config']) +""" + +def _emscripten_cache_impl(repository_ctx): + # Read the default emscripten configuration file + default_config = repository_ctx.read( + repository_ctx.path( + Label("@emsdk//emscripten_toolchain:default_config") + ) + ) + + # TODO I need a cross platform way to get embuilder and bin/node + if repository_ctx.attr.libraries or repository_ctx.attr.flags: + # Get paths to tools (toolchain is not yet setup, so we cannot use emscripten_config) + embuilder_path = repository_ctx.path(Label("@emscripten_bin_linux//:emscripten/embuilder.py")) + binaryen_root = embuilder_path.dirname.dirname + llvm_root = binaryen_root.get_child("bin") + nodejs = repository_ctx.path(Label("@nodejs//:node_files")).dirname.get_child('bin/node') + # Create configuration file + embuilder_config_content = "LLVM_ROOT = '{}'\n".format(llvm_root) + embuilder_config_content += "NODE_JS = '{}'\n".format(nodejs) + embuilder_config_content += "BINARYEN_ROOT = '{}'\n".format(binaryen_root) + embuilder_config_content += "CACHE = 'cache'\n" + repository_ctx.file('embuilder_config', embuilder_config_content) + embuilder_config_path = repository_ctx.path('embuilder_config') + # Prepare the command line + if repository_ctx.attr.libraries: + libraries = repository_ctx.attr.libraries + else: + # if no libraries are requested, build everything + libraries = ["ALL"] + flags = ["--em-config", embuilder_config_path] + repository_ctx.attr.flags + embuilder_args = [embuilder_path] + flags + ["build"] + libraries + # Run embuilder + repository_ctx.report_progress("Building secondary cache") + repository_ctx.execute(embuilder_args, quiet=False) + # Override Emscripten's cache with the secondary cache + default_config += "CACHE = '{}'\n".format(repository_ctx.path('cache')) + # Create the configuration file for the toolchain and export + repository_ctx.file('emscripten_config', default_config) + repository_ctx.file('BUILD.bazel', BUILD_FILE_CONTENT_TEMPLATE) + +_emscripten_cache = repository_rule( + implementation = _emscripten_cache_impl, + attrs = { + "flags": attr.string_list(), + "libraries": attr.string_list(), + }, + local = True +) + +def emscripten_cache(flags = [], libraries = []): + _emscripten_cache( + name = "emscripten_cache", + flags = flags, + libraries = libraries, + ) diff --git a/bazel/emscripten_deps.bzl b/bazel/emscripten_deps.bzl index 6d369453ca..16fd8ee1c6 100644 --- a/bazel/emscripten_deps.bzl +++ b/bazel/emscripten_deps.bzl @@ -27,6 +27,7 @@ filegroup( name = "emcc_common", srcs = [ "emscripten/emcc.py", + "emscripten/embuilder.py", "emscripten/emscripten-version.txt", "emscripten/cache/sysroot_install.stamp", "emscripten/src/settings.js", diff --git a/bazel/emscripten_toolchain/BUILD.bazel b/bazel/emscripten_toolchain/BUILD.bazel index fb8a6c1b2e..3cd086064e 100644 --- a/bazel/emscripten_toolchain/BUILD.bazel +++ b/bazel/emscripten_toolchain/BUILD.bazel @@ -5,7 +5,7 @@ package(default_visibility = ["//visibility:public"]) filegroup( name = "common_files", srcs = [ - "emscripten_config", + "@emscripten_cache//:emscripten_config", "env.sh", "env.bat", "@nodejs//:node_files", @@ -60,7 +60,7 @@ cc_library(name = "malloc") emscripten_cc_toolchain_config_rule( name = "wasm", cpu = "wasm", - em_config = "emscripten_config", + em_config = "@emscripten_cache//:emscripten_config", emscripten_binaries = "@emsdk//:compiler_files", script_extension = select({ "@bazel_tools//src/conditions:host_windows": "bat", diff --git a/bazel/emscripten_toolchain/emscripten_config b/bazel/emscripten_toolchain/default_config similarity index 100% rename from bazel/emscripten_toolchain/emscripten_config rename to bazel/emscripten_toolchain/default_config From ea4386a3a6ceeb8c4bbbc898e596d8bd1d8f7c42 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 14 Jun 2024 15:55:25 +0000 Subject: [PATCH 02/18] Remove hard-coded @emscripten_bin_linux// reference --- bazel/emscripten_cache.bzl | 45 ++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index ca4a7bebcc..aadd9af21b 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -3,6 +3,44 @@ package(default_visibility = ['//visibility:public']) exports_files(['emscripten_config']) """ +def get_binaryen_root(repository_ctx): + """ + Retrieve the path to the Emscripten binary directory + + This function determines the correct Emscripten binary directory path by + examining the operating system (OS) and architecture (arch) of the + environment. It supports Linux, macOS, and Windows operating systems with + specific architectures. + + Args: + repository_ctx: The repository context object which provides information + about the OS and architecture, and methods to obtain paths and labels. + + Returns: + str: The directory path to the Emscripten binaries for the detected OS + and architecture. + + """ + if repository_ctx.os.name.startswith('linux'): + if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: + return repository_ctx.path(Label("@emscripten_bin_linux//:all")).dirname + elif 'aarch64' in repository_ctx.os.arch: + return repository_ctx.path(Label("@emscripten_bin_linux_arm64//:all")).dirname + else: + repository_ctx.fail('Unsupported architecture for Linux') + elif repository_ctx.os.name.startswith('mac'): + if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: + return repository_ctx.path(Label("@emscripten_bin_mac//:all")).dirname + elif 'aarch64' in repository_ctx.os.arch: + return repository_ctx.path(Label("@emscripten_bin_mac_arm64//:all")).dirname + else: + repository_ctx.fail('Unsupported architecture for MacOS') + elif repository_ctx.os.name.startswith('windows'): + return repository_ctx.path(Label("@emscripten_bin_win//:all")).dirname + else: + repository_ctx.fail('Unsupported operating system') + return '' + def _emscripten_cache_impl(repository_ctx): # Read the default emscripten configuration file default_config = repository_ctx.read( @@ -11,11 +49,9 @@ def _emscripten_cache_impl(repository_ctx): ) ) - # TODO I need a cross platform way to get embuilder and bin/node if repository_ctx.attr.libraries or repository_ctx.attr.flags: - # Get paths to tools (toolchain is not yet setup, so we cannot use emscripten_config) - embuilder_path = repository_ctx.path(Label("@emscripten_bin_linux//:emscripten/embuilder.py")) - binaryen_root = embuilder_path.dirname.dirname + binaryen_root = get_binaryen_root(repository_ctx) + embuilder_path = binaryen_root.get_child('emscripten/embuilder') llvm_root = binaryen_root.get_child("bin") nodejs = repository_ctx.path(Label("@nodejs//:node_files")).dirname.get_child('bin/node') # Create configuration file @@ -38,6 +74,7 @@ def _emscripten_cache_impl(repository_ctx): repository_ctx.execute(embuilder_args, quiet=False) # Override Emscripten's cache with the secondary cache default_config += "CACHE = '{}'\n".format(repository_ctx.path('cache')) + # Create the configuration file for the toolchain and export repository_ctx.file('emscripten_config', default_config) repository_ctx.file('BUILD.bazel', BUILD_FILE_CONTENT_TEMPLATE) From e3ae46e6715f04404de041e6d953351c986bca25 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 14 Jun 2024 16:24:23 +0000 Subject: [PATCH 03/18] Add emscripten_cache to test_external's workspace --- bazel/test_external/WORKSPACE | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bazel/test_external/WORKSPACE b/bazel/test_external/WORKSPACE index 03f07f5070..3bd1aa1dc0 100644 --- a/bazel/test_external/WORKSPACE +++ b/bazel/test_external/WORKSPACE @@ -11,6 +11,10 @@ load("@emsdk//:emscripten_deps.bzl", "emscripten_deps") emscripten_deps() +load("@emsdk//:emscripten_cache.bzl", "emscripten_cache") + +emscripten_cache() + load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") register_emscripten_toolchains() From f77fcf41685a91732b5d90a6cf143663bf043810 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 14 Jun 2024 19:33:40 +0200 Subject: [PATCH 04/18] Add tests that generate and use a secondary cache with LTO --- bazel/test_secondary_lto_cache/.bazelrc | 1 + bazel/test_secondary_lto_cache/.gitignore | 4 ++++ bazel/test_secondary_lto_cache/BUILD | 18 +++++++++++++++++ bazel/test_secondary_lto_cache/MODULE.bazel | 1 + bazel/test_secondary_lto_cache/WORKSPACE | 20 +++++++++++++++++++ bazel/test_secondary_lto_cache/hello-world.cc | 6 ++++++ test/test_bazel.ps1 | 6 ++++++ test/test_bazel.sh | 4 ++++ test/test_bazel_mac.sh | 4 ++++ 9 files changed, 64 insertions(+) create mode 100644 bazel/test_secondary_lto_cache/.bazelrc create mode 100644 bazel/test_secondary_lto_cache/.gitignore create mode 100644 bazel/test_secondary_lto_cache/BUILD create mode 100644 bazel/test_secondary_lto_cache/MODULE.bazel create mode 100644 bazel/test_secondary_lto_cache/WORKSPACE create mode 100644 bazel/test_secondary_lto_cache/hello-world.cc diff --git a/bazel/test_secondary_lto_cache/.bazelrc b/bazel/test_secondary_lto_cache/.bazelrc new file mode 100644 index 0000000000..fbd75a7ea7 --- /dev/null +++ b/bazel/test_secondary_lto_cache/.bazelrc @@ -0,0 +1 @@ +build --incompatible_enable_cc_toolchain_resolution diff --git a/bazel/test_secondary_lto_cache/.gitignore b/bazel/test_secondary_lto_cache/.gitignore new file mode 100644 index 0000000000..3e67f7873e --- /dev/null +++ b/bazel/test_secondary_lto_cache/.gitignore @@ -0,0 +1,4 @@ +bazel-bin +bazel-out +bazel-test_external +bazel-testlogs \ No newline at end of file diff --git a/bazel/test_secondary_lto_cache/BUILD b/bazel/test_secondary_lto_cache/BUILD new file mode 100644 index 0000000000..ae7f925df3 --- /dev/null +++ b/bazel/test_secondary_lto_cache/BUILD @@ -0,0 +1,18 @@ +load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") + +cc_binary( + name = "hello-world", + srcs = ["hello-world.cc"], + linkopts = ["-flto"], +) + +wasm_cc_binary( + name = "hello-world-wasm", + cc_target = ":hello-world", + outputs = [ + "hello-world.js", + "hello-world.wasm", + ], +) + + diff --git a/bazel/test_secondary_lto_cache/MODULE.bazel b/bazel/test_secondary_lto_cache/MODULE.bazel new file mode 100644 index 0000000000..7f51cebf45 --- /dev/null +++ b/bazel/test_secondary_lto_cache/MODULE.bazel @@ -0,0 +1 @@ +bazel_dep(name = "platforms", version = "0.0.9") diff --git a/bazel/test_secondary_lto_cache/WORKSPACE b/bazel/test_secondary_lto_cache/WORKSPACE new file mode 100644 index 0000000000..53b0fc22ff --- /dev/null +++ b/bazel/test_secondary_lto_cache/WORKSPACE @@ -0,0 +1,20 @@ +local_repository( + name = "emsdk", + path = "..", +) + +load("@emsdk//:deps.bzl", "deps") + +deps() + +load("@emsdk//:emscripten_deps.bzl", "emscripten_deps") + +emscripten_deps() + +load("@emsdk//:emscripten_cache.bzl", "emscripten_cache") + +emscripten_cache(flags = "--lto") + +load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") + +register_emscripten_toolchains() diff --git a/bazel/test_secondary_lto_cache/hello-world.cc b/bazel/test_secondary_lto_cache/hello-world.cc new file mode 100644 index 0000000000..ee72c53171 --- /dev/null +++ b/bazel/test_secondary_lto_cache/hello-world.cc @@ -0,0 +1,6 @@ +#include + +int main(int argc, char** argv) { + std::cout << "hello world!" << std::endl; + return 0; +} diff --git a/test/test_bazel.ps1 b/test/test_bazel.ps1 index b417262e27..5c20a15251 100644 --- a/test/test_bazel.ps1 +++ b/test/test_bazel.ps1 @@ -22,3 +22,9 @@ if (-not $?) { Exit $LastExitCode } # Test use of the closure compiler bazel build //:hello-embind-wasm --compilation_mode opt # release if (-not $?) { Exit $LastExitCode } + +Set-Location ..\test_secondary_lto_cache + +bazel build //:hello-world-wasm +if (-not $?) { Exit $LastExitCode } + diff --git a/test/test_bazel.sh b/test/test_bazel.sh index d8d40e1716..da29588842 100755 --- a/test/test_bazel.sh +++ b/test/test_bazel.sh @@ -33,3 +33,7 @@ bazel build //:hello-embind-wasm --compilation_mode dbg # debug bazel build //:hello-embind-wasm --compilation_mode opt # release # This function should not be minified if the externs file is loaded correctly. grep "customJSFunctionToTestClosure" bazel-bin/hello-embind-wasm/hello-embind.js + +cd ../test_secondary_lto_cache +bazel build //:hello-world-wasm + diff --git a/test/test_bazel_mac.sh b/test/test_bazel_mac.sh index 4b58b502bd..9077873eab 100755 --- a/test/test_bazel_mac.sh +++ b/test/test_bazel_mac.sh @@ -27,3 +27,7 @@ bazel build //hello-world:hello-world-wasm-simd cd test_external bazel build //long_command_line:long_command_line_wasm bazel build //:hello-world-wasm + +cd ../test_secondary_lto_cache +bazel build //:hello-world-wasm + From 4856c766585d11f92a1f96f3be6ff464fc1ce97a Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 14 Jun 2024 17:40:23 +0000 Subject: [PATCH 05/18] Fix flags argument to emscripten_cache in test --- bazel/test_secondary_lto_cache/WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/test_secondary_lto_cache/WORKSPACE b/bazel/test_secondary_lto_cache/WORKSPACE index 53b0fc22ff..39948b2cd5 100644 --- a/bazel/test_secondary_lto_cache/WORKSPACE +++ b/bazel/test_secondary_lto_cache/WORKSPACE @@ -13,7 +13,7 @@ emscripten_deps() load("@emsdk//:emscripten_cache.bzl", "emscripten_cache") -emscripten_cache(flags = "--lto") +emscripten_cache(flags = ["--lto"]) load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") From 2818b5a6319e60f19207a518dfb856b601ef475d Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Sat, 15 Jun 2024 13:00:35 +0000 Subject: [PATCH 06/18] Fix error handling --- bazel/emscripten_cache.bzl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index aadd9af21b..ccfe0c8d8a 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -27,19 +27,18 @@ def get_binaryen_root(repository_ctx): elif 'aarch64' in repository_ctx.os.arch: return repository_ctx.path(Label("@emscripten_bin_linux_arm64//:all")).dirname else: - repository_ctx.fail('Unsupported architecture for Linux') + fail('Unsupported architecture for Linux') elif repository_ctx.os.name.startswith('mac'): if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: return repository_ctx.path(Label("@emscripten_bin_mac//:all")).dirname elif 'aarch64' in repository_ctx.os.arch: return repository_ctx.path(Label("@emscripten_bin_mac_arm64//:all")).dirname else: - repository_ctx.fail('Unsupported architecture for MacOS') + fail('Unsupported architecture for MacOS') elif repository_ctx.os.name.startswith('windows'): return repository_ctx.path(Label("@emscripten_bin_win//:all")).dirname else: - repository_ctx.fail('Unsupported operating system') - return '' + fail('Unsupported operating system') def _emscripten_cache_impl(repository_ctx): # Read the default emscripten configuration file @@ -71,7 +70,9 @@ def _emscripten_cache_impl(repository_ctx): embuilder_args = [embuilder_path] + flags + ["build"] + libraries # Run embuilder repository_ctx.report_progress("Building secondary cache") - repository_ctx.execute(embuilder_args, quiet=False) + result = repository_ctx.execute(embuilder_args, quiet=False) + if result != 0: + fail("Embuilder exited with a non-zero return code") # Override Emscripten's cache with the secondary cache default_config += "CACHE = '{}'\n".format(repository_ctx.path('cache')) From f3f2c7ec849bdd18955a3cf029feaab36a4ff17a Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Sat, 15 Jun 2024 16:54:28 +0000 Subject: [PATCH 07/18] Fix various errors 1. Use an absolute path for CACHE in embuilder_config 2. Fix various issues with error handling (fail, return_code, etc) --- bazel/emscripten_cache.bzl | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index ccfe0c8d8a..82946659b9 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -50,16 +50,19 @@ def _emscripten_cache_impl(repository_ctx): if repository_ctx.attr.libraries or repository_ctx.attr.flags: binaryen_root = get_binaryen_root(repository_ctx) - embuilder_path = binaryen_root.get_child('emscripten/embuilder') llvm_root = binaryen_root.get_child("bin") - nodejs = repository_ctx.path(Label("@nodejs//:node_files")).dirname.get_child('bin/node') + emscripten_root = binaryen_root.get_child("emscripten") + embuilder_path = emscripten_root.get_child("embuilder") + cache_path = repository_ctx.path('cache') + nodejs = repository_ctx.path(Label("@nodejs//:node_files")).dirname.get_child("bin/node") # Create configuration file - embuilder_config_content = "LLVM_ROOT = '{}'\n".format(llvm_root) - embuilder_config_content += "NODE_JS = '{}'\n".format(nodejs) + embuilder_config_content = "NODE_JS = '{}'\n".format(nodejs) + embuilder_config_content += "LLVM_ROOT = '{}'\n".format(llvm_root) embuilder_config_content += "BINARYEN_ROOT = '{}'\n".format(binaryen_root) - embuilder_config_content += "CACHE = 'cache'\n" - repository_ctx.file('embuilder_config', embuilder_config_content) - embuilder_config_path = repository_ctx.path('embuilder_config') + embuilder_config_content += "EMSCRIPTEN_ROOT = '{}'\n".format(emscripten_root) + embuilder_config_content += "CACHE = '{}'\n".format(cache_path) + repository_ctx.file("embuilder_config", embuilder_config_content) + embuilder_config_path = repository_ctx.path("embuilder_config") # Prepare the command line if repository_ctx.attr.libraries: libraries = repository_ctx.attr.libraries @@ -70,11 +73,11 @@ def _emscripten_cache_impl(repository_ctx): embuilder_args = [embuilder_path] + flags + ["build"] + libraries # Run embuilder repository_ctx.report_progress("Building secondary cache") - result = repository_ctx.execute(embuilder_args, quiet=False) - if result != 0: + result = repository_ctx.execute(embuilder_args, quiet=True) + if result.return_code != 0: fail("Embuilder exited with a non-zero return code") # Override Emscripten's cache with the secondary cache - default_config += "CACHE = '{}'\n".format(repository_ctx.path('cache')) + default_config += "CACHE = '{}'\n".format(cache_path) # Create the configuration file for the toolchain and export repository_ctx.file('emscripten_config', default_config) From 75656b1a9787875ae59ffb49f7842575a51a2b7f Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Sat, 15 Jun 2024 17:54:49 +0000 Subject: [PATCH 08/18] Multiple fixes 1. Use a real file in get_binaryen_root so that dirname works under Windows 2. Enable embuilders output for debugging purposes --- bazel/emscripten_cache.bzl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index 82946659b9..e9996277f9 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -8,9 +8,9 @@ def get_binaryen_root(repository_ctx): Retrieve the path to the Emscripten binary directory This function determines the correct Emscripten binary directory path by - examining the operating system (OS) and architecture (arch) of the - environment. It supports Linux, macOS, and Windows operating systems with - specific architectures. + examining the operating system and architecture of the environment. It + supports Linux, macOS, and Windows operating systems with specific + architectures. Args: repository_ctx: The repository context object which provides information @@ -23,20 +23,20 @@ def get_binaryen_root(repository_ctx): """ if repository_ctx.os.name.startswith('linux'): if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_linux//:all")).dirname + return repository_ctx.path(Label("@emscripten_bin_linux//:emscripten_config")).dirname elif 'aarch64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_linux_arm64//:all")).dirname + return repository_ctx.path(Label("@emscripten_bin_linux_arm64//:emscripten_config")).dirname else: fail('Unsupported architecture for Linux') elif repository_ctx.os.name.startswith('mac'): if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_mac//:all")).dirname + return repository_ctx.path(Label("@emscripten_bin_mac//:emscripten_config")).dirname elif 'aarch64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_mac_arm64//:all")).dirname + return repository_ctx.path(Label("@emscripten_bin_mac_arm64//:emscripten_config")).dirname else: fail('Unsupported architecture for MacOS') elif repository_ctx.os.name.startswith('windows'): - return repository_ctx.path(Label("@emscripten_bin_win//:all")).dirname + return repository_ctx.path(Label("@emscripten_bin_win//:emscripten_config")).dirname else: fail('Unsupported operating system') @@ -73,7 +73,7 @@ def _emscripten_cache_impl(repository_ctx): embuilder_args = [embuilder_path] + flags + ["build"] + libraries # Run embuilder repository_ctx.report_progress("Building secondary cache") - result = repository_ctx.execute(embuilder_args, quiet=True) + result = repository_ctx.execute(embuilder_args, quiet=False) if result.return_code != 0: fail("Embuilder exited with a non-zero return code") # Override Emscripten's cache with the secondary cache From d5e279579c7da4fd3e3fd76cf03a4fcc17acb2dd Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Sat, 15 Jun 2024 17:57:04 +0000 Subject: [PATCH 09/18] Make tests faster by not building the entire cache --- bazel/test_secondary_lto_cache/BUILD | 5 ++++- bazel/test_secondary_lto_cache/WORKSPACE | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/bazel/test_secondary_lto_cache/BUILD b/bazel/test_secondary_lto_cache/BUILD index ae7f925df3..9a4462bca2 100644 --- a/bazel/test_secondary_lto_cache/BUILD +++ b/bazel/test_secondary_lto_cache/BUILD @@ -3,7 +3,10 @@ load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") cc_binary( name = "hello-world", srcs = ["hello-world.cc"], - linkopts = ["-flto"], + linkopts = [ + "-sAUTO_NATIVE_LIBRARIES=0", + "-flto", + ], ) wasm_cc_binary( diff --git a/bazel/test_secondary_lto_cache/WORKSPACE b/bazel/test_secondary_lto_cache/WORKSPACE index 39948b2cd5..f79a3110f1 100644 --- a/bazel/test_secondary_lto_cache/WORKSPACE +++ b/bazel/test_secondary_lto_cache/WORKSPACE @@ -13,7 +13,21 @@ emscripten_deps() load("@emsdk//:emscripten_cache.bzl", "emscripten_cache") -emscripten_cache(flags = ["--lto"]) +emscripten_cache( + flags = ["--lto"], + libraries = [ + "crtbegin", + "libprintf_long_double-debug", + "libstubs-debug", + "libnoexit", + "libc-debug", + "libdlmalloc", + "libcompiler_rt", + "libc++-noexcept", + "libc++abi-debug-noexcept", + "libsockets" + ] +) load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") From abfa9ce17baa0586ad7fbf66abf19370960ea023 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Sat, 15 Jun 2024 18:18:50 +0000 Subject: [PATCH 10/18] Use BUILD.bazel to get the root directory of repositories --- bazel/emscripten_cache.bzl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index e9996277f9..81db5b300c 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -23,20 +23,20 @@ def get_binaryen_root(repository_ctx): """ if repository_ctx.os.name.startswith('linux'): if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_linux//:emscripten_config")).dirname + return repository_ctx.path(Label("@emscripten_bin_linux//:BUILD.bazel")).dirname elif 'aarch64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_linux_arm64//:emscripten_config")).dirname + return repository_ctx.path(Label("@emscripten_bin_linux_arm64//:BUILD.bazel")).dirname else: fail('Unsupported architecture for Linux') elif repository_ctx.os.name.startswith('mac'): if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_mac//:emscripten_config")).dirname + return repository_ctx.path(Label("@emscripten_bin_mac//:BUILD.bazel")).dirname elif 'aarch64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_mac_arm64//:emscripten_config")).dirname + return repository_ctx.path(Label("@emscripten_bin_mac_arm64//:BUILD.bazel")).dirname else: fail('Unsupported architecture for MacOS') elif repository_ctx.os.name.startswith('windows'): - return repository_ctx.path(Label("@emscripten_bin_win//:emscripten_config")).dirname + return repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname else: fail('Unsupported operating system') @@ -54,7 +54,7 @@ def _emscripten_cache_impl(repository_ctx): emscripten_root = binaryen_root.get_child("emscripten") embuilder_path = emscripten_root.get_child("embuilder") cache_path = repository_ctx.path('cache') - nodejs = repository_ctx.path(Label("@nodejs//:node_files")).dirname.get_child("bin/node") + nodejs = repository_ctx.path(Label("@nodejs//:BUILD.bazel")).dirname.get_child("bin").get_child("node") # Create configuration file embuilder_config_content = "NODE_JS = '{}'\n".format(nodejs) embuilder_config_content += "LLVM_ROOT = '{}'\n".format(llvm_root) From 311c58764d45a99765b2d65fd30f3e2a4f64cf39 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Sat, 15 Jun 2024 18:45:55 +0000 Subject: [PATCH 11/18] Include the .exe extension on Windows for Node.js --- bazel/emscripten_cache.bzl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index 81db5b300c..16565bacb6 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -3,7 +3,7 @@ package(default_visibility = ['//visibility:public']) exports_files(['emscripten_config']) """ -def get_binaryen_root(repository_ctx): +def get_root_and_binext(repository_ctx): """ Retrieve the path to the Emscripten binary directory @@ -23,20 +23,20 @@ def get_binaryen_root(repository_ctx): """ if repository_ctx.os.name.startswith('linux'): if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_linux//:BUILD.bazel")).dirname + return (repository_ctx.path(Label("@emscripten_bin_linux//:BUILD.bazel")).dirname, '') elif 'aarch64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_linux_arm64//:BUILD.bazel")).dirname + return (repository_ctx.path(Label("@emscripten_bin_linux_arm64//:BUILD.bazel")).dirname, '') else: fail('Unsupported architecture for Linux') elif repository_ctx.os.name.startswith('mac'): if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_mac//:BUILD.bazel")).dirname + return (repository_ctx.path(Label("@emscripten_bin_mac//:BUILD.bazel")).dirname, '') elif 'aarch64' in repository_ctx.os.arch: - return repository_ctx.path(Label("@emscripten_bin_mac_arm64//:BUILD.bazel")).dirname + return (repository_ctx.path(Label("@emscripten_bin_mac_arm64//:BUILD.bazel")).dirname, '') else: fail('Unsupported architecture for MacOS') elif repository_ctx.os.name.startswith('windows'): - return repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname + return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.exe') else: fail('Unsupported operating system') @@ -49,16 +49,16 @@ def _emscripten_cache_impl(repository_ctx): ) if repository_ctx.attr.libraries or repository_ctx.attr.flags: - binaryen_root = get_binaryen_root(repository_ctx) - llvm_root = binaryen_root.get_child("bin") - emscripten_root = binaryen_root.get_child("emscripten") + root, binext = get_root_and_binext(repository_ctx) + llvm_root = root.get_child("bin") + emscripten_root = root.get_child("emscripten") embuilder_path = emscripten_root.get_child("embuilder") cache_path = repository_ctx.path('cache') nodejs = repository_ctx.path(Label("@nodejs//:BUILD.bazel")).dirname.get_child("bin").get_child("node") # Create configuration file - embuilder_config_content = "NODE_JS = '{}'\n".format(nodejs) + embuilder_config_content = "NODE_JS = '{}{}'\n".format(nodejs, binext) embuilder_config_content += "LLVM_ROOT = '{}'\n".format(llvm_root) - embuilder_config_content += "BINARYEN_ROOT = '{}'\n".format(binaryen_root) + embuilder_config_content += "BINARYEN_ROOT = '{}'\n".format(root) embuilder_config_content += "EMSCRIPTEN_ROOT = '{}'\n".format(emscripten_root) embuilder_config_content += "CACHE = '{}'\n".format(cache_path) repository_ctx.file("embuilder_config", embuilder_config_content) From e6b94bcb546fd211e0ea3b27eb647866ad44eef9 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Sun, 16 Jun 2024 12:32:11 +0000 Subject: [PATCH 12/18] Drop support for secondary caches on Windows --- bazel/emscripten_cache.bzl | 59 ++++++++++++++++++++------------------ test/test_bazel.ps1 | 6 ++-- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index 16565bacb6..dd4f5186a1 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -3,24 +3,21 @@ package(default_visibility = ['//visibility:public']) exports_files(['emscripten_config']) """ -def get_root_and_binext(repository_ctx): - """ - Retrieve the path to the Emscripten binary directory +EMBUILDER_FILE_CONTENT_TEMPLATE = """ +CACHE = '{cache}' +EMSCRIPTEN_ROOT = '{emscripten_root}' +BINARYEN_ROOT = '{binaryen_root}' +LLVM_ROOT = '{llvm_root}' - This function determines the correct Emscripten binary directory path by - examining the operating system and architecture of the environment. It - supports Linux, macOS, and Windows operating systems with specific - architectures. +import platform - Args: - repository_ctx: The repository context object which provides information - about the OS and architecture, and methods to obtain paths and labels. - - Returns: - str: The directory path to the Emscripten binaries for the detected OS - and architecture. +system = platform.system() +machine = "arm64" if platform.machine() in ('arm64', 'aarch64') else "amd64" +nodejs_binary = "bin/nodejs/node.exe" if(system =="Windows") else "bin/node" +NODE_JS = '{external_root}/nodejs_{{}}_{{}}/{{}}'.format(system.lower(), machine, nodejs_binary) +""" - """ +def get_root_and_script_ext(repository_ctx): if repository_ctx.os.name.startswith('linux'): if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: return (repository_ctx.path(Label("@emscripten_bin_linux//:BUILD.bazel")).dirname, '') @@ -36,7 +33,8 @@ def get_root_and_binext(repository_ctx): else: fail('Unsupported architecture for MacOS') elif repository_ctx.os.name.startswith('windows'): - return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.exe') + fail('Using a secondary cache is not supported on Windows') + #return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.bat') else: fail('Unsupported operating system') @@ -49,35 +47,40 @@ def _emscripten_cache_impl(repository_ctx): ) if repository_ctx.attr.libraries or repository_ctx.attr.flags: - root, binext = get_root_and_binext(repository_ctx) + root, script_ext = get_root_and_script_ext(repository_ctx) llvm_root = root.get_child("bin") emscripten_root = root.get_child("emscripten") - embuilder_path = emscripten_root.get_child("embuilder") - cache_path = repository_ctx.path('cache') - nodejs = repository_ctx.path(Label("@nodejs//:BUILD.bazel")).dirname.get_child("bin").get_child("node") + cache = repository_ctx.path("cache") + # Ugly hack to get the "external" directory (needed for Windows/Node.js) + external_root = repository_ctx.path(Label("@nodejs//:BUILD.bazel")).dirname.dirname # Create configuration file - embuilder_config_content = "NODE_JS = '{}{}'\n".format(nodejs, binext) - embuilder_config_content += "LLVM_ROOT = '{}'\n".format(llvm_root) - embuilder_config_content += "BINARYEN_ROOT = '{}'\n".format(root) - embuilder_config_content += "EMSCRIPTEN_ROOT = '{}'\n".format(emscripten_root) - embuilder_config_content += "CACHE = '{}'\n".format(cache_path) + embuilder_config_content = EMBUILDER_FILE_CONTENT_TEMPLATE.format( + cache=cache, + emscripten_root=emscripten_root, + binaryen_root=root, + llvm_root=llvm_root, + external_root=external_root, + ) repository_ctx.file("embuilder_config", embuilder_config_content) embuilder_config_path = repository_ctx.path("embuilder_config") + embuilder_path = "{}{}".format(emscripten_root.get_child("embuilder"), script_ext) # Prepare the command line if repository_ctx.attr.libraries: libraries = repository_ctx.attr.libraries else: - # if no libraries are requested, build everything + # If no libraries are requested, build everything libraries = ["ALL"] flags = ["--em-config", embuilder_config_path] + repository_ctx.attr.flags embuilder_args = [embuilder_path] + flags + ["build"] + libraries # Run embuilder repository_ctx.report_progress("Building secondary cache") - result = repository_ctx.execute(embuilder_args, quiet=False) + result = repository_ctx.execute(embuilder_args, quiet=True) if result.return_code != 0: + # Windows fails here because external/nodejs_windows_amd64/bin/nodejs/node.exe + # does not exist at this point (while the equivalent on Linux and MacOS does) fail("Embuilder exited with a non-zero return code") # Override Emscripten's cache with the secondary cache - default_config += "CACHE = '{}'\n".format(cache_path) + default_config += "CACHE = '{}'\n".format(cache) # Create the configuration file for the toolchain and export repository_ctx.file('emscripten_config', default_config) diff --git a/test/test_bazel.ps1 b/test/test_bazel.ps1 index 5c20a15251..1c19b5c852 100644 --- a/test/test_bazel.ps1 +++ b/test/test_bazel.ps1 @@ -23,8 +23,8 @@ if (-not $?) { Exit $LastExitCode } bazel build //:hello-embind-wasm --compilation_mode opt # release if (-not $?) { Exit $LastExitCode } -Set-Location ..\test_secondary_lto_cache +# Set-Location ..\test_secondary_lto_cache -bazel build //:hello-world-wasm -if (-not $?) { Exit $LastExitCode } +# bazel build //:hello-world-wasm +# if (-not $?) { Exit $LastExitCode } From 0f584ea29ab4f9746d8b31898c174289fefcfd22 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 21 Jun 2024 07:41:40 +0000 Subject: [PATCH 13/18] Update .gitignore for secondary cache test --- bazel/test_secondary_lto_cache/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/test_secondary_lto_cache/.gitignore b/bazel/test_secondary_lto_cache/.gitignore index 3e67f7873e..9d50f0bb1e 100644 --- a/bazel/test_secondary_lto_cache/.gitignore +++ b/bazel/test_secondary_lto_cache/.gitignore @@ -1,4 +1,4 @@ bazel-bin bazel-out -bazel-test_external +bazel-test_secondary_lto_cache bazel-testlogs \ No newline at end of file From c0242f8e65b4136092f361626694672ea8e05909 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 21 Jun 2024 08:13:41 +0000 Subject: [PATCH 14/18] Remove dependency on Node.js --- bazel/emscripten_cache.bzl | 33 ++++++++++++--------------------- test/test_bazel.ps1 | 6 +++--- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index dd4f5186a1..83fc936c8e 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -3,18 +3,10 @@ package(default_visibility = ['//visibility:public']) exports_files(['emscripten_config']) """ -EMBUILDER_FILE_CONTENT_TEMPLATE = """ +EMBUILDER_CONFIG_TEMPLATE = """ CACHE = '{cache}' -EMSCRIPTEN_ROOT = '{emscripten_root}' BINARYEN_ROOT = '{binaryen_root}' LLVM_ROOT = '{llvm_root}' - -import platform - -system = platform.system() -machine = "arm64" if platform.machine() in ('arm64', 'aarch64') else "amd64" -nodejs_binary = "bin/nodejs/node.exe" if(system =="Windows") else "bin/node" -NODE_JS = '{external_root}/nodejs_{{}}_{{}}/{{}}'.format(system.lower(), machine, nodejs_binary) """ def get_root_and_script_ext(repository_ctx): @@ -33,8 +25,7 @@ def get_root_and_script_ext(repository_ctx): else: fail('Unsupported architecture for MacOS') elif repository_ctx.os.name.startswith('windows'): - fail('Using a secondary cache is not supported on Windows') - #return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.bat') + return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.bat') else: fail('Unsupported operating system') @@ -49,21 +40,16 @@ def _emscripten_cache_impl(repository_ctx): if repository_ctx.attr.libraries or repository_ctx.attr.flags: root, script_ext = get_root_and_script_ext(repository_ctx) llvm_root = root.get_child("bin") - emscripten_root = root.get_child("emscripten") cache = repository_ctx.path("cache") - # Ugly hack to get the "external" directory (needed for Windows/Node.js) - external_root = repository_ctx.path(Label("@nodejs//:BUILD.bazel")).dirname.dirname # Create configuration file - embuilder_config_content = EMBUILDER_FILE_CONTENT_TEMPLATE.format( + embuilder_config_content = EMBUILDER_CONFIG_TEMPLATE.format( cache=cache, - emscripten_root=emscripten_root, binaryen_root=root, llvm_root=llvm_root, - external_root=external_root, ) repository_ctx.file("embuilder_config", embuilder_config_content) embuilder_config_path = repository_ctx.path("embuilder_config") - embuilder_path = "{}{}".format(emscripten_root.get_child("embuilder"), script_ext) + embuilder_path = "{}{}".format(root.get_child("emscripten").get_child("embuilder"), script_ext) # Prepare the command line if repository_ctx.attr.libraries: libraries = repository_ctx.attr.libraries @@ -74,10 +60,15 @@ def _emscripten_cache_impl(repository_ctx): embuilder_args = [embuilder_path] + flags + ["build"] + libraries # Run embuilder repository_ctx.report_progress("Building secondary cache") - result = repository_ctx.execute(embuilder_args, quiet=True) + result = repository_ctx.execute( + embuilder_args, + quiet=False, + environment = { + "EM_IGNORE_SANITY": "1", + "EM_NODE_JS": "empty", + } + ) if result.return_code != 0: - # Windows fails here because external/nodejs_windows_amd64/bin/nodejs/node.exe - # does not exist at this point (while the equivalent on Linux and MacOS does) fail("Embuilder exited with a non-zero return code") # Override Emscripten's cache with the secondary cache default_config += "CACHE = '{}'\n".format(cache) diff --git a/test/test_bazel.ps1 b/test/test_bazel.ps1 index 1c19b5c852..5c20a15251 100644 --- a/test/test_bazel.ps1 +++ b/test/test_bazel.ps1 @@ -23,8 +23,8 @@ if (-not $?) { Exit $LastExitCode } bazel build //:hello-embind-wasm --compilation_mode opt # release if (-not $?) { Exit $LastExitCode } -# Set-Location ..\test_secondary_lto_cache +Set-Location ..\test_secondary_lto_cache -# bazel build //:hello-world-wasm -# if (-not $?) { Exit $LastExitCode } +bazel build //:hello-world-wasm +if (-not $?) { Exit $LastExitCode } From 263d6b1a314e1804ff7a3f9817059e1191e395f3 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 21 Jun 2024 08:28:10 +0000 Subject: [PATCH 15/18] Hide output while building the cache --- bazel/emscripten_cache.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl index 83fc936c8e..b1dcebabac 100644 --- a/bazel/emscripten_cache.bzl +++ b/bazel/emscripten_cache.bzl @@ -62,7 +62,7 @@ def _emscripten_cache_impl(repository_ctx): repository_ctx.report_progress("Building secondary cache") result = repository_ctx.execute( embuilder_args, - quiet=False, + quiet=True, environment = { "EM_IGNORE_SANITY": "1", "EM_NODE_JS": "empty", From cde46af2bfae69f4569550d7df76c6acd9fb574c Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 21 Jun 2024 08:33:42 +0000 Subject: [PATCH 16/18] Fix typo in README.md --- bazel/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/README.md b/bazel/README.md index f1758d9b05..6143a54fc9 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -76,7 +76,7 @@ same flags that can be passed to embuilder. ```starlark load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache") -emsdk_emscripten_cache(flags = "--lto") +emsdk_emscripten_cache(flags = ["--lto"]) ``` See `test_external/` for an example using [embind](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html). From 312260f61d7918917e431141155ff0d76120c880 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Sat, 22 Jun 2024 12:20:15 +0000 Subject: [PATCH 17/18] Move cache declaration inside of the toolchain registration --- bazel/README.md | 27 +++++-- bazel/WORKSPACE | 4 - bazel/emscripten_cache.bzl | 94 ----------------------- bazel/test_external/WORKSPACE | 4 - bazel/test_secondary_lto_cache/WORKSPACE | 14 ++-- bazel/toolchains.bzl | 96 +++++++++++++++++++++++- 6 files changed, 120 insertions(+), 119 deletions(-) delete mode 100644 bazel/emscripten_cache.bzl diff --git a/bazel/README.md b/bazel/README.md index 6143a54fc9..ce8d210b78 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -18,9 +18,6 @@ emsdk_deps() load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps") emsdk_emscripten_deps(emscripten_version = "2.0.31") -load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache") -emsdk_emscripten_cache() - load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") register_emscripten_toolchains() ``` @@ -71,12 +68,28 @@ is the preferred way, since it also unpacks the resulting tarball. The Emscripten cache shipped by default does not include LTO, 64-bit or PIC builds of the system libraries and ports. If you wish to use these features you -will need to create a secondary cache as follows. Note that the flags are the -same flags that can be passed to embuilder. +will need to declare the cache when you register the toolchain as follows. Note +that the configuration consists of the same flags that can be passed to +embuilder. If `targets` is not provided, all system libraries and ports will be +built, i.e., the `ALL` option to embuilder. ```starlark -load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache") -emsdk_emscripten_cache(flags = ["--lto"]) +load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") +register_emscripten_toolchains(cache = { + "configuration": ["--lto"], + "targets": [ + "crtbegin", + "libprintf_long_double-debug", + "libstubs-debug", + "libnoexit", + "libc-debug", + "libdlmalloc", + "libcompiler_rt", + "libc++-noexcept", + "libc++abi-debug-noexcept", + "libsockets" + ] +}) ``` See `test_external/` for an example using [embind](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html). diff --git a/bazel/WORKSPACE b/bazel/WORKSPACE index ab9206fcf3..bd5cce5b0e 100644 --- a/bazel/WORKSPACE +++ b/bazel/WORKSPACE @@ -8,10 +8,6 @@ load(":emscripten_deps.bzl", "emscripten_deps") emscripten_deps() -load(":emscripten_cache.bzl", "emscripten_cache") - -emscripten_cache() - load(":toolchains.bzl", "register_emscripten_toolchains") register_emscripten_toolchains() diff --git a/bazel/emscripten_cache.bzl b/bazel/emscripten_cache.bzl deleted file mode 100644 index b1dcebabac..0000000000 --- a/bazel/emscripten_cache.bzl +++ /dev/null @@ -1,94 +0,0 @@ -BUILD_FILE_CONTENT_TEMPLATE = """ -package(default_visibility = ['//visibility:public']) -exports_files(['emscripten_config']) -""" - -EMBUILDER_CONFIG_TEMPLATE = """ -CACHE = '{cache}' -BINARYEN_ROOT = '{binaryen_root}' -LLVM_ROOT = '{llvm_root}' -""" - -def get_root_and_script_ext(repository_ctx): - if repository_ctx.os.name.startswith('linux'): - if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: - return (repository_ctx.path(Label("@emscripten_bin_linux//:BUILD.bazel")).dirname, '') - elif 'aarch64' in repository_ctx.os.arch: - return (repository_ctx.path(Label("@emscripten_bin_linux_arm64//:BUILD.bazel")).dirname, '') - else: - fail('Unsupported architecture for Linux') - elif repository_ctx.os.name.startswith('mac'): - if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: - return (repository_ctx.path(Label("@emscripten_bin_mac//:BUILD.bazel")).dirname, '') - elif 'aarch64' in repository_ctx.os.arch: - return (repository_ctx.path(Label("@emscripten_bin_mac_arm64//:BUILD.bazel")).dirname, '') - else: - fail('Unsupported architecture for MacOS') - elif repository_ctx.os.name.startswith('windows'): - return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.bat') - else: - fail('Unsupported operating system') - -def _emscripten_cache_impl(repository_ctx): - # Read the default emscripten configuration file - default_config = repository_ctx.read( - repository_ctx.path( - Label("@emsdk//emscripten_toolchain:default_config") - ) - ) - - if repository_ctx.attr.libraries or repository_ctx.attr.flags: - root, script_ext = get_root_and_script_ext(repository_ctx) - llvm_root = root.get_child("bin") - cache = repository_ctx.path("cache") - # Create configuration file - embuilder_config_content = EMBUILDER_CONFIG_TEMPLATE.format( - cache=cache, - binaryen_root=root, - llvm_root=llvm_root, - ) - repository_ctx.file("embuilder_config", embuilder_config_content) - embuilder_config_path = repository_ctx.path("embuilder_config") - embuilder_path = "{}{}".format(root.get_child("emscripten").get_child("embuilder"), script_ext) - # Prepare the command line - if repository_ctx.attr.libraries: - libraries = repository_ctx.attr.libraries - else: - # If no libraries are requested, build everything - libraries = ["ALL"] - flags = ["--em-config", embuilder_config_path] + repository_ctx.attr.flags - embuilder_args = [embuilder_path] + flags + ["build"] + libraries - # Run embuilder - repository_ctx.report_progress("Building secondary cache") - result = repository_ctx.execute( - embuilder_args, - quiet=True, - environment = { - "EM_IGNORE_SANITY": "1", - "EM_NODE_JS": "empty", - } - ) - if result.return_code != 0: - fail("Embuilder exited with a non-zero return code") - # Override Emscripten's cache with the secondary cache - default_config += "CACHE = '{}'\n".format(cache) - - # Create the configuration file for the toolchain and export - repository_ctx.file('emscripten_config', default_config) - repository_ctx.file('BUILD.bazel', BUILD_FILE_CONTENT_TEMPLATE) - -_emscripten_cache = repository_rule( - implementation = _emscripten_cache_impl, - attrs = { - "flags": attr.string_list(), - "libraries": attr.string_list(), - }, - local = True -) - -def emscripten_cache(flags = [], libraries = []): - _emscripten_cache( - name = "emscripten_cache", - flags = flags, - libraries = libraries, - ) diff --git a/bazel/test_external/WORKSPACE b/bazel/test_external/WORKSPACE index 3bd1aa1dc0..03f07f5070 100644 --- a/bazel/test_external/WORKSPACE +++ b/bazel/test_external/WORKSPACE @@ -11,10 +11,6 @@ load("@emsdk//:emscripten_deps.bzl", "emscripten_deps") emscripten_deps() -load("@emsdk//:emscripten_cache.bzl", "emscripten_cache") - -emscripten_cache() - load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") register_emscripten_toolchains() diff --git a/bazel/test_secondary_lto_cache/WORKSPACE b/bazel/test_secondary_lto_cache/WORKSPACE index f79a3110f1..d0bdd3b4c0 100644 --- a/bazel/test_secondary_lto_cache/WORKSPACE +++ b/bazel/test_secondary_lto_cache/WORKSPACE @@ -11,11 +11,11 @@ load("@emsdk//:emscripten_deps.bzl", "emscripten_deps") emscripten_deps() -load("@emsdk//:emscripten_cache.bzl", "emscripten_cache") +load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") -emscripten_cache( - flags = ["--lto"], - libraries = [ +register_emscripten_toolchains(cache = { + "configuration": ["--lto"], + "targets": [ "crtbegin", "libprintf_long_double-debug", "libstubs-debug", @@ -27,8 +27,4 @@ emscripten_cache( "libc++abi-debug-noexcept", "libsockets" ] -) - -load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains") - -register_emscripten_toolchains() +}) diff --git a/bazel/toolchains.bzl b/bazel/toolchains.bzl index 6d50b3d3d5..ae71413dfd 100644 --- a/bazel/toolchains.bzl +++ b/bazel/toolchains.bzl @@ -1,2 +1,96 @@ -def register_emscripten_toolchains(): +BUILD_FILE_CONTENT_TEMPLATE = """ +package(default_visibility = ['//visibility:public']) +exports_files(['emscripten_config']) +""" + +EMBUILDER_CONFIG_TEMPLATE = """ +CACHE = '{cache}' +BINARYEN_ROOT = '{binaryen_root}' +LLVM_ROOT = '{llvm_root}' +""" + +def get_root_and_script_ext(repository_ctx): + if repository_ctx.os.name.startswith('linux'): + if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: + return (repository_ctx.path(Label("@emscripten_bin_linux//:BUILD.bazel")).dirname, '') + elif 'aarch64' in repository_ctx.os.arch: + return (repository_ctx.path(Label("@emscripten_bin_linux_arm64//:BUILD.bazel")).dirname, '') + else: + fail('Unsupported architecture for Linux') + elif repository_ctx.os.name.startswith('mac'): + if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch: + return (repository_ctx.path(Label("@emscripten_bin_mac//:BUILD.bazel")).dirname, '') + elif 'aarch64' in repository_ctx.os.arch: + return (repository_ctx.path(Label("@emscripten_bin_mac_arm64//:BUILD.bazel")).dirname, '') + else: + fail('Unsupported architecture for MacOS') + elif repository_ctx.os.name.startswith('windows'): + return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.bat') + else: + fail('Unsupported operating system') + +def _emscripten_cache_impl(repository_ctx): + # Read the default emscripten configuration file + default_config = repository_ctx.read( + repository_ctx.path( + Label("@emsdk//emscripten_toolchain:default_config") + ) + ) + + if repository_ctx.attr.targets or repository_ctx.attr.configuration: + root, script_ext = get_root_and_script_ext(repository_ctx) + llvm_root = root.get_child("bin") + cache = repository_ctx.path("cache") + # Create configuration file + embuilder_config_content = EMBUILDER_CONFIG_TEMPLATE.format( + cache=cache, + binaryen_root=root, + llvm_root=llvm_root, + ) + repository_ctx.file("embuilder_config", embuilder_config_content) + embuilder_config_path = repository_ctx.path("embuilder_config") + embuilder_path = "{}{}".format(root.get_child("emscripten").get_child("embuilder"), script_ext) + # Prepare the command line + if repository_ctx.attr.targets: + targets = repository_ctx.attr.targets + else: + # If no targets are requested, build everything + targets = ["ALL"] + flags = ["--em-config", embuilder_config_path] + repository_ctx.attr.configuration + embuilder_args = [embuilder_path] + flags + ["build"] + targets + # Run embuilder + repository_ctx.report_progress("Building secondary cache") + result = repository_ctx.execute( + embuilder_args, + quiet=True, + environment = { + "EM_IGNORE_SANITY": "1", + "EM_NODE_JS": "empty", + } + ) + if result.return_code != 0: + fail("Embuilder exited with a non-zero return code") + # Override Emscripten's cache with the secondary cache + default_config += "CACHE = '{}'\n".format(cache) + + # Create the configuration file for the toolchain and export + repository_ctx.file('emscripten_config', default_config) + repository_ctx.file('BUILD.bazel', BUILD_FILE_CONTENT_TEMPLATE) + +_emscripten_cache = repository_rule( + implementation = _emscripten_cache_impl, + attrs = { + "configuration": attr.string_list(), + "targets": attr.string_list(), + }, + local = True +) + +def register_emscripten_toolchains(cache = {}): + _emscripten_cache( + name = "emscripten_cache", + configuration = cache["configuration"] if "configuration" in cache else [], + targets = cache["targets"] if "targets" in cache else [], + ) + native.register_toolchains(str(Label("//emscripten_toolchain:cc-toolchain-wasm"))) From 8f9e1557d186272f17d13040136a16bd257a590b Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Tue, 25 Jun 2024 09:00:39 +0200 Subject: [PATCH 18/18] Remove trailing spaces --- bazel/test_secondary_lto_cache/BUILD | 2 -- 1 file changed, 2 deletions(-) diff --git a/bazel/test_secondary_lto_cache/BUILD b/bazel/test_secondary_lto_cache/BUILD index 9a4462bca2..f364f3778f 100644 --- a/bazel/test_secondary_lto_cache/BUILD +++ b/bazel/test_secondary_lto_cache/BUILD @@ -17,5 +17,3 @@ wasm_cc_binary( "hello-world.wasm", ], ) - -