[Bazel] Support generating a secondary cache

bazel/README.md

@@ -18,6 +18,9 @@ emsdk_deps()
 load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps")
 emsdk_emscripten_deps(emscripten_version = "2.0.31")
 
+load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache")
+emsdk_emscripten_cache()
+
 load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains")
 register_emscripten_toolchains()
 ```
@@ -66,4 +69,14 @@ rules.
 and all of its dependencies, and does not require amending `.bazelrc`. This
 is the preferred way, since it also unpacks the resulting tarball.
 
+The Emscripten cache shipped by default does not include LTO, 64-bit or PIC
+builds of the system libraries and ports. If you wish to use these features you
+will need to create a secondary cache as follows. Note that the flags are the
+same flags that can be passed to embuilder.
+
+```starlark
+load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache")
+emsdk_emscripten_cache(flags = "--lto")
+```
+
 See `test_external/` for an example using [embind](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html).

bazel/WORKSPACE

@@ -8,6 +8,10 @@ load(":emscripten_deps.bzl", "emscripten_deps")
 
 emscripten_deps()
 
+load(":emscripten_cache.bzl", "emscripten_cache")
+
+emscripten_cache()
+
 load(":toolchains.bzl", "register_emscripten_toolchains")
 
 register_emscripten_toolchains()

bazel/emscripten_cache.bzl

@@ -0,0 +1,103 @@
+BUILD_FILE_CONTENT_TEMPLATE = """
+package(default_visibility = ['//visibility:public'])
+exports_files(['emscripten_config'])
+"""
+
+EMBUILDER_FILE_CONTENT_TEMPLATE = """
+CACHE = '{cache}'
+EMSCRIPTEN_ROOT = '{emscripten_root}'
+BINARYEN_ROOT = '{binaryen_root}'
+LLVM_ROOT = '{llvm_root}'
+
+import platform
+
+system = platform.system()
+machine = "arm64" if platform.machine() in ('arm64', 'aarch64') else "amd64"
+nodejs_binary = "bin/nodejs/node.exe" if(system =="Windows") else "bin/node"
+NODE_JS = '{external_root}/nodejs_{{}}_{{}}/{{}}'.format(system.lower(), machine, nodejs_binary)
+"""
+
+def get_root_and_script_ext(repository_ctx):
+    if repository_ctx.os.name.startswith('linux'):
+        if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch:
+            return (repository_ctx.path(Label("@emscripten_bin_linux//:BUILD.bazel")).dirname, '')
+        elif 'aarch64' in repository_ctx.os.arch:
+            return (repository_ctx.path(Label("@emscripten_bin_linux_arm64//:BUILD.bazel")).dirname, '')
+        else:
+            fail('Unsupported architecture for Linux')
+    elif repository_ctx.os.name.startswith('mac'):
+        if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch:
+            return (repository_ctx.path(Label("@emscripten_bin_mac//:BUILD.bazel")).dirname, '')
+        elif 'aarch64' in repository_ctx.os.arch:
+            return (repository_ctx.path(Label("@emscripten_bin_mac_arm64//:BUILD.bazel")).dirname, '')
+        else:
+            fail('Unsupported architecture for MacOS')
+    elif repository_ctx.os.name.startswith('windows'):
+        fail('Using a secondary cache is not supported on Windows')
+        #return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.bat')
+    else:
+        fail('Unsupported operating system')
+
+def _emscripten_cache_impl(repository_ctx):
+    # Read the default emscripten configuration file
+    default_config = repository_ctx.read(
+        repository_ctx.path(
+            Label("@emsdk//emscripten_toolchain:default_config")
+        )
+    )
+
+    if repository_ctx.attr.libraries or repository_ctx.attr.flags:
+        root, script_ext = get_root_and_script_ext(repository_ctx)
+        llvm_root = root.get_child("bin")
+        emscripten_root = root.get_child("emscripten")
+        cache = repository_ctx.path("cache")
+        # Ugly hack to get the "external" directory (needed for Windows/Node.js)
+        external_root = repository_ctx.path(Label("@nodejs//:BUILD.bazel")).dirname.dirname
+        # Create configuration file
+        embuilder_config_content = EMBUILDER_FILE_CONTENT_TEMPLATE.format(
+            cache=cache,
+            emscripten_root=emscripten_root,
+            binaryen_root=root,
+            llvm_root=llvm_root,
+            external_root=external_root,
+        )
+        repository_ctx.file("embuilder_config", embuilder_config_content)
+        embuilder_config_path = repository_ctx.path("embuilder_config")
+        embuilder_path = "{}{}".format(emscripten_root.get_child("embuilder"), script_ext)
+        # Prepare the command line
+        if repository_ctx.attr.libraries:
+            libraries = repository_ctx.attr.libraries
+        else:
+            # If no libraries are requested, build everything
+            libraries = ["ALL"]
+        flags = ["--em-config", embuilder_config_path] + repository_ctx.attr.flags
+        embuilder_args = [embuilder_path] + flags + ["build"] + libraries
+        # Run embuilder
+        repository_ctx.report_progress("Building secondary cache")
+        result = repository_ctx.execute(embuilder_args, quiet=True)
+        if result.return_code != 0:
+            # Windows fails here because external/nodejs_windows_amd64/bin/nodejs/node.exe
+            # does not exist at this point (while the equivalent on Linux and MacOS does)
+            fail("Embuilder exited with a non-zero return code")
+        # Override Emscripten's cache with the secondary cache
+        default_config += "CACHE = '{}'\n".format(cache)
+
+    # Create the configuration file for the toolchain and export
+    repository_ctx.file('emscripten_config', default_config)
+    repository_ctx.file('BUILD.bazel', BUILD_FILE_CONTENT_TEMPLATE)
+
+_emscripten_cache = repository_rule(
+    implementation = _emscripten_cache_impl,
+    attrs = {
+        "flags": attr.string_list(),
+        "libraries": attr.string_list(),
+    },
+    local = True
+)
+
+def emscripten_cache(flags = [], libraries = []):
+    _emscripten_cache(
+        name = "emscripten_cache",
+        flags = flags,
+        libraries = libraries,
+    )

bazel/emscripten_deps.bzl

@@ -27,6 +27,7 @@ filegroup(
     name = "emcc_common",
     srcs = [
         "emscripten/emcc.py",
+        "emscripten/embuilder.py",
         "emscripten/emscripten-version.txt",
         "emscripten/cache/sysroot_install.stamp",
         "emscripten/src/settings.js",

bazel/emscripten_toolchain/BUILD.bazel

@@ -5,7 +5,7 @@ package(default_visibility = ["//visibility:public"])
 filegroup(
     name = "common_files",
     srcs = [
-        "emscripten_config",
+        "@emscripten_cache//:emscripten_config",
         "env.sh",
         "env.bat",
         "@nodejs//:node_files",
@@ -60,7 +60,7 @@ cc_library(name = "malloc")
 emscripten_cc_toolchain_config_rule(
     name = "wasm",
     cpu = "wasm",
-    em_config = "emscripten_config",
+    em_config = "@emscripten_cache//:emscripten_config",
     emscripten_binaries = "@emsdk//:compiler_files",
     script_extension = select({
         "@bazel_tools//src/conditions:host_windows": "bat",

bazel/test_external/WORKSPACE

@@ -11,6 +11,10 @@ load("@emsdk//:emscripten_deps.bzl", "emscripten_deps")
 
 emscripten_deps()
 
+load("@emsdk//:emscripten_cache.bzl", "emscripten_cache")
+
+emscripten_cache()
+
 load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains")
 
 register_emscripten_toolchains()

bazel/test_secondary_lto_cache/.bazelrc

@@ -0,0 +1 @@
+build --incompatible_enable_cc_toolchain_resolution

bazel/test_secondary_lto_cache/.gitignore

@@ -0,0 +1,4 @@
+bazel-bin
+bazel-out
+bazel-test_external
+bazel-testlogs

bazel/test_secondary_lto_cache/BUILD

@@ -0,0 +1,21 @@
+load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary")
+
+cc_binary(
+    name = "hello-world",
+    srcs = ["hello-world.cc"],
+    linkopts = [
+        "-sAUTO_NATIVE_LIBRARIES=0",
+        "-flto",
+    ],
+)
+
+wasm_cc_binary(
+    name = "hello-world-wasm",
+    cc_target = ":hello-world",
+    outputs = [
+        "hello-world.js",
+        "hello-world.wasm",
+    ],
+)
+
+

bazel/test_secondary_lto_cache/MODULE.bazel

@@ -0,0 +1 @@
+bazel_dep(name = "platforms", version = "0.0.9")

bazel/test_secondary_lto_cache/WORKSPACE

@@ -0,0 +1,34 @@
+local_repository(
+    name = "emsdk",
+    path = "..",
+)
+
+load("@emsdk//:deps.bzl", "deps")
+
+deps()
+
+load("@emsdk//:emscripten_deps.bzl", "emscripten_deps")
+
+emscripten_deps()
+
+load("@emsdk//:emscripten_cache.bzl", "emscripten_cache")
+
+emscripten_cache(
+    flags = ["--lto"],
+    libraries = [
+        "crtbegin",
+        "libprintf_long_double-debug",
+        "libstubs-debug",
+        "libnoexit",
+        "libc-debug",
+        "libdlmalloc",
+        "libcompiler_rt",
+        "libc++-noexcept",
+        "libc++abi-debug-noexcept",
+        "libsockets"
+    ]
+)
+
+load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains")
+
+register_emscripten_toolchains()

bazel/test_secondary_lto_cache/hello-world.cc

@@ -0,0 +1,6 @@
+#include <iostream>
+
+int main(int argc, char** argv) {
+  std::cout << "hello world!" << std::endl;
+  return 0;
+}

test/test_bazel.ps1

@@ -22,3 +22,9 @@ if (-not $?) { Exit $LastExitCode }
 # Test use of the closure compiler
 bazel build //:hello-embind-wasm --compilation_mode opt # release
 if (-not $?) { Exit $LastExitCode }
+
+# Set-Location ..\test_secondary_lto_cache
+
+# bazel build //:hello-world-wasm
+# if (-not $?) { Exit $LastExitCode }
+

test/test_bazel.sh

@@ -33,3 +33,7 @@ bazel build //:hello-embind-wasm --compilation_mode dbg # debug
 bazel build //:hello-embind-wasm --compilation_mode opt # release
 # This function should not be minified if the externs file is loaded correctly.
 grep "customJSFunctionToTestClosure" bazel-bin/hello-embind-wasm/hello-embind.js
+
+cd ../test_secondary_lto_cache
+bazel build //:hello-world-wasm
+

test/test_bazel_mac.sh

@@ -27,3 +27,7 @@ bazel build //hello-world:hello-world-wasm-simd
 cd test_external
 bazel build //long_command_line:long_command_line_wasm
 bazel build //:hello-world-wasm
+
+cd ../test_secondary_lto_cache
+bazel build //:hello-world-wasm
+