diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b22c8f654..6ec789d231 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,16 +34,17 @@ jobs: - checkout - run: name: Install debian packages - command: apt-get update -q && apt-get install -q -y cmake build-essential openjdk-8-jre-headless - - run: scripts/test.sh + command: apt-get update -q && apt-get install -q -y cmake build-essential openjdk-8-jre-headless ksh zsh + - run: test/test.sh + - run: test/test_source_env.sh - run: name: test.py command: | source emsdk_env.sh - scripts/test.py + test/test.py test-mac: macos: - xcode: "9.0" + xcode: "12.2.0" environment: EMSDK_NOTTY: "1" # Without this, any `brew installl` command will result in self-update of @@ -54,15 +55,12 @@ jobs: - run: name: Install cmake command: brew install cmake - - run: - name: Install python 3 - command: brew install python3 - - run: scripts/test.sh + - run: test/test.sh - run: name: test.py command: | source emsdk_env.sh - scripts/test.py + test/test.py test-windows: executor: name: win/vs2019 @@ -83,12 +81,125 @@ jobs: - run: name: Install latest shell: cmd.exe - command: scripts\test.bat + command: test\test.bat - run: name: test.py command: | source emsdk_env.sh - python scripts/test.py + python test/test.py + + - run: + name: flagless (process/shell) test + shell: powershell.exe + command: | + test/test_activation.ps1 + + - run: + name: --permanent test + shell: powershell.exe + command: | + $env:PERMANENT_FLAG="--permanent" + test/test_activation.ps1 + + - run: + name: --system test + shell: powershell.exe + command: | + $env:SYSTEM_FLAG="--system" + test/test_activation.ps1 + + - run: + name: Process/Shell PATH preservation test + shell: powershell.exe + command: | + test/test_path_preservation.ps1 + + - run: + name: User PATH preservation test + shell: powershell.exe + command: | + $env:PERMANENT_FLAG="--permanent" + test/test_path_preservation.ps1 + + - run: + name: System PATH preservation test + shell: powershell.exe + command: | + $env:SYSTEM_FLAG="--system" + test/test_path_preservation.ps1 + + build-docker-image: + executor: bionic + steps: + - checkout + - run: + name: install docker + command: apt-get update -q && apt-get install -q -y docker.io + - setup_remote_docker: + version: 19.03.13 + # Build and test the tip-of-tree build of EMSDK + - run: + name: build + command: make -C ./docker version=tot build + - run: + name: test + command: make -C ./docker version=tot test + + publish-docker-image: + executor: bionic + steps: + - checkout + - run: + name: install docker + command: apt-get update -q && apt-get install -q -y docker.io + - setup_remote_docker: + version: 19.03.13 + - run: + name: build + command: make -C ./docker version=${CIRCLE_TAG} build + - run: + name: test + command: make -C ./docker version=${CIRCLE_TAG} test + - run: + name: push image + command: | + docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" + make -C ./docker version=${CIRCLE_TAG} alias=latest push + + test-bazel-linux: + executor: bionic + steps: + - checkout + - run: apt-get install -q -y curl gnupg + - run: curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg + - run: mv bazel.gpg /etc/apt/trusted.gpg.d/ + - run: echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list + - run: + name: install pip + command: | + apt-get update -q + apt-get install -q -y python3-pip + - run: pip3 install absl-py + - run: + name: install bazel + command: | + apt-get install -q -y bazel + - run: test/test_bazel.sh + + test-bazel-mac: + macos: + xcode: "12.2.0" + environment: + EMSDK_NOTTY: "1" + HOMEBREW_NO_AUTO_UPDATE: "1" + steps: + - checkout + - run: brew install grep + - run: + name: install bazel + command: | + brew install bazel + - run: test/test_bazel_mac.sh workflows: flake8: @@ -103,3 +214,18 @@ workflows: test-windows: jobs: - test-windows + build-docker-image: + jobs: + - build-docker-image + - publish-docker-image: + filters: + branches: + ignore: /.*/ + tags: + only: /.*/ + test-bazel-linux: + jobs: + - test-bazel-linux + test-bazel-mac: + jobs: + - test-bazel-mac diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..dfeb00e532 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +# Ignore all subdirectories +*/* + +# Allow to run the test script inside the Docker container +!/docker/test_dockerimage.sh + +# Ignore unnecessary files inside top-level directory +*.bat +*.csh +*.fish +*.ps1 +*.pyc +.emscripten +.emscripten.old +.emscripten_cache +.emscripten_cache__last_clear +.emscripten_sanity +.emscripten_sanity_wasm +.flake8 +emscripten-releases-tot.txt +README.md diff --git a/.flake8 b/.flake8 index 31829b56d9..b67d9c2198 100644 --- a/.flake8 +++ b/.flake8 @@ -6,6 +6,7 @@ ignore = E121, # Continuation line under-indented for hanging indent E722 # bare excepts exclude = + ./llvm ./gnu ./upstream ./fastcomp diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 0000000000..1807a00f71 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,18 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 365 +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 30 +# Issues with these labels will never be considered stale +exemptLabels: + - pinned + - security +# Label to use when marking an issue as stale +staleLabel: wontfix +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because there has been no + activity in the past year. It will be closed automatically if no further + activity occurs in the next 30 days. Feel free to re-open at any time if this + issue is still relevant. +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: false diff --git a/.gitignore b/.gitignore index 75afb979b6..229c0b6ba9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,10 +9,6 @@ __pycache__ /.emscripten_sanity /.emscripten_sanity_wasm -# Auto-generated by `active` -/emsdk_set_env.bat -/emsdk_set_env.sh - # Tags files that get generated at runtime /emscripten-releases-tot.txt diff --git a/README.md b/README.md index 1e6fe00401..3d10f87613 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,8 @@ https://emscripten.org/docs/building_from_source/toolchain_what_is_needed.html. ### Mac OS X -- `python`: Version 2.7.0 or above. +- For Intel-based Macs, macOS 10.13 or newer. For ARM64 M1 based Macs, macOS + 11.0 or newer. - `java`: For running closure compiler (optional). After installing emscripten via emsdk, typing 'emcc --help' should pop up a OS X dialog "Java is not installed. To open java, you need a Java SE 6 runtime. Would you like to @@ -117,8 +118,7 @@ SDK from the local hard drive completely. `emsdk update` will fetch package information for all the new tools and SDK versions. After that, run `emsdk install ` to install a new -version. The command `emsdk update-tags` obtains a list of all new tagged -releases from GitHub without updating Emscripten SDK itself. +version. ### How do I install an old Emscripten compiler version? @@ -139,8 +139,8 @@ categories. To obtain and build latest upstream wasm SDK from source, run ``` -emsdk install sdk-upstream-master-64bit -emsdk activate sdk-upstream-master-64bit +emsdk install sdk-upstream-main-64bit +emsdk activate sdk-upstream-main-64bit ``` You can use this target for example to bootstrap developing patches to LLVM, @@ -156,8 +156,8 @@ https://emscripten.org/docs/contributing/developers_guide.html?highlight=develop ### When working on git branches compiled from source, how do I update to a newer compiler version? Unlike tags and precompiled versions, a few of the SDK packages are based on -"moving" git branches and compiled from source (e.g. sdk-upstream-master, -sdk-master, emscripten-master, binaryen-master). Because of that, the +"moving" git branches and compiled from source (e.g. sdk-upstream-main, +sdk-main, emscripten-main, binaryen-main). Because of that, the compiled versions will eventually go out of date as new commits are introduced to the development branches. To update an old compiled installation of one of this branches, simply reissue the "emsdk install" command on that tool/SDK. This @@ -176,9 +176,8 @@ reissuing `emsdk install`. You can toggle between different tools and SDK versions by running `emsdk activate `. Activating a tool will set up `~/.emscripten` to -point to that particular tool. On Windows, you can pass the option `--global` to -the `activate` command to register the environment permanently to the system -registry for all users. +point to that particular tool. On Windows, you can pass the option `--permanent` to +the `activate` command to register the environment permanently for the current user. Use `--system` to do this for all users. ### How do I build multiple projects with different SDK versions in parallel? @@ -199,11 +198,11 @@ where you directly interact with the github repositories. This allows you to obtain new features and latest fixes immediately as they are pushed to the github repository, without having to wait for release to be tagged. You do not need a github account or a fork of Emscripten to do this. To switch to using the -latest upstream git development branch `master`, run the following: +latest upstream git development branch `main`, run the following: emsdk install git-1.9.4 # Install git. Skip if the system already has it. - emsdk install sdk-upstream-master-64bit # Clone+pull the latest emscripten-core/emscripten/master. - emsdk activate sdk-upstream-master-64bit # Set the master SDK as the currently active one. + emsdk install sdk-upstream-main-64bit # Clone+pull the latest emscripten-core/emscripten/main. + emsdk activate sdk-upstream-main-64bit # Set the main SDK as the currently active one. ### How do I use my own Emscripten github fork with the SDK? @@ -213,16 +212,16 @@ acquainted with working on multiple remotes in a git clone, these steps should be familiar to you. This is useful in the case when you want to make your own modifications to the Emscripten toolchain, but still keep using the SDK environment and tools. To set up your own fork as the currently active -Emscripten toolchain, first install the `sdk-master` SDK like shown in the +Emscripten toolchain, first install the `sdk-main` SDK like shown in the previous section, and then run the following commands in the emsdk directory: - cd emscripten/master + cd emscripten/main # Add a git remote link to your own repository. git remote add myremote https://github.com/mygituseraccount/emscripten.git # Obtain the changes in your link. git fetch myremote - # Switch the emscripten-master tool to use your fork. - git checkout -b mymaster --track myremote/master + # Switch the emscripten-main tool to use your fork. + git checkout -b mymain --track myremote/main In this way you can utilize the Emscripten SDK tools while using your own git fork. You can switch back and forth between remotes via the `git checkout` diff --git a/bazel/BUILD b/bazel/BUILD new file mode 100644 index 0000000000..9aa46d9379 --- /dev/null +++ b/bazel/BUILD @@ -0,0 +1,44 @@ +package(default_visibility = ['//visibility:public']) + +config_setting( + name = "linux", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + +config_setting( + name = "macos", + constraint_values = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], +) + +config_setting( + name = "windows", + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], +) + +alias( + name = "binaries", + actual = select({ + ":linux": "@emscripten_bin_linux//:all", + ":macos": "@emscripten_bin_mac//:all", + ":windows": "@emscripten_bin_win//:all", + }), +) + +alias( + name = "node_modules", + actual = select({ + ":linux": "@emscripten_npm_linux//:node_modules", + ":macos": "@emscripten_npm_mac//:node_modules", + ":windows": "@emscripten_npm_win//:node_modules", + }), +) + diff --git a/bazel/README.md b/bazel/README.md new file mode 100644 index 0000000000..dc849fd27a --- /dev/null +++ b/bazel/README.md @@ -0,0 +1,61 @@ +# Bazel Emscripten toolchain + +## Setup Instructions + +In `WORKSPACE` file, put: +``` +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "emsdk", + strip_prefix = "emsdk-c1589b55641787d55d53e883852035beea9aec3f/bazel", + url = "https://github.com/emscripten-core/emsdk/archive/c1589b55641787d55d53e883852035beea9aec3f.tar.gz", + sha256 = "7a58a9996b113d3e0675df30b5f17e28aa47de2e684a844f05394fe2f6f12e8e", +) + +load("@emsdk//:deps.bzl", emsdk_deps = "deps") +emsdk_deps() + +load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps") +emsdk_emscripten_deps() +``` + +## Building + +### Using --config=wasm + +Put the following lines into your `.bazelrc`: +``` +build:wasm --crosstool_top=//emscripten_toolchain:everything +build:wasm --cpu=wasm +build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain +``` + +Simply pass `--config=wasm` when building a normal `cc_binary`. The result of +this build will be a tar archive containing any files produced by emscripten. + +### Using wasm_cc_binary +First, write a new rule wrapping your `cc_binary`. + +``` +load("@rules_cc//cc:defs.bzl", "cc_binary") +load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") + +cc_binary( + name = "hello-world", + srcs = ["hello-world.cc"], +) + +wasm_cc_binary( + name = "hello-world-wasm", + cc_target = ":hello-world", +) +``` + +Now you can run `bazel build :hello-world-wasm`. The result of this build will +be the individual files produced by emscripten. Note that some of these files +may be empty. This is because bazel has no concept of optional outputs for +rules. + +`wasm_cc_binary` uses transition to use emscripten toolchain on `cc_target` +and all of its dependencies, and does not require amending `.bazelrc`. This +is the preferred way, since it also unpacks the resulting tarball. diff --git a/bazel/WORKSPACE b/bazel/WORKSPACE new file mode 100644 index 0000000000..22311ff4a0 --- /dev/null +++ b/bazel/WORKSPACE @@ -0,0 +1,7 @@ +workspace(name = "emsdk") + +load(":deps.bzl", "deps") +deps() + +load(":emscripten_deps.bzl", "emscripten_deps") +emscripten_deps() diff --git a/bazel/bazelrc b/bazel/bazelrc new file mode 100644 index 0000000000..85801e8381 --- /dev/null +++ b/bazel/bazelrc @@ -0,0 +1,5 @@ +build:wasm --crosstool_top=//emscripten_toolchain:everything + +build:wasm --cpu=wasm + +build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain diff --git a/bazel/deps.bzl b/bazel/deps.bzl new file mode 100644 index 0000000000..0b37e1fdb8 --- /dev/null +++ b/bazel/deps.bzl @@ -0,0 +1,11 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def deps(): + excludes = native.existing_rules().keys() + + if "build_bazel_rules_nodejs" not in excludes: + http_archive( + name = "build_bazel_rules_nodejs", + sha256 = "0f2de53628e848c1691e5729b515022f5a77369c76a09fbe55611e12731c90e3", + urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/2.0.1/rules_nodejs-2.0.1.tar.gz"], + ) diff --git a/bazel/emscripten_deps.bzl b/bazel/emscripten_deps.bzl new file mode 100644 index 0000000000..e6a30beea0 --- /dev/null +++ b/bazel/emscripten_deps.bzl @@ -0,0 +1,76 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@build_bazel_rules_nodejs//:index.bzl", "npm_install") +load(":revisions.bzl", "EMSCRIPTEN_TAGS") + +def _parse_version(v): + return [int(u) for u in v.split(".")] + +def emscripten_deps(emscripten_version = "latest"): + version = emscripten_version + + if version == "latest": + version = reversed(sorted(EMSCRIPTEN_TAGS.keys(), key=_parse_version))[0] + + if version not in EMSCRIPTEN_TAGS.keys(): + error_msg = "Emscripten version {} not found.".format(version) + error_msg += " Look at @emsdk//:revisions.bzl for the list " + error_msg += "of currently supported versions." + fail(error_msg) + + revision = EMSCRIPTEN_TAGS[version] + + emscripten_url = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/{}/{}/wasm-binaries.tbz2" + + # This could potentially backfire for projects with multiple emscripten + # dependencies that use different emscripten versions + excludes = native.existing_rules().keys() + if "emscripten_bin_linux" not in excludes: + http_archive( + name = "emscripten_bin_linux", + strip_prefix = "install", + url = emscripten_url.format("linux", revision.hash), + sha256 = revision.sha_linux, + build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + type = "tar.bz2", + ) + + if "emscripten_bin_mac" not in excludes: + http_archive( + name = "emscripten_bin_mac", + strip_prefix = "install", + url = emscripten_url.format("mac", revision.hash), + sha256 = revision.sha_mac, + build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + type = "tar.bz2", + ) + + if "emscripten_bin_win" not in excludes: + http_archive( + name = "emscripten_bin_win", + strip_prefix = "install", + url = emscripten_url.format("win", revision.hash), + sha256 = revision.sha_win, + build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + type = "tar.bz2", + ) + + if "emscripten_npm_linux" not in excludes: + npm_install( + name = "emscripten_npm_linux", + package_json = "@emscripten_bin_linux//:emscripten/package.json", + package_lock_json = "@emscripten_bin_linux//:emscripten/package-lock.json", + ) + + if "emscripten_npm_mac" not in excludes: + npm_install( + name = "emscripten_npm_mac", + package_json = "@emscripten_bin_mac//:emscripten/package.json", + package_lock_json = "@emscripten_bin_mac//:emscripten/package-lock.json", + ) + + if "emscripten_npm_win" not in excludes: + npm_install( + name = "emscripten_npm_win", + package_json = "@emscripten_bin_win//:emscripten/package.json", + package_lock_json = "@emscripten_bin_win//:emscripten/package-lock.json", + ) diff --git a/bazel/emscripten_toolchain/BUILD.bazel b/bazel/emscripten_toolchain/BUILD.bazel new file mode 100644 index 0000000000..488da5a027 --- /dev/null +++ b/bazel/emscripten_toolchain/BUILD.bazel @@ -0,0 +1,81 @@ +load(":crosstool.bzl", "emscripten_cc_toolchain_config_rule") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "common-script-includes", + srcs = [ + "emar.sh", + "emcc.sh", + "emscripten_config", + "env.sh", + "@nodejs//:node_files", + "@emsdk//:binaries", + "@emsdk//:node_modules", + ], +) + +filegroup( + name = "compile-emscripten", + srcs = [":common-script-includes"], +) + +filegroup( + name = "link-emscripten", + srcs = [ + "emcc_link.sh", + "link_wrapper.py", + ":common-script-includes", + "@emsdk//:binaries", + "@nodejs//:node_files", + ], +) + +filegroup( + name = "every-file", + srcs = [ + ":compile-emscripten", + ":link-emscripten", + "@emsdk//:binaries", + "@nodejs//:node_files", + ], +) + +filegroup(name = "empty") + +# dlmalloc.bc is implictly added by the emscripten toolchain +cc_library(name = "malloc") + +emscripten_cc_toolchain_config_rule( + name = "wasm", + cpu = "wasm", + em_config = "emscripten_config", + emscripten_binaries = "@emsdk//:binaries", +) + +cc_toolchain( + name = "cc-compiler-wasm", + all_files = ":every-file", + ar_files = ":common-script-includes", + as_files = ":empty", + compiler_files = ":compile-emscripten", + dwp_files = ":empty", + linker_files = ":link-emscripten", + objcopy_files = ":empty", + strip_files = ":empty", + toolchain_config = "wasm", + toolchain_identifier = "emscripten-wasm", +) + +cc_toolchain_suite( + name = "everything", + toolchains = { + "wasm": ":cc-compiler-wasm", + "wasm|emscripten": ":cc-compiler-wasm", + }, +) + +py_binary( + name = "wasm_binary", + srcs = ["wasm_binary.py"], +) diff --git a/bazel/emscripten_toolchain/crosstool.bzl b/bazel/emscripten_toolchain/crosstool.bzl new file mode 100644 index 0000000000..0672ef7597 --- /dev/null +++ b/bazel/emscripten_toolchain/crosstool.bzl @@ -0,0 +1,1110 @@ +"""This module encapsulates logic to create emscripten_cc_toolchain_config rule.""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "action_config", + "env_entry", + "env_set", + "feature", + "feature_set", + "flag_group", + "tool", + "tool_path", + "variable_with_value", + "with_feature_set", + _flag_set = "flag_set", +) +load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") + +def flag_set(flags = None, features = None, not_features = None, **kwargs): + """Extension to flag_set which allows for a "simple" form. + + The simple form allows specifying flags as a simple list instead of a flag_group + if enable_if or expand_if semantics are not required. + + Similarly, the simple form allows passing features/not_features if they are a simple + list of semantically "and" features. + (i.e. "asan" and "dbg", rather than "asan" or "dbg") + + Args: + flags: list, set of flags + features: list, set of features required to be enabled. + not_features: list, set of features required to not be enabled. + **kwargs: The rest of the args for flag_set. + + Returns: + flag_set + """ + if flags: + if kwargs.get("flag_groups"): + fail("Cannot set flags and flag_groups") + else: + kwargs["flag_groups"] = [flag_group(flags = flags)] + + if features or not_features: + if kwargs.get("with_features"): + fail("Cannot set features/not_feature and with_features") + kwargs["with_features"] = [with_feature_set( + features = features or [], + not_features = not_features or [], + )] + return _flag_set(**kwargs) + +CROSSTOOL_DEFAULT_WARNINGS = [ + "-Wall", +] + +def _impl(ctx): + target_cpu = ctx.attr.cpu + toolchain_identifier = "emscripten-" + target_cpu + target_system_name = target_cpu + "-unknown-emscripten" + + host_system_name = "i686-unknown-linux-gnu" + + target_libc = "musl/js" + + abi_version = "emscripten_syscalls" + + compiler = "emscripten" + abi_libc_version = "default" + + cc_target_os = "emscripten" + + emscripten_dir = ctx.attr.emscripten_binaries.label.workspace_root + + builtin_sysroot = emscripten_dir + "/emscripten/cache/sysroot" + + ################################################################ + # Tools + ################################################################ + clang_tool = tool(path = "emcc.sh") + clif_match_tool = tool(path = "dummy_clif_matcher") + link_tool = tool(path = "emcc_link.sh") + archive_tool = tool(path = "emar.sh") + strip_tool = tool(path = "NOT_USED_STRIP_TOOL") + + #### Legacy tool paths (much of this is redundant with action_configs, but + #### these are still used for some things) + tool_paths = [ + tool_path(name = "ar", path = "emar.sh"), + tool_path(name = "cpp", path = "/bin/false"), + tool_path(name = "gcc", path = "emcc.sh"), + tool_path(name = "gcov", path = "/bin/false"), + tool_path(name = "ld", path = "emcc_link.sh"), + tool_path(name = "nm", path = "NOT_USED"), + tool_path(name = "objdump", path = "/bin/false"), + tool_path(name = "strip", path = "NOT_USED"), + ] + + ################################################################ + # Action Configs + ################################################################ + + cpp_compile_action = action_config( + action_name = ACTION_NAMES.cpp_compile, + tools = [clang_tool], + ) + + cpp_module_compile_action = action_config( + action_name = ACTION_NAMES.cpp_module_compile, + tools = [clang_tool], + ) + + cpp_module_codegen_action = action_config( + action_name = ACTION_NAMES.cpp_module_codegen, + tools = [clang_tool], + ) + + clif_match_action = action_config( + action_name = ACTION_NAMES.clif_match, + tools = [clif_match_tool], + ) + + cpp_link_dynamic_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_dynamic_library, + tools = [link_tool], + ) + + strip_action = action_config( + action_name = ACTION_NAMES.strip, + tools = [strip_tool], + ) + + preprocess_assemble_action = action_config( + action_name = ACTION_NAMES.preprocess_assemble, + tools = [clang_tool], + ) + + cpp_header_parsing_action = action_config( + action_name = ACTION_NAMES.cpp_header_parsing, + tools = [clang_tool], + ) + + cpp_link_static_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_static_library, + enabled = True, + flag_sets = [ + flag_set( + flag_groups = [ + flag_group( + flags = ["rcsD", "%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + flag_set( + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + ], + ), + flag_set( + flag_groups = [ + flag_group( + flags = ["@%{linker_param_file}"], + expand_if_available = "linker_param_file", + ), + ], + ), + ], + tools = [archive_tool], + ) + + c_compile_action = action_config( + action_name = ACTION_NAMES.c_compile, + tools = [clang_tool], + ) + + linkstamp_compile_action = action_config( + action_name = ACTION_NAMES.linkstamp_compile, + tools = [clang_tool], + ) + + assemble_action = action_config( + action_name = ACTION_NAMES.assemble, + tools = [clang_tool], + ) + + cpp_link_executable_action = action_config( + action_name = ACTION_NAMES.cpp_link_executable, + tools = [link_tool], + ) + + cpp_link_nodeps_dynamic_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_nodeps_dynamic_library, + tools = [link_tool], + ) + + action_configs = [ + strip_action, + c_compile_action, + cpp_compile_action, + linkstamp_compile_action, + assemble_action, + preprocess_assemble_action, + cpp_header_parsing_action, + cpp_module_compile_action, + cpp_module_codegen_action, + cpp_link_executable_action, + cpp_link_dynamic_library_action, + cpp_link_nodeps_dynamic_library_action, + cpp_link_static_library_action, + clif_match_action, + ] + + all_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.lto_backend, + ] + + all_cpp_compile_actions = [ + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ] + + preprocessor_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ] + + all_link_actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ] + + ################################################################ + # Features + ################################################################ + + features = [ + # This set of magic "feature"s are important configuration information for blaze. + feature(name = "no_legacy_features", enabled = True), + feature( + name = "has_configured_linker_path", + enabled = True, + ), + + # Blaze requests this feature by default, but we don't care. + feature(name = "dependency_file"), + + # Blaze requests this feature by default, but we don't care. + feature(name = "random_seed"), + + # Formerly "needsPic" attribute + feature(name = "supports_pic", enabled = False), + + # Blaze requests this feature by default. + # Blaze also tests if this feature is supported, before setting the "pic" build-variable. + feature(name = "pic"), + + # Blaze requests this feature if fission is requested + # Blaze also tests if it's supported to see if we support fission. + feature(name = "per_object_debug_info"), + + # Blaze requests this feature by default. + # Blaze also tests if this feature is supported before setting preprocessor_defines + # (...but why?) + feature(name = "preprocessor_defines"), + + # Blaze requests this feature by default. + # Blaze also tests if this feature is supported before setting includes. (...but why?) + feature(name = "include_paths"), + + # Blaze tests if this feature is enabled in order to create implicit + # "nodeps" .so outputs from cc_library rules. + feature(name = "supports_dynamic_linker", enabled = False), + + # Blaze requests this feature when linking a cc_binary which is + # "dynamic" aka linked against nodeps-dynamic-library cc_library + # outputs. + feature(name = "dynamic_linking_mode"), + + #### Configuration features + feature( + name = "crosstool_cpu", + enabled = True, + implies = ["crosstool_cpu_" + target_cpu], + ), + feature( + name = "crosstool_cpu_asmjs", + provides = ["variant:crosstool_cpu"], + ), + feature( + name = "crosstool_cpu_wasm", + provides = ["variant:crosstool_cpu"], + ), + + # These 3 features will be automatically enabled by blaze in the + # corresponding build mode. + feature( + name = "opt", + provides = ["variant:crosstool_build_mode"], + ), + feature( + name = "dbg", + provides = ["variant:crosstool_build_mode"], + ), + feature( + name = "fastbuild", + provides = ["variant:crosstool_build_mode"], + ), + + #### User-settable features + + # Set if enabling exceptions. + feature(name = "exceptions"), + + # This feature overrides the default optimization to prefer execution speed + # over binary size (like clang -O3). + feature( + name = "optimized_for_speed", + provides = ["variant:crosstool_optimization_mode"], + ), + + # This feature overrides the default optimization to prefer binary size over + # execution speed (like clang -Oz). + feature( + name = "optimized_for_size", + provides = ["variant:crosstool_optimization_mode"], + ), + + # Convenience aliases / alt-spellings. + feature( + name = "optimize_for_speed", + implies = ["optimized_for_speed"], + ), + feature( + name = "optimize_for_size", + implies = ["optimized_for_size"], + ), + + # This feature allows easier use of profiling tools by preserving mangled + # C++ names. This does everything profiling_funcs does and more. + feature(name = "profiling"), + + # This feature emits only enough debug info for function names to appear + # in profiles. + feature(name = "profiling_funcs"), + + # This feature allows source maps to be generated. + feature( + name = "source_maps", + implies = ["full_debug_info"], + ), + feature( + name = "dwarf_debug_info", + implies = ["profiling"], + ), + + # Turns on full debug info (-g4). + feature(name = "full_debug_info"), + + # Enables the use of "Emscripten" Pthread implementation. + # https://kripken.github.io/emscripten-site/docs/porting/pthreads.html + # https://github.com/kripken/emscripten/wiki/Pthreads-with-WebAssembly + feature(name = "use_pthreads"), + + # If enabled, the runtime will exit when main() completes. + feature(name = "exit_runtime"), + + # Primarily for toolchain maintainers: + feature(name = "emcc_debug"), + feature(name = "emcc_debug_link"), + feature( + name = "llvm_backend", + requires = [feature_set(features = ["crosstool_cpu_wasm"])], + enabled = True, + ), + + # Remove once flag is flipped. + # See https://github.com/bazelbuild/bazel/issues/7687 + feature( + name = "do_not_split_linking_cmdline", + ), + + # Adds simd support, only available with the llvm backend. + feature( + name = "wasm_simd", + requires = [feature_set(features = ["llvm_backend"])], + ), + feature( + name = "precise_long_double_printf", + enabled = True, + ), + feature( + name = "wasm_warnings_as_errors", + enabled = True, + ), + + # ASan and UBSan. See also: + # https://emscripten.org/docs/debugging/Sanitizers.html + feature(name = "wasm_asan"), + feature(name = "wasm_ubsan"), + + feature( + name = "output_format_js", + enabled = True, + ), + ] + + crosstool_default_flag_sets = [ + # Compile, Link, and CC_FLAGS make variable + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = ["--sysroot=%{sysroot}"], + expand_if_available = "sysroot", + ), + ], + ), + # Compile + Link + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + # This forces color diagnostics even on Forge (where we don't have an + # attached terminal). + flags = [ + "-fdiagnostics-color", + ], + ), + # C++ compiles (and implicitly link) + flag_set( + actions = all_cpp_compile_actions, + flags = [ + "-fno-exceptions", + ], + not_features = ["exceptions"], + ), + flag_set( + actions = all_cpp_compile_actions, + flags = [ + "-fexceptions", + ], + features = ["exceptions"], + ), + # All compiles (and implicitly link) + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = [ + "-fno-strict-aliasing", + "-funsigned-char", + "-no-canonical-prefixes", + ], + ), + # Language Features + flag_set( + actions = all_cpp_compile_actions, + flags = ["-std=gnu++17", "-nostdinc", "-nostdinc++",], + ), + + # Emscripten-specific settings: + flag_set( + actions = all_compile_actions + all_link_actions, + flags = ["-s", "WASM=0"], + features = ["crosstool_cpu_asmjs"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-s", "USE_PTHREADS=1"], + features = ["use_pthreads"], + ), + flag_set( + actions = all_link_actions, + flags = ["-s", "EXIT_RUNTIME=1"], + features = ["exit_runtime"], + ), + flag_set( + actions = all_compile_actions + all_link_actions, + flags = ["-pthread"], + features = ["llvm_backend", "use_pthreads"], + ), + flag_set( + actions = all_compile_actions + all_link_actions, + flags = ["-msimd128"], + features = ["wasm_simd"], + ), + flag_set( + actions = all_link_actions, + flags = ["-s", "PRINTF_LONG_DOUBLE=1"], + features = ["precise_long_double_printf"], + ), + flag_set( + actions = all_link_actions, + flags = ["--oformat=js"], + features = ["output_format_js"], + ), + + # Opt + flag_set( + actions = preprocessor_compile_actions, + flags = ["-DNDEBUG"], + features = ["opt"], + ), + flag_set( + actions = all_compile_actions, + flags = ["-fomit-frame-pointer"], + features = ["opt"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-O3"], + features = ["opt"], + ), + # Users can override opt-level with semantic names... + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-Oz"], + features = ["optimized_for_size", "opt"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-O3"], + features = ["optimized_for_speed", "opt"], + ), + + # Fastbuild + flag_set( + actions = all_compile_actions, + flags = ["-fomit-frame-pointer"], + features = ["fastbuild"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-O2"], + features = ["fastbuild"], + ), + + # Dbg + flag_set( + actions = all_compile_actions, + flags = ["-fno-omit-frame-pointer"], + features = ["dbg"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-g", "-O0"], + features = ["dbg"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = [ + "-g4", + "-fsanitize=address", + "-O1", + "-DADDRESS_SANITIZER=1", + "-fno-omit-frame-pointer", + ], + features = ["wasm_asan"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = [ + "-g4", + "-fsanitize=undefined", + "-O1", + "-DUNDEFINED_BEHAVIOR_SANITIZER=1", + "-fno-omit-frame-pointer", + "-fno-sanitize=vptr", + ], + features = ["wasm_ubsan"], + ), + + # Profiling provides full debug info and a special --profiling flag + # to control name mangling + flag_set( + actions = all_link_actions, + flags = ["--profiling"], + features = ["profiling"], + ), + flag_set( + actions = all_link_actions, + flags = ["--profiling_funcs"], + features = ["profiling_funcs"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-g4"], + features = ["full_debug_info"], + ), + flag_set( + actions = all_link_actions, + flags = ["-gseparate-dwarf"], + features = ["dwarf_debug_info"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-fdebug-compilation-dir=."], + features = ["dwarf_debug_info"], + ), + # Generic warning flag list + flag_set( + actions = all_compile_actions, + flags = CROSSTOOL_DEFAULT_WARNINGS, + ), + + # Defines and Includes and Paths and such + flag_set( + actions = all_compile_actions, + flag_groups = [ + flag_group(flags = ["-fPIC"], expand_if_available = "pic"), + ], + ), + flag_set( + actions = preprocessor_compile_actions, + flag_groups = [ + flag_group( + flags = ["-D%{preprocessor_defines}"], + iterate_over = "preprocessor_defines", + ), + ], + ), + flag_set( + actions = preprocessor_compile_actions, + flag_groups = [ + flag_group( + flags = ["-include", "%{includes}"], + iterate_over = "includes", + expand_if_available = "includes", + ), + ], + ), + flag_set( + actions = preprocessor_compile_actions, + flag_groups = [ + flag_group( + flags = ["-iquote", "%{quote_include_paths}"], + iterate_over = "quote_include_paths", + ), + flag_group( + flags = ["-I%{include_paths}"], + iterate_over = "include_paths", + ), + flag_group( + flags = ["-isystem", "%{system_include_paths}"], + iterate_over = "system_include_paths", + ), + ], + ), + + ## Linking options (not libs -- those go last) + + # Generic link options + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flags = ["-shared"], + ), + + # Linker search paths and objects: + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "runtime_library_search_directories", + flag_groups = [ + flag_group( + flags = [ + "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_true = "is_cc_test", + ), + flag_group( + flags = [ + "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_false = "is_cc_test", + ), + ], + expand_if_available = "runtime_library_search_directories", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-L%{library_search_directories}"], + iterate_over = "library_search_directories", + expand_if_available = "library_search_directories", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + # This is actually a list of object files from the linkstamp steps + flags = ["%{linkstamp_paths}"], + iterate_over = "linkstamp_paths", + expand_if_available = "linkstamp_paths", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["@%{thinlto_param_file}"], + expand_if_available = "libraries_to_link", + expand_if_true = "thinlto_param_file", + ), + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["-Wl,--start-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["-Wl,-whole-archive"], + expand_if_true = "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "interface_library", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "static_library", + ), + ), + flag_group( + flags = ["-l%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "dynamic_library", + ), + ), + flag_group( + flags = ["-l:%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "versioned_dynamic_library", + ), + ), + flag_group( + flags = ["-Wl,-no-whole-archive"], + expand_if_true = "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["-Wl,--end-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + ], + ), + + # Configure the header parsing and preprocessing. + flag_set( + actions = [ACTION_NAMES.cpp_header_parsing], + flags = ["-xc++-header", "-fsyntax-only"], + features = ["parse_headers"], + ), + + # Note: user compile flags should be nearly last -- you probably + # don't want to put any more features after this! + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["%{user_compile_flags}"], + iterate_over = "user_compile_flags", + expand_if_available = "user_compile_flags", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{user_link_flags}"], + iterate_over = "user_link_flags", + expand_if_available = "user_link_flags", + ), + ], + ), + ## Options which need to go late -- after all the user options -- go here. + flag_set( + # One might hope that these options would only be needed for C++ + # compiles. But, sadly, users compile ".c" files with custom + # copts=["-x", "c++"], and expect that to be able to find C++ stdlib + # headers. It might be worth pondering how blaze could support this sort + # of use-case better. + actions = preprocessor_compile_actions + + [ACTION_NAMES.cc_flags_make_variable], + flags = [ + "-iwithsysroot" + "/include/c++/v1", + "-iwithsysroot" + "/include/compat", + "-iwithsysroot" + "/include", + "-isystem", emscripten_dir + "/lib/clang/13.0.0/include", + ], + ), + # Inputs and outputs + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-MD", "-MF", "%{dependency_file}"], + expand_if_available = "dependency_file", + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-c", "%{source_file}"], + expand_if_available = "source_file", + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-S"], + expand_if_available = "output_assembly_file", + ), + flag_group( + flags = ["-E"], + expand_if_available = "output_preprocess_file", + ), + flag_group( + flags = ["-o", "%{output_file}"], + expand_if_available = "output_file", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-o", "%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + # And finally, the params file! + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["@%{linker_param_file}"], + expand_if_available = "linker_param_file", + ), + ], + ), + flag_set( + actions = all_compile_actions, + flags = [ + "-Wno-builtin-macro-redefined", + # Genrules may not escape quotes enough for these, so + # don't put them into $(CC_FLAGS): + '-D__DATE__="redacted"', + '-D__TIMESTAMP__="redacted"', + '-D__TIME__="redacted"', + ], + ), + flag_set( + actions = all_compile_actions, + flags = ["-Werror"], + features = ["wasm_warnings_as_errors"], + ), + ] + + crosstool_default_env_sets = [ + # Globals + env_set( + actions = all_compile_actions + + all_link_actions + + [ACTION_NAMES.cpp_link_static_library], + env_entries = [ + env_entry( + key = "EM_BIN_PATH", + value = emscripten_dir, + ), + env_entry( + key = "EM_CONFIG_PATH", + value = ctx.file.em_config.path, + ), + ], + ), + # Use llvm backend. Off by default, enabled via --features=llvm_backend + env_set( + actions = all_compile_actions + + all_link_actions + + [ACTION_NAMES.cpp_link_static_library], + env_entries = [env_entry(key = "EMCC_WASM_BACKEND", value = "1")], + with_features = [with_feature_set(features = ["llvm_backend"])], + ), + # Debug compile and link. Off by default, enabled via --features=emcc_debug + env_set( + actions = all_compile_actions, + env_entries = [env_entry(key = "EMCC_DEBUG", value = "1")], + with_features = [with_feature_set(features = ["emcc_debug"])], + ), + + # Debug only link step. Off by default, enabled via --features=emcc_debug_link + env_set( + actions = all_link_actions, + env_entries = [env_entry(key = "EMCC_DEBUG", value = "1")], + with_features = [ + with_feature_set(features = ["emcc_debug"]), + with_feature_set(features = ["emcc_debug_link"]), + ], + ), + ] + + crosstool_default_flags_feature = feature( + name = "crosstool_default_flags", + enabled = True, + flag_sets = crosstool_default_flag_sets, + env_sets = crosstool_default_env_sets, + ) + + features.append(crosstool_default_flags_feature) + + cxx_builtin_include_directories = [ + emscripten_dir + "/emscripten/cache/sysroot/include/c++/v1", + emscripten_dir + "/emscripten/cache/sysroot/include/compat", + emscripten_dir + "/emscripten/cache/sysroot/include", + emscripten_dir + "/lib/clang/13.0.0/include", + ] + + artifact_name_patterns = [] + + make_variables = [] + + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + artifact_name_patterns = artifact_name_patterns, + cxx_builtin_include_directories = cxx_builtin_include_directories, + toolchain_identifier = toolchain_identifier, + host_system_name = host_system_name, + target_system_name = target_system_name, + target_cpu = target_cpu, + target_libc = target_libc, + compiler = compiler, + abi_version = abi_version, + abi_libc_version = abi_libc_version, + tool_paths = tool_paths, + make_variables = make_variables, + builtin_sysroot = builtin_sysroot, + cc_target_os = cc_target_os, + ) + +emscripten_cc_toolchain_config_rule = rule( + implementation = _impl, + attrs = { + "cpu": attr.string(mandatory = True, values = ["asmjs", "wasm"]), + "em_config": attr.label(mandatory = True, allow_single_file=True), + "emscripten_binaries": attr.label(mandatory = True), + }, + provides = [CcToolchainConfigInfo], +) diff --git a/bazel/emscripten_toolchain/emar.sh b/bazel/emscripten_toolchain/emar.sh new file mode 100755 index 0000000000..e4279f16a1 --- /dev/null +++ b/bazel/emscripten_toolchain/emar.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source external/emsdk/emscripten_toolchain/env.sh + +exec python3 $EMSCRIPTEN/emar.py "$@" diff --git a/bazel/emscripten_toolchain/emcc.sh b/bazel/emscripten_toolchain/emcc.sh new file mode 100755 index 0000000000..7f3699be1d --- /dev/null +++ b/bazel/emscripten_toolchain/emcc.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source external/emsdk/emscripten_toolchain/env.sh + +exec python3 $EMSCRIPTEN/emcc.py "$@" diff --git a/bazel/emscripten_toolchain/emcc_link.sh b/bazel/emscripten_toolchain/emcc_link.sh new file mode 100755 index 0000000000..24d806d029 --- /dev/null +++ b/bazel/emscripten_toolchain/emcc_link.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source external/emsdk/emscripten_toolchain/env.sh + +exec python3 external/emsdk/emscripten_toolchain/link_wrapper.py "$@" diff --git a/bazel/emscripten_toolchain/emscripten.BUILD b/bazel/emscripten_toolchain/emscripten.BUILD new file mode 100644 index 0000000000..6f11852874 --- /dev/null +++ b/bazel/emscripten_toolchain/emscripten.BUILD @@ -0,0 +1,6 @@ +package(default_visibility = ['//visibility:public']) + +filegroup( + name = "all", + srcs = glob(["**"]), +) diff --git a/bazel/emscripten_toolchain/emscripten_config b/bazel/emscripten_toolchain/emscripten_config new file mode 100644 index 0000000000..d1275ff978 --- /dev/null +++ b/bazel/emscripten_toolchain/emscripten_config @@ -0,0 +1,12 @@ +import os +import platform + +ROOT_DIR = os.environ["ROOT_DIR"] +EMSCRIPTEN_ROOT = os.environ["EMSCRIPTEN"] +BINARYEN_ROOT = ROOT_DIR + "/" + os.environ["EM_BIN_PATH"] +LLVM_ROOT = BINARYEN_ROOT + "/bin" +FROZEN_CACHE = True + +system = platform.system() +nodejs_binary = "node.exe" if(system =="Windows") else "bin/node" +NODE_JS = ROOT_DIR + "/external/nodejs_{}_amd64/{}".format(system.lower(), nodejs_binary) diff --git a/bazel/emscripten_toolchain/env.sh b/bazel/emscripten_toolchain/env.sh new file mode 100755 index 0000000000..d66a3a1fde --- /dev/null +++ b/bazel/emscripten_toolchain/env.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +export ROOT_DIR=`(pwd -P)` +export EMSCRIPTEN=$ROOT_DIR/$EM_BIN_PATH/emscripten +export EM_CONFIG=$ROOT_DIR/$EM_CONFIG_PATH diff --git a/bazel/emscripten_toolchain/link_wrapper.py b/bazel/emscripten_toolchain/link_wrapper.py new file mode 100644 index 0000000000..1e26bde7f5 --- /dev/null +++ b/bazel/emscripten_toolchain/link_wrapper.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +"""wrapper around emcc link step. + +This wrapper currently serves the following purposes. + +1. When building with --config=wasm the final output is multiple files, usually + at least one .js and one .wasm file. Since the cc_binary link step only + allows a single output, we must tar up the outputs into a single file. + +2. Add quotes around arguments that need them in the response file to work + around a bazel quirk. + +3. Ensure the external_debug_info section of the wasm points at the correct + bazel path. +""" + +from __future__ import print_function + +import argparse +import os +import subprocess +import sys + +# Only argument should be @path/to/parameter/file +assert sys.argv[1][0] == '@' +param_filename = sys.argv[1][1:] +param_file_args = [l.strip() for l in open(param_filename, 'r').readlines()] + +# Re-write response file if needed. +if any(' ' in a for a in param_file_args): + new_param_filename = param_filename + '.modified' + with open(new_param_filename, 'w') as f: + for param in param_file_args: + if ' ' in param: + f.write('"%s"' % param) + else: + f.write(param) + f.write('\n') + sys.argv[1] = '@' + new_param_filename + +emcc_py = os.path.join(os.environ['EMSCRIPTEN'], 'emcc.py') +rtn = subprocess.call(['python3', emcc_py] + sys.argv[1:]) +if rtn != 0: + sys.exit(1) + +# Parse the arguments that we gave to the linker to determine what the output +# file is named and what the output format is. +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument('-o') +parser.add_argument('--oformat') +options = parser.parse_known_args(param_file_args)[0] +output_file = options.o +oformat = options.oformat +outdir = os.path.dirname(output_file) +base_name = os.path.basename(output_file) + +# The output file name is the name of the build rule that was built. +# Add an appropriate file extension based on --oformat. +if oformat is not None: + base_name_split = os.path.splitext(base_name) + + # If the output name has no extension, give it the appropriate extension. + if not base_name_split[1]: + os.rename(output_file, output_file + '.' + oformat) + + # If the output name does have an extension and it matches the output format, + # change the base_name so it doesn't have an extension. + elif base_name_split[1] == '.' + oformat: + base_name = base_name_split[0] + + # If the output name does have an extension and it does not match the output + # format, change the base_name so it doesn't have an extension and rename + # the output_file so it has the proper extension. + # Note that if you do something like name your build rule "foo.js" and pass + # "--oformat=html", emscripten will write to the same file for both the js and + # html output, overwriting the js output entirely with the html. + # Please don't do that. + else: + base_name = base_name_split[0] + os.rename(output_file, os.path.join(outdir, base_name + '.' + oformat)) + +files = [] +extensions = [ + '.js', + '.wasm', + '.wasm.map', + '.js.mem', + '.fetch.js', + '.worker.js', + '.data', + '.js.symbols', + '.wasm.debug.wasm', + '.html' +] + +for ext in extensions: + filename = base_name + ext + if os.path.exists(os.path.join(outdir, filename)): + files.append(filename) + +wasm_base = os.path.join(outdir, base_name + '.wasm') +if os.path.exists(wasm_base + '.debug.wasm') and os.path.exists(wasm_base): + # If we have a .wasm.debug.wasm file and a .wasm file, we need to rewrite the + # section in the .wasm file that refers to it. The path that's in there + # is the blaze output path; we want it to be just the filename. + + llvm_objcopy = os.path.join( + os.environ['EMSCRIPTEN'], 'llvm-bin/llvm-objcopy') + # First, check to make sure the .wasm file has the header that needs to be + # rewritten. + rtn = subprocess.call([ + llvm_objcopy, + '--dump-section=external_debug_info=/dev/null', + wasm_base], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if rtn == 0: + # If llvm-objcopy did not return an error, the external_debug_info section + # must exist, so we're good to continue. + + # Next we need to convert length of the filename to LEB128. + # Start by converting the length of the filename to a bit string. + bit_string = '{0:b}'.format(len(base_name + '.wasm.debug.wasm')) + + # Pad the bit string with 0s so that its length is a multiple of 7. + while len(bit_string) % 7 != 0: + bit_string = '0' + bit_string + + # Break up our bit string into chunks of 7. + # We do this backwards because the final format is little-endian. + final_bytes = bytearray() + for i in reversed(range(0, len(bit_string), 7)): + binary_part = bit_string[i:i + 7] + if i != 0: + # Every chunk except the last one needs to be prepended with '1'. + # The length of each chunk is 7, so that one has an implicit '0'. + binary_part = '1' + binary_part + final_bytes.append(int(binary_part, 2)) + # Finally, add the actual filename. + final_bytes.extend((base_name + '.wasm.debug.wasm').encode()) + + # Write our length + filename bytes to a temp file. + with open('debugsection.tmp', 'wb+') as f: + f.write(final_bytes) + f.close() + + # First delete the old section. + subprocess.check_call([ + llvm_objcopy, + wasm_base, + '--remove-section=external_debug_info']) + # Rewrite section with the new size and filename from the temp file. + subprocess.check_call([ + llvm_objcopy, + wasm_base, + '--add-section=external_debug_info=debugsection.tmp']) + +# If we have more than one output file then create tarball +if len(files) > 1: + cmd = ['tar', 'cf', 'tmp.tar'] + files + subprocess.check_call(cmd, cwd=outdir) + os.rename(os.path.join(outdir, 'tmp.tar'), output_file) +elif len(files) == 1: + # Otherwise, if only have a single output than move it to the expected name + if files[0] != os.path.basename(output_file): + os.rename(os.path.join(outdir, files[0]), output_file) +else: + print('emcc.py did not appear to output any known files!') + sys.exit(1) + +sys.exit(0) diff --git a/bazel/emscripten_toolchain/wasm_binary.py b/bazel/emscripten_toolchain/wasm_binary.py new file mode 100644 index 0000000000..641c0d6f28 --- /dev/null +++ b/bazel/emscripten_toolchain/wasm_binary.py @@ -0,0 +1,84 @@ +"""Unpackages a bazel emscripten archive for use in a bazel BUILD rule. + +This script will take a tar archive containing the output of the emscripten +toolchain. This file contains any output files produced by a wasm_cc_binary or a +cc_binary built with --config=wasm. The files are extracted into the given +output path. + +The name of archive is expected to be of the format `foo` or `foo.XXX` and +the contents are expected to be foo.js and foo.wasm. + +Several optional files may also be in the archive, including but not limited to +foo.js.mem, pthread-main.js, and foo.wasm.map. + +If the file is not a tar archive, the passed file will simply be copied to its +destination. + +This script and its accompanying Bazel rule should allow you to extract a +WebAssembly binary into a larger web application. +""" + +import argparse +import os +import subprocess +import sys + + +def ensure(f): + if not os.path.exists(f): + with open(f, 'w'): + pass + + +def check(f): + if not os.path.exists(f): + raise Exception('Expected file in archive: %s' % f) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--archive', help='The archive to extract from.') + parser.add_argument('--output_path', help='The path to extract into.') + args = parser.parse_args() + + basename = os.path.basename(args.archive) + stem = basename.split('.')[0] + + # Check the type of the input file + mimetype_bytes = subprocess.check_output(['file', '-Lb', '--mime-type', '--mime-encoding', args.archive]) + mimetype = mimetype_bytes.decode(sys.stdout.encoding) + + # If we have a tar, extract all files. If we have just a single file, copy it. + if 'tar' in mimetype: + subprocess.check_call( + ['tar', 'xf', args.archive, '-C', args.output_path]) + elif 'binary' in mimetype: + subprocess.check_call([ + 'cp', + args.archive, + os.path.join(args.output_path, stem + '.wasm')]) + elif 'text' in mimetype: + subprocess.check_call([ + 'cp', + args.archive, + os.path.join(args.output_path, stem + '.js')]) + else: + subprocess.check_call(['cp', args.archive, args.output_path]) + + # At least one of these two files should exist at this point. + ensure(os.path.join(args.output_path, stem + '.js')) + ensure(os.path.join(args.output_path, stem + '.wasm')) + + # And can optionally contain these extra files. + ensure(os.path.join(args.output_path, stem + '.wasm.map')) + ensure(os.path.join(args.output_path, stem + '.worker.js')) + ensure(os.path.join(args.output_path, stem + '.js.mem')) + ensure(os.path.join(args.output_path, stem + '.data')) + ensure(os.path.join(args.output_path, stem + '.fetch.js')) + ensure(os.path.join(args.output_path, stem + '.js.symbols')) + ensure(os.path.join(args.output_path, stem + '.wasm.debug.wasm')) + ensure(os.path.join(args.output_path, stem + '.html')) + + +if __name__ == '__main__': + main() diff --git a/bazel/emscripten_toolchain/wasm_cc_binary.bzl b/bazel/emscripten_toolchain/wasm_cc_binary.bzl new file mode 100644 index 0000000000..9d6804117b --- /dev/null +++ b/bazel/emscripten_toolchain/wasm_cc_binary.bzl @@ -0,0 +1,152 @@ +"""wasm_cc_binary rule for compiling C++ targets to WebAssembly. +""" + +def _wasm_transition_impl(settings, attr): + _ignore = (settings, attr) + + features = list(settings["//command_line_option:features"]) + linkopts = list(settings["//command_line_option:linkopt"]) + + if attr.threads == "emscripten": + # threads enabled + features.append("use_pthreads") + elif attr.threads == "off": + # threads disabled + features.append("-use_pthreads") + + if attr.exit_runtime == True: + features.append("exit_runtime") + + if attr.backend == "llvm": + features.append("llvm_backend") + elif attr.backend == "emscripten": + features.append("-llvm_backend") + + if attr.simd: + features.append("wasm_simd") + + return { + "//command_line_option:compiler": "emscripten", + "//command_line_option:crosstool_top": "@emsdk//emscripten_toolchain:everything", + "//command_line_option:cpu": "wasm", + "//command_line_option:features": features, + "//command_line_option:dynamic_mode": "off", + "//command_line_option:linkopt": linkopts, + "//command_line_option:platforms": [], + "//command_line_option:custom_malloc": "@emsdk//emscripten_toolchain:malloc", + } + +_wasm_transition = transition( + implementation = _wasm_transition_impl, + inputs = [ + "//command_line_option:features", + "//command_line_option:linkopt", + ], + outputs = [ + "//command_line_option:compiler", + "//command_line_option:cpu", + "//command_line_option:crosstool_top", + "//command_line_option:features", + "//command_line_option:dynamic_mode", + "//command_line_option:linkopt", + "//command_line_option:platforms", + "//command_line_option:custom_malloc", + ], +) + +def _wasm_binary_impl(ctx): + cc_target = ctx.attr.cc_target[0] + + args = [ + "--output_path={}".format(ctx.outputs.loader.dirname), + ] + [ + ctx.expand_location("--archive=$(location {})".format( + cc_target.label, + ), [cc_target]), + ] + outputs = [ + ctx.outputs.loader, + ctx.outputs.wasm, + ctx.outputs.map, + ctx.outputs.mem, + ctx.outputs.fetch, + ctx.outputs.worker, + ctx.outputs.data, + ctx.outputs.symbols, + ctx.outputs.dwarf, + ctx.outputs.html, + ] + + ctx.actions.run( + inputs = ctx.files.cc_target, + outputs = outputs, + arguments = args, + executable = ctx.executable._wasm_binary_extractor, + ) + + return DefaultInfo( + files = depset(outputs), + # This is needed since rules like web_test usually have a data + # dependency on this target. + data_runfiles = ctx.runfiles(transitive_files = depset(outputs)), + ) + +def _wasm_binary_outputs(name, cc_target): + basename = cc_target.name + basename = basename.split(".")[0] + outputs = { + "loader": "{}/{}.js".format(name, basename), + "wasm": "{}/{}.wasm".format(name, basename), + "map": "{}/{}.wasm.map".format(name, basename), + "mem": "{}/{}.js.mem".format(name, basename), + "fetch": "{}/{}.fetch.js".format(name, basename), + "worker": "{}/{}.worker.js".format(name, basename), + "data": "{}/{}.data".format(name, basename), + "symbols": "{}/{}.js.symbols".format(name, basename), + "dwarf": "{}/{}.wasm.debug.wasm".format(name, basename), + "html": "{}/{}.html".format(name, basename), + } + + return outputs + +# Wraps a C++ Blaze target, extracting the appropriate files. +# +# This rule will transition to the emscripten toolchain in order +# to build the the cc_target as a WebAssembly binary. +# +# Args: +# name: The name of the rule. +# cc_target: The cc_binary or cc_library to extract files from. +wasm_cc_binary = rule( + implementation = _wasm_binary_impl, + attrs = { + "backend": attr.string( + default = "_default", + values = ["_default", "emscripten", "llvm"], + ), + "cc_target": attr.label( + cfg = _wasm_transition, + mandatory = True, + ), + "exit_runtime": attr.bool( + default = False, + ), + "threads": attr.string( + default = "_default", + values = ["_default", "emscripten", "off"], + ), + "simd": attr.bool( + default = False, + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + "_wasm_binary_extractor": attr.label( + executable = True, + allow_files = True, + cfg = "exec", + default = Label("@emsdk//emscripten_toolchain:wasm_binary"), + ), + }, + outputs = _wasm_binary_outputs, +) diff --git a/bazel/emscripten_toolchain/wasm_rules.bzl b/bazel/emscripten_toolchain/wasm_rules.bzl new file mode 100644 index 0000000000..f8dce22eec --- /dev/null +++ b/bazel/emscripten_toolchain/wasm_rules.bzl @@ -0,0 +1,6 @@ +"""Rules related to C++ and WebAssembly. +""" + +load(":wasm_cc_binary.bzl", _wasm_cc_binary = "wasm_cc_binary") + +wasm_cc_binary = _wasm_cc_binary diff --git a/bazel/hello-world/BUILD b/bazel/hello-world/BUILD new file mode 100644 index 0000000000..07fbed5510 --- /dev/null +++ b/bazel/hello-world/BUILD @@ -0,0 +1,23 @@ +load("@rules_cc//cc:defs.bzl", "cc_binary") +load("//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") + +cc_binary( + name = "hello-world", + srcs = ["hello-world.cc"], +) + +cc_binary( + name = "hello-world-simd", + srcs = ["hello-world-simd.cc"], +) + +wasm_cc_binary( + name = "hello-world-wasm", + cc_target = ":hello-world", +) + +wasm_cc_binary( + name = "hello-world-wasm-simd", + cc_target = ":hello-world-simd", + simd = True, +) diff --git a/bazel/hello-world/hello-world-simd.cc b/bazel/hello-world/hello-world-simd.cc new file mode 100644 index 0000000000..649adab20a --- /dev/null +++ b/bazel/hello-world/hello-world-simd.cc @@ -0,0 +1,10 @@ +#include + +void multiply_arrays(int* out, int* in_a, int* in_b, int size) { + for (int i = 0; i < size; i += 4) { + v128_t a = wasm_v128_load(&in_a[i]); + v128_t b = wasm_v128_load(&in_b[i]); + v128_t prod = wasm_i32x4_mul(a, b); + wasm_v128_store(&out[i], prod); + } +} diff --git a/bazel/hello-world/hello-world.cc b/bazel/hello-world/hello-world.cc new file mode 100644 index 0000000000..ee72c53171 --- /dev/null +++ b/bazel/hello-world/hello-world.cc @@ -0,0 +1,6 @@ +#include + +int main(int argc, char** argv) { + std::cout << "hello world!" << std::endl; + return 0; +} diff --git a/bazel/revisions.bzl b/bazel/revisions.bzl new file mode 100644 index 0000000000..eae75d13d3 --- /dev/null +++ b/bazel/revisions.bzl @@ -0,0 +1,41 @@ +# This file is automatically updated by emsdk/scripts/update_bazel_workspace.sh +# DO NOT MODIFY + +EMSCRIPTEN_TAGS = { + "2.0.18": struct( + hash = "c2ac7520fad29a7937ed60ab6a95b08eb374c7ba", + sha_linux = "e9f777de592f606b10104b2efe5179a7a8f44e3a9dffa1e3aaf73e05eb8893d7", + sha_mac = "86b1dd62e424e3788bf132292a694a25ca9b0875d06f50d0f5d424593697452c", + sha_win = "49ce07bda6be070251db44a08fcc05cae21ffdbd7522423a0c79bde635e87e28", + ), + "2.0.17": struct( + hash = "f5c45e60392b82f603e3a8039c62db294fab02d2", + sha_linux = "b40a4874057e4cace600f8ee9787dcbe236e3dc5b2fff5c2ecb0e867e426f99c", + sha_mac = "081f61abf7d5ac0ec31aaffc5550013d4093ea4ea39520b7a32b7448d2a6ee70", + sha_win = "45d06e597e6a1185a76200bd0481495e7298800a4805045d9cdbcce6311c91b2", + ), + "2.0.16": struct( + hash = "80d9674f2fafa6b9346d735c42d5c52b8cc8aa8e", + sha_linux = "e527638b224d9a30dc7e5fa4b9bd2eb2ab76ad306739ba8cacf5a5e333933a2a", + sha_mac = "061020eb0e3ee0611dc5a0008ccc7778168a4f838d49ca41c0aad8c52c1a01c9", + sha_win = "99364ed0388f928e0594f790662bf3a30c2894b0eff81797e1b64f62128561cb", + ), + "2.0.15": struct( + hash = "89202930a98fe7f9ed59b574469a9471b0bda7dd", + sha_linux = "7ff49fc63adf29970f6e7af1df445d7f554bdbbb2606db1cb5d3567ce69df1db", + sha_mac = "e35cced1514ad0da40584f8dd6f76aabf847ce0fa82c6dc8dd9442fb74ed6d0d", + sha_win = "31d5f8107c87833cea57edc57613bba4b36b16152772f744c5ad204594b4e666", + ), + "2.0.14": struct( + hash = "fc5562126762ab26c4757147a3b4c24e85a7289e", + sha_linux = "e466cd47ddd4bf0acd645412fdf08eda6d232484e48e5a2643e08062a7a4cf56", + sha_mac = "1c554c08459b7025638ca4eddba0d35babe8c26b202a70a74e9442d577896211", + sha_win = "428bc6094671937af96f26d803871fc5cd83d4d2b1c1df45fa6873a9bc5cac51", + ), + "2.0.13": struct( + hash = "ce0e4a4d1cab395ee5082a60ebb4f3891a94b256", + sha_linux = "8986ed886e111c661099c5147126b8a379a4040aab6a1f572fe01f0f9b99a343", + sha_mac = "88c91332c8c76fed14ebf0edc9a08f586012f54f04ad61e5b1b6d02bf96bdeab", + sha_win = "9fb3b945b7bd56e34d17ec04de4cce475f26c49d161aee9d9c0b8b1434591f88", + ), +} diff --git a/bazel/test_external/BUILD b/bazel/test_external/BUILD new file mode 100644 index 0000000000..73568cf388 --- /dev/null +++ b/bazel/test_external/BUILD @@ -0,0 +1,12 @@ +load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") + +cc_binary( + name = "hello-world", + srcs = ["hello-world.cc"], +) + +wasm_cc_binary( + name = "hello-world-wasm", + cc_target = ":hello-world", +) + diff --git a/bazel/test_external/WORKSPACE b/bazel/test_external/WORKSPACE new file mode 100644 index 0000000000..f0a446c24e --- /dev/null +++ b/bazel/test_external/WORKSPACE @@ -0,0 +1,10 @@ +local_repository( + name = "emsdk", + path = "..", +) + +load("@emsdk//:deps.bzl", "deps") +deps() + +load("@emsdk//:emscripten_deps.bzl", "emscripten_deps") +emscripten_deps() diff --git a/bazel/test_external/hello-world.cc b/bazel/test_external/hello-world.cc new file mode 100644 index 0000000000..ee72c53171 --- /dev/null +++ b/bazel/test_external/hello-world.cc @@ -0,0 +1,6 @@ +#include + +int main(int argc, char** argv) { + std::cout << "hello world!" << std::endl; + return 0; +} diff --git a/docker/Dockerfile b/docker/Dockerfile index a1a5363e5f..c80c23c3e3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,192 +1,107 @@ -FROM debian:buster AS stage_build +FROM ubuntu:focal AS stage_build -# ------------------------------------------------------------------------------ - -# Supports only 1.38.40+, accepts also '-upstream' variants -ARG EMSCRIPTEN_VERSION=1.39.11 -ARG EMSDK_CHANGESET=master - -# ------------------------------------------------------------------------------ - -# NOTE: Any change of following variables should be reflected in ./entrypoint file +ARG EMSCRIPTEN_VERSION=tot ENV EMSDK /emsdk -ENV EM_DATA ${EMSDK}/.data -ENV EM_CONFIG ${EMSDK}/.emscripten -ENV EM_CACHE ${EM_DATA}/cache -ENV EM_PORTS ${EM_DATA}/ports # ------------------------------------------------------------------------------ RUN echo "## Start building" \ - \ -&& echo "## Update and install packages" \ - && apt-get -qq -y update \ - && apt-get -qq install -y --no-install-recommends \ - libxml2 \ - wget \ - git-core \ - ca-certificates \ - build-essential \ - file \ - python python-pip \ - python3 python3-pip \ - \ -&& echo "## Done" - -RUN echo "## Get EMSDK" \ - && git clone https://github.com/emscripten-core/emsdk.git ${EMSDK} \ - && cd ${EMSDK} && git reset --hard ${EMSDK_CHANGESET} \ - \ - && ./emsdk.py update-tags \ -&& echo "## Done" - -RUN echo "## Install Emscripten" \ - && cd ${EMSDK} \ + && echo "## Update and install packages" \ + && apt-get -qq -y update \ + && apt-get -qq install -y --no-install-recommends \ + binutils \ + build-essential \ + ca-certificates \ + file \ + git \ + python3 \ + python3-pip \ + && echo "## Done" + +# Copy the contents of this repository to the container +COPY . ${EMSDK} + +RUN echo "## Install Emscripten" \ + && cd ${EMSDK} \ && ./emsdk install ${EMSCRIPTEN_VERSION} \ - \ -&& echo "## Done" + && echo "## Done" # This generates configuration that contains all valid paths according to installed SDK +# TODO(sbc): We should be able to use just emcc -v here but it doesn't +# currently create the sanity file. RUN cd ${EMSDK} \ - && echo "## Generate standard configuration" \ - \ - && ./emsdk activate ${EMSCRIPTEN_VERSION} --embedded \ - && ./emsdk construct_env > /dev/null \ - && cat ${EMSDK}/emsdk_set_env.sh \ - \ - # remove wrongly created entry with EM_CACHE, variable will be picked up from ENV - && sed -i -e "/EM_CACHE/d" ${EMSDK}/emsdk_set_env.sh \ - # add a link to tools like asm2wasm in a system path - # asm2wasm (and friends might be places either in ./upstream of ./fastcomp folder, hence detection is needed) - && printf "export PATH=$(dirname $(find . -name asm2wasm -exec readlink -f {} +)):\$PATH\n" >> ${EMSDK}/emsdk_set_env.sh \ - \ -&& echo "## Done" - -# Create a structure and make mutable folders accessible for r/w -RUN cd ${EMSDK} \ - && echo "## Create .data structure" \ - && for mutable_dir in ${EM_DATA} ${EM_PORTS} ${EM_CACHE} ${EMSDK}/zips ${EMSDK}/tmp; do \ - mkdir -p ${mutable_dir}; \ - chmod -R 777 ${mutable_dir}; \ - done \ - \ -&& echo "## Done" - -# Create an entrypoint that activates Emscripten SDK and helps running this image as non-root user -COPY entrypoint ${EMSDK}/ - - -# Create symbolic links for critical Emscripten Tools -# This is important for letting people using Emscripten in Dockerfiles without activation -# As each Emscripten release is placed to a different folder (i.e. /emsdk/emscripten/tag-1.38.31) -RUN echo "## Create symbolic links" \ - && . ${EMSDK}/emsdk_set_env.sh \ - \ - && mkdir -p ${EMSDK}/llvm ${EMSDK}/emscripten ${EMSDK}/binaryen \ - \ - && ln -s $(dirname $(which node))/.. ${EMSDK}/node/current \ - && ln -s $(dirname $(which clang))/.. ${EMSDK}/llvm/clang \ - && ln -s $(dirname $(which emcc)) ${EMSDK}/emscripten/sdk \ - \ - && ln -s $(dirname $(which asm2wasm)) ${EMSDK}/binaryen/bin \ - \ - && echo "## Done" - -# Clean up emscripten installation and strip some symbols -RUN echo "## Aggresive optimization: Remove debug symbols" \ -&& apt-get -qq -y update && apt-get -qq install -y --no-install-recommends \ - binutils \ - && . ${EMSDK}/emsdk_set_env.sh \ + && echo "## Generate standard configuration" \ + && ./emsdk activate ${EMSCRIPTEN_VERSION} \ + && chmod 777 ${EMSDK}/upstream/emscripten \ + && chmod -R 777 ${EMSDK}/upstream/emscripten/cache \ + && echo "int main() { return 0; }" > hello.c \ + && ${EMSDK}/upstream/emscripten/emcc -c hello.c \ + && cat ${EMSDK}/upstream/emscripten/cache/sanity.txt \ + && echo "## Done" + +# Cleanup Emscripten installation and strip some symbols +RUN echo "## Aggressive optimization: Remove debug symbols" \ + && cd ${EMSDK} && . ./emsdk_env.sh \ # Remove debugging symbols from embedded node (extra 7MB) && strip -s `which node` \ # Tests consume ~80MB disc space - && rm -fr ${EMSDK}/llvm/clang/emscripten/tests \ + && rm -fr ${EMSDK}/upstream/emscripten/tests \ + # Fastcomp is not supported + && rm -fr ${EMSDK}/upstream/fastcomp \ # strip out symbols from clang (~extra 50MB disc space) - && find ${EMSDK}/llvm/clang/bin -type f -exec strip -s {} + || true \ - && find ${EMSDK}/llvm/clang/fastcomp/bin -type f -exec strip -s {} + || true \ -&& echo "## Done" - -# Populate Emscripten SDK cache with libc++, to improve further compilation times. -RUN echo "## Pre-populate cache" \ - && . ${EMSDK}/emsdk_set_env.sh \ - \ - && embuilder.py build SYSTEM \ - \ - && mkdir -p /tmp/emscripten_test \ - && cd /tmp/emscripten_test \ - \ - && printf '#include \nint main(){std::cout << "HELLO FROM DOCKER C++"< test.cpp \ - && em++ --std=c++11 test.cpp -o test.js -s WASM=0 && node test.js \ - && em++ --std=c++11 -g3 test.cpp -o test.js -s WASM=0 && node test.js \ - && em++ --std=c++11 test.cpp -o test.js -s WASM=1 && node test.js \ - \ - && cd / \ - && rm -fr /tmp/emscripten_test \ - \ - # some files were created, and we need to make sure that those can be accessed by non-root people - && chmod -R 777 ${EM_DATA} \ - \ - # cleanup - && find ${EMSDK} -name "*.pyc" -exec rm {} \; \ - \ - && echo "## Done" + && find ${EMSDK}/upstream/bin -type f -exec strip -s {} + || true \ + && echo "## Done" # ------------------------------------------------------------------------------ # -------------------------------- STAGE DEPLOY -------------------------------- # ------------------------------------------------------------------------------ -FROM debian:buster-slim AS stage_deploy +FROM ubuntu:focal AS stage_deploy COPY --from=stage_build /emsdk /emsdk # Fallback in case Emscripten isn't activated. -# This will let use tools offered by this image inside other Docker images (sub-stages) or with custom / no entrypoint -ENV EMSDK /emsdk -ENV EMSCRIPTEN=${EMSDK}/emscripten/sdk - -ENV EM_DATA ${EMSDK}/.data -ENV EM_CONFIG ${EMSDK}/.emscripten -ENV EM_CACHE ${EM_DATA}/cache -ENV EM_PORTS ${EM_DATA}/ports - -# Fallback in case Emscripten isn't activated -# Expose Major tools to system PATH, so that emcc, node, asm2wasm etc can be used without activation -ENV PATH="${EMSDK}:${EMSDK}/emscripten/sdk:${EMSDK}/llvm/clang/bin:${EMSDK}/node/current/bin:${EMSDK}/binaryen/bin:${PATH}" - -# Use entrypoint that's coming from emscripten-slim image. It sets all required system paths and variables -ENTRYPOINT ["/emsdk/entrypoint"] - +# This will let use tools offered by this image inside other Docker images +# (sub-stages) or with custom / no entrypoint +ENV EMSDK=/emsdk \ + EM_CONFIG=/emsdk/.emscripten \ + EMSDK_NODE=/emsdk/node/14.15.5_64bit/bin/node \ + PATH="/emsdk:/emsdk/upstream/emscripten:/emsdk/upstream/bin:/emsdk/node/14.15.5_64bit/bin:${PATH}" # ------------------------------------------------------------------------------ # Create a 'standard` 1000:1000 user -# Thanks to that this image can be executed as non-root user and created files will not require root access level on host machine -# Please note that this solution even if widely spread (i.e. Node.js uses it) is far from perfect as user 1000:1000 might not exist on -# host machine, and in this case running any docker image will cause other random problems (mostly due `$HOME` pointing to `/`) -# This extra user works nicely with entrypoint provided in `/emsdk/entrypoint` as it detects case explained before. +# Thanks to that this image can be executed as non-root user and created files +# will not require root access level on host machine Please note that this +# solution even if widely spread (i.e. Node.js uses it) is far from perfect as +# user 1000:1000 might not exist on host machine, and in this case running any +# docker image will cause other random problems (mostly due `$HOME` pointing to +# `/`) RUN echo "## Create emscripten user (1000:1000)" \ - && groupadd --gid 1000 emscripten \ - && useradd --uid 1000 --gid emscripten --shell /bin/bash --create-home emscripten \ - \ -&& echo "## Done" + && groupadd --gid 1000 emscripten \ + && useradd --uid 1000 --gid emscripten --shell /bin/bash --create-home emscripten \ + && echo "umask 0000" >> /etc/bash.bashrc \ + && echo ". /emsdk/emsdk_env.sh" >> /etc/bash.bashrc \ + && echo "## Done" # ------------------------------------------------------------------------------ RUN echo "## Update and install packages" \ -# mitigate problem with create symlink to man for base debian image - && mkdir -p /usr/share/man/man1/ \ - \ - && apt-get -qq -y update && apt-get -qq install -y --no-install-recommends \ + && apt-get -qq -y update \ + # Somewhere in here apt sets up tzdata which asks for your time zone and blocks + # waiting for the answer which you can't give as docker build doesn't read from + # the terninal. The env vars set here avoid the interactive prompt and set the TZ. + && DEBIAN_FRONTEND="noninteractive" TZ="America/San_Francisco" apt-get -qq install -y --no-install-recommends \ + sudo \ libxml2 \ ca-certificates \ - python \ python3 \ - python-pip \ python3-pip \ wget \ curl \ zip \ unzip \ git \ + git-lfs \ ssh-client \ build-essential \ make \ @@ -194,36 +109,20 @@ RUN echo "## Update and install packages" \ libidn11 \ cmake \ openjdk-11-jre-headless \ - \ # Standard Cleanup on Debian images - && apt-get -y clean \ - && apt-get -y autoclean \ - && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf /var/cache/debconf/*-old \ - && rm -rf /usr/share/doc/* \ - && rm -rf /usr/share/man/?? \ - && rm -rf /usr/share/man/??_* \ -&& echo "## Done" - -# ------------------------------------------------------------------------------ -# Internal test suite of tools that this image provides -COPY test_dockerimage.sh ${EMSDK}/ - -RUN echo "## Internal Testing of image (activated)" \ - && . ${EMSDK}/emsdk_set_env.sh \ - && ${EMSDK}/test_dockerimage.sh \ - \ -&& echo "## Done" - -RUN echo "## Internal Testing of image (not-activated)" \ - && ${EMSDK}/test_dockerimage.sh \ - \ -&& echo "## Done" + && apt-get -y clean \ + && apt-get -y autoclean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/cache/debconf/*-old \ + && rm -rf /usr/share/doc/* \ + && rm -rf /usr/share/man/?? \ + && rm -rf /usr/share/man/??_* \ + && echo "## Done" # ------------------------------------------------------------------------------ -# Copy this Dockerimage into image, so that it will be possible to recreate it later -COPY Dockerfile /emsdk/dockerfiles/emscripten-core/emsdk/ +# Use commonly used /src as working directory +WORKDIR /src LABEL maintainer="kontakt@trzeci.eu" \ org.label-schema.name="emscripten" \ diff --git a/docker/Makefile b/docker/Makefile index 9527fe8f4c..0c961ca56b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,21 +1,28 @@ -#!/usr/bin/env make +# A Makefile to build, test, tag and publish the Emscripten SDK Docker container. # Emscripten version to build: Should match the version that has been already released. -# i.e.: 1.38.45, 1.38.45-upstream +# i.e.: 1.39.18 version = alias = +image_name ?= emscripten/emsdk + .TEST: ifndef version $(error argument 'version' is not set. Please call `make version=SOME_VERSION ...`) endif -build: .TEST - docker build --build-arg=EMSCRIPTEN_VERSION=${version} --tag emscripten/emsdk:${version} . +build: Dockerfile .TEST + cd .. && docker build --network host --build-arg=EMSCRIPTEN_VERSION=${version} -t ${image_name}:${version} -f docker/$< . + +test: test_dockerimage.sh .TEST + # test as non-root + docker run --rm -u `id -u`:`id -g` -w /emsdk/docker --net=host ${image_name}:${version} \ + bash $< push: .TEST - docker push emscripten/emsdk:${version} + docker push ${image_name}:${version} ifdef alias - docker tag emscripten/emsdk:${version} emscripten/emsdk:${alias} - docker push emscripten/emsdk:${alias} + docker tag ${image_name}:${version} ${image_name}:${alias} + docker push ${image_name}:${alias} endif diff --git a/docker/README.md b/docker/README.md index 5a72b8380b..7b9d301483 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,6 @@ # Dockerfile for EMSDK -This Dockerfile builds a self-contained version of emsdk that enables emscripten to be used without any +This Dockerfile builds a self-contained version of Emscripten SDK that enables Emscripten to be used without any other installation on the host system. It is published at https://hub.docker.com/u/emscripten/emsdk @@ -23,7 +23,7 @@ EOF # compile with docker image docker run \ --rm \ - -v $(pwd):$(pwd) \ + -v $(pwd):/src \ -u $(id -u):$(id -g) \ emscripten/emsdk \ emcc helloworld.cpp -o helloworld.js @@ -47,11 +47,11 @@ Teardown of compilation command: ### Building Dockerfile -This image requires to specify following build arguments: +This image has following optional arguments -| arg | description | -| --- | --- | -| `EMSCRIPTEN_VERSION` | One of released version of Emscripten. For example `1.38.45`
Can be used with `-upstream` variant like: `1.38.45-upstream`
Minimal supported version is **1.38.40**| +| arg | default value | description | +| --- | --- | --- | +| `EMSCRIPTEN_VERSION` | `tot`
(special case, tip-of-tree) | One of released version of Emscripten. For example `2.0.0`
Minimal supported version is **1.39.0** | **Building** @@ -59,13 +59,15 @@ This step will build Dockerfile as given tag on local machine ```bash # using docker docker build \ - --build-arg=EMSCRIPTEN_VERSION=1.38.43-upstream \ - --tag emscripten/emsdk:1.38.43-upstream \ + --network host \ + --build-arg=EMSCRIPTEN_VERSION=1.39.17 \ + -t emscripten/emsdk:1.39.17 \ + -f docker/Dockerfile \ . ``` ```bash # using predefined make target -make version=1.38.43-upstream build +make version=1.39.17 build test ``` **Tagging** @@ -78,35 +80,24 @@ This step will take local image and push to default docker registry. You need to ```bash # using docker -docker push emscripten/emsdk:1.38.43-upstream +docker push emscripten/emsdk:1.39.17 ``` ```bash # using predefined make target -make version=1.38.43-upstream push +make version=1.39.17 push ``` -In case of pushing the most recent version, this version should be also tagged as `latest` or `latest-upstream` and pushed. +In case of pushing the most recent version, this version should be also tagged as `latest` and pushed. ```bash # using docker cli - -# in case of fastcomp variant (default backend) -docker tag emscripten/emsdk:1.38.43 emscripten/emsdk:latest +docker tag emscripten/emsdk:1.39.17 emscripten/emsdk:latest docker push emscripten/emsdk:latest -# in case of upstream variant -docker tag emscripten/emsdk:1.38.43-upstream emscripten/emsdk:latest-upstream -docker push emscripten/emsdk:latest-upstream - -``` - ```bash -# using predefined make target - -make version=1.38.43-upstream alias=latest-upstream push - +# using make +make version=1.39.17 alias=latest push ``` - ### Extending If your project uses packages that this image doesn't provide you might want to: @@ -116,20 +107,19 @@ If your project uses packages that this image doesn't provide you might want to: 1. create own Dockerfile that holds: ```dockerfile # Point at any base image that you find suitable to extend. - FROM emscripten/emsdk:1.38.25 + FROM emscripten/emsdk:1.39.17 # Install required tools that are useful for your project i.e. ninja-build RUN apt update && apt install -y ninja-build - ``` + 2. build it - ```shell + ```bash docker build -t extended_emscripten . ``` 3. test - ```shell + ```bash docker run --rm extended_emscripten ninja --version - # Python 2.7.16 + # 1.10.0 ``` - diff --git a/docker/entrypoint b/docker/entrypoint deleted file mode 100755 index 8e7c702845..0000000000 --- a/docker/entrypoint +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -# In case when mapped user id by `docker run -u` is not created inside docker image -# The `$HOME` variable points to `/` - which prevents any tool to write to, as it requires root access -# In such case we set `$HOME` to `/tmp` as it should r/w for everyone - -if [ "$HOME" = "/" ] ; then - export HOME=/tmp -fi - -# In case of running as root, use `umask` to reduce problem of file permission on host -if [ "$(id -g)" = "0" ] && [ "$(id -u)" = "0" ] ; -then - umask 0000 -fi - -# Export this image specific Environment variables -# Those variables are important to use dedicated folder for all cache and predefined config file -export EM_CONFIG=/emsdk/.emscripten -export EM_CACHE=/emsdk/.data/cache -export EM_PORTS=/emsdk/.data/ports - -# Activate Emscripten SDK -. ${EMSDK}/emsdk_set_env.sh - -# Evaluate a command that's coming after `docker run` / `docker exec` -"$@" diff --git a/docker/test_dockerimage.sh b/docker/test_dockerimage.sh index 7f5cfbea75..4f44806c5f 100755 --- a/docker/test_dockerimage.sh +++ b/docker/test_dockerimage.sh @@ -1,19 +1,30 @@ -#!/bin/bash +#!/usr/bin/env bash set -ex -which asm2wasm +if [ $EUID -eq 0 ]; then + sudo -u nobody `which emcc` --version +fi + which llvm-ar which emsdk node --version npm --version python3 --version pip3 --version -python --version -pip --version em++ --version emcc --version java -version cmake --version -# cleanup after test -find ${EMSDK} -name "*.pyc" -exec rm {} \; +exit_code=0 + +# test emcc compilation +echo 'int main() { return 0; }' | emcc -o /tmp/main.js -xc - +node /tmp/main.js || exit_code=$? +if [ $exit_code -ne 0 ]; then + echo "Node exited with non-zero exit code: $exit_code" + exit $exit_code +fi + +# test embuilder +embuilder build zlib --force diff --git a/emscripten-releases-tags.txt b/emscripten-releases-tags.txt index 8206654625..7865a835b4 100644 --- a/emscripten-releases-tags.txt +++ b/emscripten-releases-tags.txt @@ -1,22 +1,48 @@ { - "latest": "1.39.15", + "latest": "2.0.18", "releases": { + "2.0.18": "c2ac7520fad29a7937ed60ab6a95b08eb374c7ba", + "2.0.17": "f5c45e60392b82f603e3a8039c62db294fab02d2", + "2.0.16": "80d9674f2fafa6b9346d735c42d5c52b8cc8aa8e", + "2.0.15": "89202930a98fe7f9ed59b574469a9471b0bda7dd", + "2.0.14": "fc5562126762ab26c4757147a3b4c24e85a7289e", + "2.0.13": "ce0e4a4d1cab395ee5082a60ebb4f3891a94b256", + "2.0.12": "dcf819a7821f8db0c8f15ac336fea8960ec204f5", + "2.0.11": "4764c5c323a474f7ba28ae991b0c9024fccca43c", + "2.0.10": "37fc7647c754ac9a28ad588c143b82286de0ef71", + "2.0.9": "d8e430f9a9b6e87502f826c39e7684852f59624f", + "2.0.8": "e4ed6c79f4db8b175d9bbe55869b697aba9bcf2a", + "2.0.7": "d7a29d82b320e471203b69d43aaf03b560eedc54", + "2.0.6": "4ba921c8c8fe2e8cae071ca9889d5c27f5debd87", + "2.0.5": "461f0f118d8d8e6cfd84e077f3eb010c17a39032", + "2.0.4": "eefeb3e623af023844ac477d70d1fd8a668f5110", + "2.0.3": "7a7f38ca19da152d4cd6da4776921a0f1e3f3e3f", + "2.0.2": "ede25d889a0abe63360d4c5d420087c8753b8bbe", + "2.0.1": "13e29bd55185e3c12802bc090b4507901856b2ba", + "2.0.0": "5974288502aab433d45f53511e961aaca4079d86", + "1.40.1": "536568644fd67d53778f6111fdd5f64ad3f4c539", + "1.40.0": "edf24e7233e0def312a08cc8dcec63a461155da1", + "1.39.20": "e7e39da9c81faecd9ecf44065cee864d76e4e34d", + "1.39.19": "665121d026cafc46c29b30d6d4c45ed73eedbb7e", + "1.39.18": "1914a1543f08cd8e41f44c2bb05f7a90d1920275", + "1.39.17": "9bc968c0e49b1c795ecddb87391b265911e2adcb", + "1.39.16": "ae5001fac3849895a873e422a2a80afc90f3b798", "1.39.15": "3880c744c068986d4ee781a61f7b2e820043e11f", "1.39.14": "574ad04affb82cc36a32dd89b2a87bea4fb30eba", "1.39.13": "7b3cd38017f7c582cfa3ac24a9f12aa6a8dca51f", "1.39.12": "e13b86d4dbd9a986525ef27d4ad8157949b9bc3a", "1.39.11": "6584e2d88570ee55914db92a3bad84f99e5bdd82", "1.39.10": "65d33d604d3fa0ebe03548378b898fc6608e9cb8", - "1.39.9": "122396dfad60e1b2a83ccefa74a1425a2e05b5cb", - "1.39.8": "9e60f34accb4627d7358223862a7e74291886ab6", - "1.39.7": "9a89fff28cc6f75e17976fce1904b280e4beb25d", - "1.39.6": "967836071d96d9b7894e492382f5fcb96423fc07", - "1.39.5": "b3ddcab6efd749d3ed937fb452ace4e39a825842", - "1.39.4": "8bb7b0bbbca74cc58741416cc955011f22ff5ccb", - "1.39.3": "b024b71038d1291ed7ec23ecd553bf2c0c8d6da6", - "1.39.2": "c630da9163a64e08de3dd948be0a0f7a175d285b", - "1.39.1": "40f3caabcef7b52bdde63d3883462414d7a25bec", - "1.39.0": "d57bfdd6d43181501bbd3fab502d57c9073ceb49", + "1.39.9": "122396dfad60e1b2a83ccefa74a1425a2e05b5cb", + "1.39.8": "9e60f34accb4627d7358223862a7e74291886ab6", + "1.39.7": "9a89fff28cc6f75e17976fce1904b280e4beb25d", + "1.39.6": "967836071d96d9b7894e492382f5fcb96423fc07", + "1.39.5": "b3ddcab6efd749d3ed937fb452ace4e39a825842", + "1.39.4": "8bb7b0bbbca74cc58741416cc955011f22ff5ccb", + "1.39.3": "b024b71038d1291ed7ec23ecd553bf2c0c8d6da6", + "1.39.2": "c630da9163a64e08de3dd948be0a0f7a175d285b", + "1.39.1": "40f3caabcef7b52bdde63d3883462414d7a25bec", + "1.39.0": "d57bfdd6d43181501bbd3fab502d57c9073ceb49", "1.38.48": "1290d9deb93d67c4649999a8f2c8d9167d38dc04", "1.38.47": "bc367c257409d676e71c5511383228b7aabf1689", "1.38.46": "c89919d252f7cea00d944bdf3bd630cd3c7e7388", @@ -35,4 +61,3 @@ "1.38.33": "3b8cff670e9233a6623563add831647e8689a86b" } } - diff --git a/emsdk b/emsdk index 6859b5458b..98b8d84ac2 100755 --- a/emsdk +++ b/emsdk @@ -6,13 +6,42 @@ # Wrapper script that runs emsdk.py -base_dir=$(dirname "$0") +# First look for python bundled in Emsdk +if [ -z "$EMSDK_PYTHON" ]; then + PYTHON3=$(dirname $0)/python/3.9.2-1_64bit/bin/python3 + PYTHON3_CERT_FILE=$(dirname $0)/python/3.9.2-1_64bit/lib/python3.9/site-packages/certifi/cacert.pem + if [ ! -f $PYTHON3 ]; then + PYTHON3=$(dirname $0)/python/3.7.4-2_64bit/bin/python3 + PYTHON3_CERT_FILE=$(dirname $0)/python/3.7.4-2_64bit/lib/python3.7/site-packages/certifi/cacert.pem + fi + if [ -f $PYTHON3 ]; then + EMSDK_PYTHON=$PYTHON3 -# Look for python3 first. This is especially important on macOS (See: + # When using our bundled python we never want the users + # PYTHONHOME or PYTHONPATH + # https://github.com/emscripten-core/emsdk/issues/598 + unset PYTHONHOME + unset PYTHONPATH + + # This is needed for MacOS. Without this, the urlopen + # code will try to use /usr/local/etc/openssl/cert.pem + # which may or may not exist on the system. + export SSL_CERT_FILE=$PYTHON3_CERT_FILE + fi +fi + +# If bundled python is not found, look for `python3` in PATH. This is especially important on macOS (See: # https://github.com/emscripten-core/emsdk/pull/273) -python=$(which python3 2> /dev/null) -if [ $? != 0 ]; then - python=python +if [ -z "$EMSDK_PYTHON" ]; then + PYTHON3=$(which python3 2> /dev/null) + if [ $? = 0 ]; then + EMSDK_PYTHON=$PYTHON3 + fi +fi + +# Finally fall back to just looking for `python` in PATH +if [ -z "$EMSDK_PYTHON" ]; then + EMSDK_PYTHON=python fi -exec "$python" "$0.py" "$@" +exec "$EMSDK_PYTHON" "$0.py" "$@" diff --git a/emsdk.bat b/emsdk.bat index 3a95825e52..70e6b709b5 100644 --- a/emsdk.bat +++ b/emsdk.bat @@ -1,51 +1,52 @@ -:: Find python from an explicit location relative to the Emscripten SDK. -@IF EXIST "%~dp0python\3.7.4_64bit\python.exe" ( - @SET EMSDK_PY="%~dp0python\3.7.4_64bit\python.exe" - @GOTO end -) +@echo off -@IF EXIST "%~dp0python\2.7.13.1_64bit\python-2.7.13.amd64\python.exe" ( - @SET EMSDK_PY="%~dp0python\2.7.13.1_64bit\python-2.7.13.amd64\python.exe" - @GOTO end -) +:: Find python from an explicit location relative to the Emscripten SDK. -@IF EXIST "%~dp0python\2.7.13.1_32bit\python-2.7.13\python.exe" ( - @SET EMSDK_PY="%~dp0python\2.7.13.1_32bit\python-2.7.13\python.exe" - @GOTO end -) +setlocal -@IF EXIST "%~dp0python\2.7.5.3_64bit\python.exe" ( - @SET EMSDK_PY="%~dp0python\2.7.5.3_64bit\python.exe" - @GOTO end +:: When using our bundled python we never want the users +:: PYTHONHOME or PYTHONPATH +:: https://github.com/emscripten-core/emsdk/issues/598 +if exist "%~dp0python\3.9.2-1_64bit\python.exe" ( + set EMSDK_PY="%~dp0python\3.9.2-1_64bit\python.exe" + set PYTHONHOME= + set PYTHONPATH= + goto end ) -@IF EXIST "%~dp0python\2.7.5.3_32bit\python.exe" ( - @SET EMSDK_PY="%~dp0python\2.7.5.3_32bit\python.exe" - @GOTO end +if exist "%~dp0python\3.7.4-pywin32_64bit\python.exe" ( + set EMSDK_PY="%~dp0python\3.7.4-pywin32_64bit\python.exe" + set PYTHONHOME= + set PYTHONPATH= + goto end ) -@IF EXIST "%~dp0python\2.7.5_64bit\python.exe" ( - @SET EMSDK_PY="%~dp0python\2.7.5_64bit\python.exe" - @GOTO end +if exist "%~dp0python\3.7.4_64bit\python.exe" ( + set EMSDK_PY="%~dp0python\3.7.4_64bit\python.exe" + set PYTHONHOME= + set PYTHONPATH= + goto end ) -@IF EXIST "%~dp0python\2.7.5.1_32bit\python.exe" ( - @SET EMSDK_PY="%~dp0python\2.7.5.1_32bit\python.exe" - @GOTO end +if exist "%~dp0python\2.7.13.1_64bit\python-2.7.13.amd64\python.exe" ( + set EMSDK_PY="%~dp0python\2.7.13.1_64bit\python-2.7.13.amd64\python.exe" + set PYTHONHOME= + set PYTHONPATH= + goto end ) :: As a last resort, access from PATH. -@SET EMSDK_PY=python +set EMSDK_PY=python :end -@call %EMSDK_PY% "%~dp0\emsdk.py" %* +call %EMSDK_PY% "%~dp0\emsdk.py" %* -@set EMSDK_PY= +endlocal -:: python is not able to set environment variables to the parent calling process, so -:: therefore have it craft a .bat file, which we invoke after finishing python execution, -:: to set up the environment variables -@IF EXIST emsdk_set_env.bat ( - @CALL emsdk_set_env.bat > NUL - @DEL /F /Q emsdk_set_env.bat +:: python is not able to set environment variables to the parent calling +:: process, so therefore have it craft a .bat file, which we invoke after +:: finishing python execution, to set up the environment variables +if exist "%~dp0\emsdk_set_env.bat" ( + call "%~dp0\emsdk_set_env.bat" > nul + del /F /Q "%~dp0\emsdk_set_env.bat" ) diff --git a/emsdk.ps1 b/emsdk.ps1 index ebd6f80247..f9ec03655a 100644 --- a/emsdk.ps1 +++ b/emsdk.ps1 @@ -1,6 +1,9 @@ $ScriptDirectory = Split-Path -parent $PSCommandPath $PythonLocations = $( + "python\3.9.2-1_64bit\python.exe", + "python\3.7.4-pywin32_64bit\python.exe", + "python\3.7.4_64bit\python.exe", "python\2.7.13.1_64bit\python-2.7.13.amd64\python.exe", "python\2.7.13.1_32bit\python-2.7.13\python.exe", "python\2.7.5.3_64bit\python.exe", @@ -31,9 +34,9 @@ $env:EMSDK_POWERSHELL = 1 # python is not able to set environment variables to the parent calling process, so # therefore have it craft a .ps1 file, which we invoke after finishing python execution, # to set up the environment variables -if (Test-Path ./emsdk_set_env.ps1) { - & ./emsdk_set_env.ps1 - Remove-Item ./emsdk_set_env.ps1 +if (Test-Path $ScriptDirectory/emsdk_set_env.ps1) { + & $ScriptDirectory/emsdk_set_env.ps1 + Remove-Item $ScriptDirectory/emsdk_set_env.ps1 } Remove-Item Env:\EMSDK_POWERSHELL diff --git a/emsdk.py b/emsdk.py old mode 100755 new mode 100644 index a6a287feda..657d1d57e5 --- a/emsdk.py +++ b/emsdk.py @@ -20,8 +20,14 @@ import subprocess import sys import sysconfig -import tempfile import zipfile +if os.name == 'nt': + try: + import winreg + except ImportError: + # old python 2 name + import _winreg as winreg + import ctypes.wintypes if sys.version_info >= (3,): from urllib.parse import urljoin @@ -31,18 +37,22 @@ from urlparse import urljoin from urllib2 import urlopen -emsdk_master_server = 'https://storage.googleapis.com/webassembly/emscripten-releases-builds/deps/' -emsdk_packages_url = emsdk_master_server +emsdk_packages_url = 'https://storage.googleapis.com/webassembly/emscripten-releases-builds/deps/' emscripten_releases_repo = 'https://chromium.googlesource.com/emscripten-releases' emscripten_releases_download_url_template = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/%s/%s/wasm-binaries.%s" -emsdk_zip_download_url = 'https://github.com/emscripten-core/emsdk/archive/master.zip' +# This was previously `master.zip` but we are transitioning to `main` and +# `HEAD.zip` works for both cases. In future we could switch this to +# `main.zip` perhaps. +emsdk_zip_download_url = 'https://github.com/emscripten-core/emsdk/archive/HEAD.zip' zips_subdir = 'zips/' +extra_release_tag = None + # Enable this to do very verbose printing about the different steps that are # being run. Useful for debugging. VERBOSE = int(os.getenv('EMSDK_VERBOSE', '0')) @@ -52,6 +62,11 @@ if os.name == 'nt' or (os.getenv('SYSTEMROOT') is not None and 'windows' in os.getenv('SYSTEMROOT').lower()) or (os.getenv('COMSPEC') is not None and 'windows' in os.getenv('COMSPEC').lower()): WINDOWS = True + +def errlog(msg): + print(msg, file=sys.stderr) + + MINGW = False MSYS = False if os.getenv('MSYSTEM'): @@ -63,17 +78,17 @@ MINGW = True if os.getenv('MSYSTEM') != 'MSYS' and os.getenv('MSYSTEM') != 'MINGW64': # https://stackoverflow.com/questions/37460073/msys-vs-mingw-internal-environment-variables - print('Warning: MSYSTEM environment variable is present, and is set to "' + os.getenv('MSYSTEM') + '". This shell has not been tested with emsdk and may not work.') + errlog('Warning: MSYSTEM environment variable is present, and is set to "' + os.getenv('MSYSTEM') + '". This shell has not been tested with emsdk and may not work.') -OSX = False +MACOS = False if platform.mac_ver()[0] != '': - OSX = True + MACOS = True LINUX = False -if not OSX and (platform.system() == 'Linux' or os.name == 'posix'): +if not MACOS and (platform.system() == 'Linux'): LINUX = True -UNIX = (OSX or LINUX) +UNIX = (MACOS or LINUX) # Pick which shell of 4 shells to use @@ -96,7 +111,6 @@ else: ENVPATH_SEPARATOR = ':' - ARCH = 'unknown' # platform.machine() may return AMD64 on windows, so standardize the case. machine = platform.machine().lower() @@ -109,8 +123,8 @@ elif platform.machine().startswith('arm'): ARCH = 'arm' else: - print("Warning: unknown machine architecture " + machine) - print() + errlog("Warning: unknown machine architecture " + machine) + errlog() # Don't saturate all cores to not steal the whole system, but be aggressive. CPU_CORES = int(os.environ.get('EMSDK_NUM_CORES', max(multiprocessing.cpu_count() - 1, 1))) @@ -135,8 +149,8 @@ def os_name(): return 'win' elif LINUX: return 'linux' - elif OSX: - return 'osx' + elif MACOS: + return 'macos' else: raise Exception('unknown OS') @@ -146,15 +160,15 @@ def os_name_for_emscripten_releases(): return 'win' elif LINUX: return 'linux' - elif OSX: + elif MACOS: return 'mac' else: raise Exception('unknown OS') -def debug_print(msg, **args): +def debug_print(msg): if VERBOSE: - print(msg, **args) + errlog(msg) def to_unix_path(p): @@ -165,15 +179,32 @@ def emsdk_path(): return to_unix_path(os.path.dirname(os.path.realpath(__file__))) -emscripten_config_directory = os.path.expanduser("~/") -# If .emscripten exists, we are configuring as embedded inside the emsdk directory. -if os.path.exists(os.path.join(emsdk_path(), '.emscripten')): - emscripten_config_directory = emsdk_path() +EMSDK_SET_ENV = "" +if POWERSHELL: + EMSDK_SET_ENV = os.path.join(emsdk_path(), 'emsdk_set_env.ps1') +else: + EMSDK_SET_ENV = os.path.join(emsdk_path(), 'emsdk_set_env.bat') + + +# Parses https://github.com/emscripten-core/emscripten/tree/d6aced8 to a pair (https://github.com/emscripten-core/emscripten, d6aced8) +def parse_github_url_and_refspec(url): + if not url: + return ('', '') + + if url.endswith(('/tree/', '/tree', '/commit/', '/commit')): + raise Exception('Malformed git URL and refspec ' + url + '!') + + if '/tree/' in url: + if url.endswith('/'): + raise Exception('Malformed git URL and refspec ' + url + '!') + return url.split('/tree/') + elif '/commit/' in url: + if url.endswith('/'): + raise Exception('Malformed git URL and refspec ' + url + '!') + return url.split('/commit/') + else: + return (url, 'main') # Assume the default branch is main in the absence of a refspec -EMSDK_SET_ENV = 'emsdk_set_env.ps1' if POWERSHELL \ - else 'emsdk_set_env.bat' if (WINDOWS and not MSYS) \ - else 'emsdk_set_env.csh' if CSH \ - else 'emsdk_set_env.sh' ARCHIVE_SUFFIXES = ('zip', '.tar', '.gz', '.xz', '.tbz2', '.bz2') @@ -210,10 +241,14 @@ def vswhere(version): program_files = os.environ['ProgramFiles(x86)'] if 'ProgramFiles(x86)' in os.environ else os.environ['ProgramFiles'] vswhere_path = os.path.join(program_files, 'Microsoft Visual Studio', 'Installer', 'vswhere.exe') output = json.loads(subprocess.check_output([vswhere_path, '-latest', '-version', '[%s.0,%s.0)' % (version, version + 1), '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', '-property', 'installationPath', '-format', 'json'])) - # Visual Studio 2017 Express is not included in the above search, and it does not have the VC.Tools.x86.x64 tool, so do a catch-all attempt as a fallback, to detect Express version. - if len(output) == 0: + # Visual Studio 2017 Express is not included in the above search, and it + # does not have the VC.Tools.x86.x64 tool, so do a catch-all attempt as a + # fallback, to detect Express version. + if not output: output = json.loads(subprocess.check_output([vswhere_path, '-latest', '-version', '[%s.0,%s.0)' % (version, version + 1), '-products', '*', '-property', 'installationPath', '-format', 'json'])) - return str(output[0]['installationPath']) if len(output) > 0 else '' + if not output: + return '' + return str(output[0]['installationPath']) except Exception: return '' @@ -270,7 +305,8 @@ def cmake_generator_prefix(): return '' -# Removes a directory tree even if it was readonly, and doesn't throw exception on failure. +# Removes a directory tree even if it was readonly, and doesn't throw exception +# on failure. def remove_tree(d): debug_print('remove_tree(' + str(d) + ')') if not os.path.exists(d): @@ -287,105 +323,73 @@ def remove_readonly_and_try_again(func, path, exc_info): debug_print('remove_tree threw an exception, ignoring: ' + str(e)) -def import_pywin32(): - if WINDOWS: - try: - import win32api - import win32con - return win32api, win32con - except Exception: - exit_with_error('Failed to import Python Windows extensions win32api and win32con. Make sure you are using the version of python available in emsdk, or install PyWin extensions to the distribution of Python you are attempting to use. (This script was launched in python instance from "' + sys.executable + '")') - - def win_set_environment_variable_direct(key, value, system=True): - prev_path = os.environ['PATH'] + folder = None try: - py = find_used_python() - if py: - py_path = to_native_path(py.expand_vars(py.activated_path)) - os.environ['PATH'] = os.environ['PATH'] + ';' + py_path - win32api, win32con = import_pywin32() if system: # Read globally from ALL USERS section. - folder = win32api.RegOpenKeyEx(win32con.HKEY_LOCAL_MACHINE, 'SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment', 0, win32con.KEY_ALL_ACCESS) + folder = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, 'SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment', 0, winreg.KEY_ALL_ACCESS) else: # Register locally from CURRENT USER section. - folder = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, 'Environment', 0, win32con.KEY_ALL_ACCESS) - win32api.RegSetValueEx(folder, key, 0, win32con.REG_EXPAND_SZ, value) + folder = winreg.OpenKeyEx(winreg.HKEY_CURRENT_USER, 'Environment', 0, winreg.KEY_ALL_ACCESS) + winreg.SetValueEx(folder, key, 0, winreg.REG_EXPAND_SZ, value) debug_print('Set key=' + key + ' with value ' + value + ' in registry.') + return True except Exception as e: # 'Access is denied.' - if e.args[0] == 5: + if e.args[3] == 5: exit_with_error('Error! Failed to set the environment variable \'' + key + '\'! Setting environment variables permanently requires administrator access. Please rerun this command with administrative privileges. This can be done for example by holding down the Ctrl and Shift keys while opening a command prompt in start menu.') - print('Failed to write environment variable ' + key + ':', file=sys.stderr) - print(str(e), file=sys.stderr) - win32api.RegCloseKey(folder) - os.environ['PATH'] = prev_path - return None - - win32api.RegCloseKey(folder) - os.environ['PATH'] = prev_path - win32api.PostMessage(win32con.HWND_BROADCAST, win32con.WM_SETTINGCHANGE, 0, 'Environment') + errlog('Failed to write environment variable ' + key + ':') + errlog(str(e)) + return False + finally: + if folder is not None: + folder.Close() -def win_get_environment_variable(key, system=True): - prev_path = os.environ['PATH'] +def win_get_environment_variable(key, system=True, user=True, fallback=True): + if (not system and not user and fallback): + # if no --system or --permanent flag is provided use shell's value + return os.environ[key] try: - py = find_used_python() - if py: - py_path = to_native_path(py.expand_vars(py.activated_path)) - os.environ['PATH'] = os.environ['PATH'] + ';' + py_path + folder = None try: - import win32api - import win32con if system: # Read globally from ALL USERS section. - folder = win32api.RegOpenKey(win32con.HKEY_LOCAL_MACHINE, 'SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment') + folder = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment') else: # Register locally from CURRENT USER section. - folder = win32api.RegOpenKey(win32con.HKEY_CURRENT_USER, 'Environment') - value = str(win32api.RegQueryValueEx(folder, key)[0]) + folder = winreg.OpenKey(winreg.HKEY_CURRENT_USER, 'Environment') + value = str(winreg.QueryValueEx(folder, key)[0]) except Exception: - # PyWin32 is not available - read via os.environ. This has the drawback + # If reading registry fails for some reason - read via os.environ. This has the drawback # that expansion items such as %PROGRAMFILES% will have been expanded, so # need to be precise not to set these back to system registry, or # expansion items would be lost. - return os.environ[key] + if fallback: + return os.environ[key] + return None + finally: + if folder is not None: + folder.Close() + except Exception as e: + # this catch is if both the registry key threw an exception and the key is not in os.environ if e.args[0] != 2: # 'The system cannot find the file specified.' - print('Failed to read environment variable ' + key + ':', file=sys.stderr) - print(str(e), file=sys.stderr) - try: - win32api.RegCloseKey(folder) - except Exception: - pass - os.environ['PATH'] = prev_path + errlog('Failed to read environment variable ' + key + ':') + errlog(str(e)) return None - win32api.RegCloseKey(folder) - os.environ['PATH'] = prev_path return value -def win_environment_variable_exists(key, system=True): - value = win_get_environment_variable(key, system) - return value is not None and len(value) > 0 - - -def win_get_active_environment_variable(key): - value = win_get_environment_variable(key, False) - if value is not None: - return value - return win_get_environment_variable(key, True) - - -def win_set_environment_variable(key, value, system=True): - debug_print('set ' + str(key) + '=' + str(value) + ', in system=' + str(system), file=sys.stderr) - previous_value = win_get_environment_variable(key, system) +def win_set_environment_variable(key, value, system, user): + debug_print('set ' + str(key) + '=' + str(value) + ', in system=' + str(system)) + previous_value = win_get_environment_variable(key, system=system, user=user) if previous_value == value: debug_print(' no need to set, since same value already exists.') # No need to elevate UAC for nothing to set the same value, skip. - return + return False if not value: try: @@ -396,31 +400,68 @@ def win_set_environment_variable(key, value, system=True): debug_print(str(cmd)) value = subprocess.call(cmd, stdout=subprocess.PIPE) except Exception: - return - return + return False + return True try: - if system: - win_set_environment_variable_direct(key, value, system) - return + if win_set_environment_variable_direct(key, value, system): + return True # Escape % signs so that we don't expand references to environment variables. value = value.replace('%', '^%') if len(value) >= 1024: - exit_with_error('ERROR! The new environment variable ' + key + ' is more than 1024 characters long! A value this long cannot be set via command line: please add the environment variable specified above to system environment manually via Control Panel.', file=sys.stderr) + exit_with_error('ERROR! The new environment variable ' + key + ' is more than 1024 characters long! A value this long cannot be set via command line: please add the environment variable specified above to system environment manually via Control Panel.') cmd = ['SETX', key, value] debug_print(str(cmd)) retcode = subprocess.call(cmd, stdout=subprocess.PIPE) if retcode != 0: - print('ERROR! Failed to set environment variable ' + key + '=' + value + '. You may need to set it manually.', file=sys.stderr) + errlog('ERROR! Failed to set environment variable ' + key + '=' + value + '. You may need to set it manually.') + else: + return True + except Exception as e: + errlog('ERROR! Failed to set environment variable ' + key + '=' + value + ':') + errlog(str(e)) + errlog('You may need to set it manually.') + + return False + + +def win_set_environment_variables(env_vars_to_add, system, user): + if not env_vars_to_add: + return + + changed = False + + for key, value in env_vars_to_add: + if win_set_environment_variable(key, value, system, user): + if not changed: + changed = True + print('Setting global environment variables:') + + print(key + ' = ' + value) + + if not changed: + print('Global environment variables up to date') + return + + # if changes were made then we need to notify other processes + try: + HWND_BROADCAST = ctypes.wintypes.HWND(0xFFFF) # win32con.HWND_BROADCAST == 65535 + WM_SETTINGCHANGE = 0x001A # win32con.WM_SETTINGCHANGE == 26 + SMTO_BLOCK = 0x0001 # win32con.SMTO_BLOCK == 1 + ctypes.windll.user32.SendMessageTimeoutA( + HWND_BROADCAST, # hWnd: notify everyone + WM_SETTINGCHANGE, # Msg: registry changed + 0, # wParam: Must be 0 when setting changed is sent by users + 'Environment', # lParam: Specifically environment variables changed + SMTO_BLOCK, # fuFlags: Wait for message to be sent or timeout + 100) # uTimeout: 100ms except Exception as e: - print('ERROR! Failed to set environment variable ' + key + '=' + value + ':', file=sys.stderr) - print(str(e), file=sys.stderr) - print('You may need to set it manually.', file=sys.stderr) + errlog('SendMessageTimeout failed with error: ' + str(e)) -def win_delete_environment_variable(key, system=True): +def win_delete_environment_variable(key, system=True, user=True): debug_print('win_delete_environment_variable(key=' + key + ', system=' + str(system) + ')') - win_set_environment_variable(key, None, system) + return win_set_environment_variable(key, None, system, user) # Returns the absolute pathname to the given path inside the Emscripten SDK. @@ -481,12 +522,12 @@ def num_files_in_directory(path): return len([name for name in os.listdir(path) if os.path.exists(os.path.join(path, name))]) -def run(cmd, cwd=None): +def run(cmd, cwd=None, quiet=False): debug_print('run(cmd=' + str(cmd) + ', cwd=' + str(cwd) + ')') process = subprocess.Popen(cmd, cwd=cwd, env=os.environ.copy()) process.communicate() - if process.returncode != 0: - print(str(cmd) + ' failed with error code ' + str(process.returncode) + '!') + if process.returncode != 0 and not quiet: + errlog(str(cmd) + ' failed with error code ' + str(process.returncode) + '!') return process.returncode @@ -513,7 +554,7 @@ def fix_potentially_long_windows_pathname(pathname): return pathname # Test if emsdk calls fix_potentially_long_windows_pathname() with long relative paths (which is problematic) if not os.path.isabs(pathname) and len(pathname) > 200: - print('Warning: Seeing a relative path "' + pathname + '" which is dangerously long for being referenced as a short Windows path name. Refactor emsdk to be able to handle this!') + errlog('Warning: Seeing a relative path "' + pathname + '" which is dangerously long for being referenced as a short Windows path name. Refactor emsdk to be able to handle this!') if pathname.startswith('\\\\?\\'): return pathname pathname = os.path.normpath(pathname.replace('/', '\\')) @@ -593,16 +634,13 @@ def unzip(source_filename, dest_dir, unpack_even_if_exists=False): move_with_overwrite(fix_potentially_long_windows_pathname(dst_filename), fix_potentially_long_windows_pathname(final_dst_filename)) if common_subdir: - try: - remove_tree(unzip_to_dir) - except: - pass + remove_tree(unzip_to_dir) except zipfile.BadZipfile as e: - print("Unzipping file '" + source_filename + "' failed due to reason: " + str(e) + "! Removing the corrupted zip file.") + errlog("Unzipping file '" + source_filename + "' failed due to reason: " + str(e) + "! Removing the corrupted zip file.") rmfile(source_filename) return False except Exception as e: - print("Unzipping file '" + source_filename + "' failed due to reason: " + str(e)) + errlog("Unzipping file '" + source_filename + "' failed due to reason: " + str(e)) return False return True @@ -681,7 +719,7 @@ def download_file(url, dstpath, download_even_if_exists=False, filename_prefix=' # Draw a progress bar 80 chars wide (in non-TTY mode) progress_max = 80 - 4 progress_shown = 0 - block_sz = 8192 + block_sz = 256 * 1024 if not TTY_OUTPUT: print(' [', end='') while True: @@ -705,9 +743,9 @@ def download_file(url, dstpath, download_even_if_exists=False, filename_prefix=' print(']') sys.stdout.flush() except Exception as e: - print("Error: Downloading URL '" + url + "': " + str(e)) + errlog("Error: Downloading URL '" + url + "': " + str(e)) if "SSL: CERTIFICATE_VERIFY_FAILED" in str(e) or "urlopen error unknown url type: https" in str(e): - print("Warning: Possibly SSL/TLS issue. Update or install Python SSL root certificates (2048-bit or greater) supplied in Python folder or https://pypi.org/project/certifi/ and try again.") + errlog("Warning: Possibly SSL/TLS issue. Update or install Python SSL root certificates (2048-bit or greater) supplied in Python folder or https://pypi.org/project/certifi/ and try again.") rmfile(file_name) return None except KeyboardInterrupt: @@ -742,7 +780,7 @@ def GIT(must_succeed=True): if must_succeed: if WINDOWS: msg = "ERROR: git executable was not found. Please install it by typing 'emsdk install git-1.9.4', or alternatively by installing it manually from http://git-scm.com/downloads . If you install git manually, remember to add it to PATH" - elif OSX: + elif MACOS: msg = "ERROR: git executable was not found. Please install git for this operation! This can be done from http://git-scm.com/ , or by installing XCode and then the XCode Command Line Tools (see http://stackoverflow.com/questions/9329243/xcode-4-4-command-line-tools )" elif LINUX: msg = "ERROR: git executable was not found. Please install git for this operation! This can be probably be done using your package manager, see http://git-scm.com/book/en/Getting-Started-Installing-Git" @@ -772,39 +810,43 @@ def git_recent_commits(repo_path, n=20): def git_clone(url, dstpath): debug_print('git_clone(url=' + url + ', dstpath=' + dstpath + ')') if os.path.isdir(os.path.join(dstpath, '.git')): - print("Repository '" + url + "' already cloned to directory '" + dstpath + "', skipping.") + debug_print("Repository '" + url + "' already cloned to directory '" + dstpath + "', skipping.") return True mkdir_p(dstpath) git_clone_args = [] if GIT_CLONE_SHALLOW: git_clone_args += ['--depth', '1'] + print('Cloning from ' + url + '...') return run([GIT(), 'clone'] + git_clone_args + [url, dstpath]) == 0 -def git_checkout_and_pull(repo_path, branch): - debug_print('git_checkout_and_pull(repo_path=' + repo_path + ', branch=' + branch + ')') - ret = run([GIT(), 'fetch', 'origin'], repo_path) +def git_checkout_and_pull(repo_path, branch_or_tag): + debug_print('git_checkout_and_pull(repo_path=' + repo_path + ', branch/tag=' + branch_or_tag + ')') + ret = run([GIT(), 'fetch', '--quiet', 'origin'], repo_path) if ret != 0: return False try: - print("Fetching latest changes to the branch '" + branch + "' for '" + repo_path + "'...") - ret = run([GIT(), 'fetch', 'origin'], repo_path) + print("Fetching latest changes to the branch/tag '" + branch_or_tag + "' for '" + repo_path + "'...") + ret = run([GIT(), 'fetch', '--quiet', 'origin'], repo_path) if ret != 0: return False - # run([GIT, 'checkout', '-b', branch, '--track', 'origin/'+branch], repo_path) # this line assumes that the user has not gone and manually messed with the # repo and added new remotes to ambiguate the checkout. - ret = run([GIT(), 'checkout', '--quiet', branch], repo_path) + ret = run([GIT(), 'checkout', '--quiet', branch_or_tag], repo_path) if ret != 0: return False - # this line assumes that the user has not gone and made local changes to the repo - ret = run([GIT(), 'merge', '--ff-only', 'origin/' + branch], repo_path) + # Test if branch_or_tag is a branch, or if it is a tag that needs to be updated + target_is_tag = run([GIT(), 'symbolic-ref', '-q', 'HEAD'], repo_path, quiet=True) + if not target_is_tag: + # update branch to latest (not needed for tags) + # this line assumes that the user has not gone and made local changes to the repo + ret = run([GIT(), 'merge', '--ff-only', 'origin/' + branch_or_tag], repo_path) if ret != 0: return False except: - print('git operation failed!') + errlog('git operation failed!') return False - print("Successfully updated and checked out branch '" + branch + "' on repository '" + repo_path + "'") + print("Successfully updated and checked out branch/tag '" + branch_or_tag + "' on repository '" + repo_path + "'") print("Current repository version: " + git_repo_version(repo_path)) return True @@ -821,7 +863,6 @@ def git_clone_checkout_and_pull(url, dstpath, branch): # Each tool can have its own build type, or it can be overridden on the command # line. def decide_cmake_build_type(tool): - global CMAKE_BUILD_TYPE_OVERRIDE if CMAKE_BUILD_TYPE_OVERRIDE: return CMAKE_BUILD_TYPE_OVERRIDE else: @@ -883,14 +924,36 @@ def build_env(generator): # To work around a build issue with older Mac OS X builds, add -stdlib=libc++ to all builds. # See https://groups.google.com/forum/#!topic/emscripten-discuss/5Or6QIzkqf0 - if OSX: + if MACOS: build_env['CXXFLAGS'] = ((build_env['CXXFLAGS'] + ' ') if hasattr(build_env, 'CXXFLAGS') else '') + '-stdlib=libc++' elif 'Visual Studio 15' in generator or 'Visual Studio 16' in generator: if 'Visual Studio 16' in generator: path = vswhere(16) else: path = vswhere(15) - build_env['VCTargetsPath'] = os.path.join(path, 'Common7\\IDE\\VC\\VCTargets') + + # Configuring CMake for Visual Studio needs and env. var VCTargetsPath to be present. + # How this is supposed to work is unfortunately very undocumented. See + # https://discourse.cmake.org/t/cmake-failed-to-get-the-value-of-vctargetspath-with-vs2019-16-7/1839/16 + # for some conversation. Try a couple of common paths if one of them would work. + # In the future as new versions of VS come out, we likely need to add new paths into this list. + if 'VCTargetsPath' not in build_env: + vctargets_paths = [ + os.path.join(path, 'MSBuild\\Microsoft\\VC\\v160\\'), + os.path.join(path, 'Common7\\IDE\\VC\\VCTargets') + ] + for p in vctargets_paths: + if os.path.isfile(os.path.join(p, 'Microsoft.Cpp.Default.props')): + debug_print('Set env. var VCTargetsPath=' + p + ' for CMake.') + build_env['VCTargetsPath'] = p + break + else: + debug_print('Searched path ' + p + ' as candidate for VCTargetsPath, not working.') + + if 'VCTargetsPath' not in build_env: + errlog('Unable to locate Visual Studio compiler installation for generator "' + generator + '"!') + errlog('Either rerun installation in Visual Studio Command Prompt, or locate directory to Microsoft.Cpp.Default.props manually') + sys.exit(1) # CMake and VS2017 cl.exe needs to have mspdb140.dll et al. in its PATH. vc_bin_paths = [vs_filewhere(path, 'amd64', 'cl.exe'), @@ -904,7 +967,7 @@ def build_env(generator): def get_generator_for_sln_file(sln_file): contents = open(sln_file, 'r').read() - if '# Visual Studio 16' in contents: # VS2019 + if '# Visual Studio 16' in contents or '# Visual Studio Version 16' in contents: # VS2019 return 'Visual Studio 16' if '# Visual Studio 15' in contents: # VS2017 return 'Visual Studio 15' @@ -940,7 +1003,6 @@ def find_msbuild(sln_file): def make_build(build_root, build_type, build_target_platform='x64'): debug_print('make_build(build_root=' + build_root + ', build_type=' + build_type + ', build_target_platform=' + build_target_platform + ')') - global CPU_CORES if CPU_CORES > 1: print('Performing a parallel build with ' + str(CPU_CORES) + ' cores.') else: @@ -971,13 +1033,13 @@ def make_build(build_root, build_type, build_target_platform='x64'): print('Running build: ' + str(make)) ret = subprocess.check_call(make, cwd=build_root, env=build_env(generator_to_use)) if ret != 0: - print('Build failed with exit code ' + ret + '!', file=sys.stderr) - print('Working directory: ' + build_root, file=sys.stderr) + errlog('Build failed with exit code ' + ret + '!') + errlog('Working directory: ' + build_root) return False except Exception as e: - print('Build failed due to exception!', file=sys.stderr) - print('Working directory: ' + build_root, file=sys.stderr) - print(str(e), file=sys.stderr) + errlog('Build failed due to exception!') + errlog('Working directory: ' + build_root) + errlog(str(e)) return False return True @@ -1008,25 +1070,25 @@ def quote_parens(x): open(os.path.join(build_root, 'recmake.' + ('bat' if WINDOWS else 'sh')), 'w').write(' '.join(map(quote_parens, cmdline))) ret = subprocess.check_call(cmdline, cwd=build_root, env=build_env(CMAKE_GENERATOR)) if ret != 0: - print('CMake invocation failed with exit code ' + ret + '!', file=sys.stderr) - print('Working directory: ' + build_root, file=sys.stderr) + errlog('CMake invocation failed with exit code ' + ret + '!') + errlog('Working directory: ' + build_root) return False except OSError as e: if e.errno == errno.ENOENT: - print(str(e), file=sys.stderr) - print('Could not run CMake, perhaps it has not been installed?', file=sys.stderr) + errlog(str(e)) + errlog('Could not run CMake, perhaps it has not been installed?') if WINDOWS: - print('Installing this package requires CMake. Get it from http://www.cmake.org/', file=sys.stderr) + errlog('Installing this package requires CMake. Get it from http://www.cmake.org/') elif LINUX: - print('Installing this package requires CMake. Get it via your system package manager (e.g. sudo apt-get install cmake), or from http://www.cmake.org/', file=sys.stderr) - elif OSX: - print('Installing this package requires CMake. Get it via a OSX package manager (Homebrew: "brew install cmake", or MacPorts: "sudo port install cmake"), or from http://www.cmake.org/', file=sys.stderr) + errlog('Installing this package requires CMake. Get it via your system package manager (e.g. sudo apt-get install cmake), or from http://www.cmake.org/') + elif MACOS: + errlog('Installing this package requires CMake. Get it via a macOS package manager (Homebrew: "brew install cmake", or MacPorts: "sudo port install cmake"), or from http://www.cmake.org/') return False raise except Exception as e: - print('CMake invocation failed due to exception!', file=sys.stderr) - print('Working directory: ' + build_root, file=sys.stderr) - print(str(e), file=sys.stderr) + errlog('CMake invocation failed due to exception!') + errlog('Working directory: ' + build_root) + errlog(str(e)) return False return True @@ -1042,8 +1104,8 @@ def xcode_sdk_version(): return subprocess.checkplatform.mac_ver()[0].split('.') -def build_llvm_fastcomp(tool): - debug_print('build_llvm_fastcomp(' + str(tool) + ')') +def build_fastcomp(tool): + debug_print('build_fastcomp(' + str(tool) + ')') fastcomp_root = tool.installation_path() fastcomp_src_root = os.path.join(fastcomp_root, 'src') # Does this tool want to be git cloned from github? @@ -1087,7 +1149,6 @@ def build_llvm_fastcomp(tool): build_type = decide_cmake_build_type(tool) # Configure - global BUILD_FOR_TESTING, ENABLE_LLVM_ASSERTIONS tests_arg = 'ON' if BUILD_FOR_TESTING else 'OFF' enable_assertions = ENABLE_LLVM_ASSERTIONS.lower() == 'on' or (ENABLE_LLVM_ASSERTIONS == 'auto' and build_type.lower() != 'release' and build_type.lower() != 'minsizerel') @@ -1114,7 +1175,7 @@ def build_llvm_fastcomp(tool): # MacOS < 10.13 workaround for LLVM build bug https://github.com/kripken/emscripten/issues/5418: # specify HAVE_FUTIMENS=0 in the build if building with target SDK that is older than 10.13. - if OSX and (not os.environ.get('LLVM_CMAKE_ARGS') or 'HAVE_FUTIMENS' not in os.environ.get('LLVM_CMAKE_ARGS')) and xcode_sdk_version() < ['10', '13']: + if MACOS and (not os.environ.get('LLVM_CMAKE_ARGS') or 'HAVE_FUTIMENS' not in os.environ.get('LLVM_CMAKE_ARGS')) and xcode_sdk_version() < ['10', '13']: print('Passing -DHAVE_FUTIMENS=0 to LLVM CMake configure to workaround https://github.com/kripken/emscripten/issues/5418. Please update to macOS 10.13 or newer') args += ['-DHAVE_FUTIMENS=0'] @@ -1127,9 +1188,10 @@ def build_llvm_fastcomp(tool): return success -# LLVM git source tree migrated to a single repository instead of multiple ones, build_llvm_monorepo() builds via that repository structure -def build_llvm_monorepo(tool): - debug_print('build_llvm_monorepo(' + str(tool) + ')') +# LLVM git source tree migrated to a single repository instead of multiple +# ones, build_llvm() builds via that repository structure +def build_llvm(tool): + debug_print('build_llvm(' + str(tool) + ')') llvm_root = tool.installation_path() llvm_src_root = os.path.join(llvm_root, 'src') success = git_clone_checkout_and_pull(tool.download_url(), llvm_src_root, tool.git_branch) @@ -1142,7 +1204,6 @@ def build_llvm_monorepo(tool): build_type = decide_cmake_build_type(tool) # Configure - global BUILD_FOR_TESTING, ENABLE_LLVM_ASSERTIONS tests_arg = 'ON' if BUILD_FOR_TESTING else 'OFF' enable_assertions = ENABLE_LLVM_ASSERTIONS.lower() == 'on' or (ENABLE_LLVM_ASSERTIONS == 'auto' and build_type.lower() != 'release' and build_type.lower() != 'minsizerel') @@ -1196,6 +1257,111 @@ def build_llvm_monorepo(tool): return success +def build_ninja(tool): + debug_print('build_ninja(' + str(tool) + ')') + root = os.path.normpath(tool.installation_path()) + src_root = os.path.join(root, 'src') + success = git_clone_checkout_and_pull(tool.download_url(), src_root, tool.git_branch) + if not success: + return False + + build_dir = llvm_build_dir(tool) + build_root = os.path.join(root, build_dir) + + build_type = decide_cmake_build_type(tool) + + # Configure + cmake_generator = CMAKE_GENERATOR + args = [] + if 'Visual Studio 16' in CMAKE_GENERATOR: # VS2019 + # With Visual Studio 16 2019, CMake changed the way they specify target arch. + # Instead of appending it into the CMake generator line, it is specified + # with a -A arch parameter. + args += ['-A', 'x64' if tool.bitness == 64 else 'x86'] + args += ['-Thost=x64'] + elif 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64: + cmake_generator += ' Win64' + args += ['-Thost=x64'] + + cmakelists_dir = os.path.join(src_root) + success = cmake_configure(cmake_generator, build_root, cmakelists_dir, build_type, args) + if not success: + return False + + # Make + success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32') + + if success: + bin_dir = os.path.join(root, 'bin') + mkdir_p(bin_dir) + exe_paths = [os.path.join(build_root, 'Release', 'ninja'), os.path.join(build_root, 'ninja')] + for e in exe_paths: + for s in ['.exe', '']: + ninja = e + s + if os.path.isfile(ninja): + dst = os.path.join(bin_dir, 'ninja' + s) + shutil.copyfile(ninja, dst) + os.chmod(dst, os.stat(dst).st_mode | stat.S_IEXEC) + + return success + + +def build_ccache(tool): + debug_print('build_ccache(' + str(tool) + ')') + root = os.path.normpath(tool.installation_path()) + src_root = os.path.join(root, 'src') + success = git_clone_checkout_and_pull(tool.download_url(), src_root, tool.git_branch) + if not success: + return False + + build_dir = llvm_build_dir(tool) + build_root = os.path.join(root, build_dir) + + build_type = decide_cmake_build_type(tool) + + # Configure + cmake_generator = CMAKE_GENERATOR + args = ['-DZSTD_FROM_INTERNET=ON'] + if 'Visual Studio 16' in CMAKE_GENERATOR: # VS2019 + # With Visual Studio 16 2019, CMake changed the way they specify target arch. + # Instead of appending it into the CMake generator line, it is specified + # with a -A arch parameter. + args += ['-A', 'x64' if tool.bitness == 64 else 'x86'] + args += ['-Thost=x64'] + elif 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64: + cmake_generator += ' Win64' + args += ['-Thost=x64'] + + cmakelists_dir = os.path.join(src_root) + success = cmake_configure(cmake_generator, build_root, cmakelists_dir, build_type, args) + if not success: + return False + + # Make + success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32') + + if success: + bin_dir = os.path.join(root, 'bin') + mkdir_p(bin_dir) + exe_paths = [os.path.join(build_root, 'Release', 'ccache'), os.path.join(build_root, 'ccache')] + for e in exe_paths: + for s in ['.exe', '']: + ccache = e + s + if os.path.isfile(ccache): + dst = os.path.join(bin_dir, 'ccache' + s) + shutil.copyfile(ccache, dst) + os.chmod(dst, os.stat(dst).st_mode | stat.S_IEXEC) + + cache_dir = os.path.join(root, 'cache') + open(os.path.join(root, 'emcc_ccache.conf'), 'w').write('''# Set maximum cache size to 10 GB: +max_size = 10G +cache_dir = %s +''' % cache_dir) + mkdir_p(cache_dir) + + return success + + # Emscripten asm.js optimizer build scripts: def optimizer_build_root(tool): build_root = tool.installation_path().strip() @@ -1210,11 +1376,7 @@ def uninstall_optimizer(tool): debug_print('uninstall_optimizer(' + str(tool) + ')') build_root = optimizer_build_root(tool) print("Deleting path '" + build_root + "'") - try: - remove_tree(build_root) - os.remove(build_root) - except: - pass + remove_tree(build_root) def is_optimizer_installed(tool): @@ -1233,21 +1395,27 @@ def find_latest_installed_tool(name): def emscripten_npm_install(tool, directory): node_tool = find_latest_installed_tool('node') if not node_tool: - print('Failed to run "npm ci" in installed Emscripten root directory ' + tool.installation_path() + '! Please install node.js first!') - return False + npm_fallback = which('npm') + if not npm_fallback: + errlog('Failed to find npm command!') + errlog('Running "npm ci" in installed Emscripten root directory ' + tool.installation_path() + ' is required!') + errlog('Please install node.js first!') + return False + node_path = os.path.dirname(npm_fallback) + else: + node_path = os.path.join(node_tool.installation_path(), 'bin') - node_path = os.path.join(node_tool.installation_path(), 'bin') npm = os.path.join(node_path, 'npm' + ('.cmd' if WINDOWS else '')) env = os.environ.copy() env["PATH"] = node_path + os.pathsep + env["PATH"] print('Running post-install step: npm ci ...') try: subprocess.check_output( - [npm, 'ci', '--production'], + [npm, 'ci', '--production', '--no-optional'], cwd=directory, stderr=subprocess.STDOUT, env=env, universal_newlines=True) except subprocess.CalledProcessError as e: - print('Error running %s:\n%s' % (e.cmd, e.output)) + errlog('Error running %s:\n%s' % (e.cmd, e.output)) return False print('Done running: npm ci') @@ -1300,11 +1468,7 @@ def uninstall_binaryen(tool): debug_print('uninstall_binaryen(' + str(tool) + ')') build_root = binaryen_build_root(tool) print("Deleting path '" + build_root + "'") - try: - remove_tree(build_root) - os.remove(build_root) - except: - pass + remove_tree(build_root) def is_binaryen_installed(tool): @@ -1405,7 +1569,7 @@ def get_required_path(active_tools): # Returns the absolute path to the file '.emscripten' for the current user on # this system. def dot_emscripten_path(): - return os.path.join(emscripten_config_directory, ".emscripten") + return os.path.join(emsdk_path(), ".emscripten") dot_emscripten = {} @@ -1435,26 +1599,13 @@ def load_dot_emscripten(): key, value = parse_key_value(line) if value != '': dot_emscripten[key] = value - # print("Got '" + key + "' = '" + value + "'") except: pass def generate_dot_emscripten(active_tools): - global emscripten_config_directory - if emscripten_config_directory == emsdk_path(): - temp_dir = sdk_path('tmp') - mkdir_p(temp_dir) - embedded = True - else: - temp_dir = tempfile.gettempdir().replace('\\', '/') - embedded = False - - cfg = '' - - if embedded: - cfg += 'import os\n' - cfg += "emsdk_path = os.path.dirname(os.environ.get('EM_CONFIG')).replace('\\\\', '/')\n" + cfg = 'import os\n' + cfg += "emsdk_path = os.path.dirname(os.environ.get('EM_CONFIG')).replace('\\\\', '/')\n" # Different tools may provide the same activated configs; the latest to be # activated is the relevant one. @@ -1467,47 +1618,40 @@ def generate_dot_emscripten(active_tools): node_fallback = which('nodejs') if not node_fallback: node_fallback = 'node' - activated_config['NODE_JS'] = "'%s'" % node_fallback + activated_config['NODE_JS'] = node_fallback for name, value in activated_config.items(): cfg += name + " = '" + value + "'\n" cfg += '''\ -TEMP_DIR = '%s' COMPILER_ENGINE = NODE_JS JS_ENGINES = [NODE_JS] -''' % temp_dir +''' - if embedded: - cfg = cfg.replace("'" + emscripten_config_directory, "emsdk_path + '") + cfg = cfg.replace("'" + emsdk_path(), "emsdk_path + '") if os.path.exists(dot_emscripten_path()): backup_path = dot_emscripten_path() + ".old" - print("Backing up old Emscripten configuration file in " + os.path.normpath(backup_path)) move_with_overwrite(dot_emscripten_path(), backup_path) with open(dot_emscripten_path(), "w") as text_file: text_file.write(cfg) # Clear old emscripten content. - try: - os.remove(os.path.join(emscripten_config_directory, ".emscripten_sanity")) - except: - pass - - print("The Emscripten configuration file " + os.path.normpath(dot_emscripten_path()) + " has been rewritten with the following contents:") - print('') - print(cfg.strip()) - print('') + rmfile(os.path.join(emsdk_path(), ".emscripten_sanity")) path_add = get_required_path(active_tools) if not WINDOWS: - emsdk_env = os.path.relpath(sdk_path('emsdk_env.sh')) - if '/' not in emsdk_env: - emsdk_env = './emsdk_env.sh' - print("To conveniently access the selected set of tools from the command line, consider adding the following directories to PATH, or call 'source " + emsdk_env + "' to do this for you.") - print('') - print(' ' + ENVPATH_SEPARATOR.join(path_add)) + emsdk_env = sdk_path('emsdk_env.sh') + print('Next steps:') + print('- To conveniently access emsdk tools from the command line,') + print(' consider adding the following directories to your PATH:') + for p in path_add: + print(' ' + p) + print('- This can be done for the current shell by running:') + print(' source "%s"' % emsdk_env) + print('- Configure emsdk in your bash profile by running:') + print(' echo \'source "%s"\' >> $HOME/.bash_profile' % emsdk_env) def find_msbuild_dir(): @@ -1529,13 +1673,6 @@ def find_msbuild_dir(): return '' -def get_installed_vstool_version(installed_path): - try: - return open(installed_path + "/version.txt", "r").read() - except: - return None - - class Tool(object): def __init__(self, data): # Convert the dictionary representation of the tool in 'data' to members of @@ -1547,7 +1684,9 @@ def __init__(self, data): setattr(self, key, value) # Cache the name ID of this Tool (these are read very often) - self.name = self.id + '-' + self.version + self.name = self.id + if self.version: + self.name += '-' + self.version if hasattr(self, 'bitness'): self.name += '-' + str(self.bitness) + 'bit' @@ -1639,15 +1778,15 @@ def compatible_with_this_os(self): if hasattr(self, 'os'): if self.os == 'all': return True - if self.compatible_with_this_arch() and ((WINDOWS and 'win' in self.os) or (LINUX and ('linux' in self.os or 'unix' in self.os)) or (OSX and ('osx' in self.os or 'unix' in self.os))): + if self.compatible_with_this_arch() and ((WINDOWS and 'win' in self.os) or (LINUX and ('linux' in self.os or 'unix' in self.os)) or (MACOS and ('macos' in self.os or 'unix' in self.os))): return True else: return False else: - if not hasattr(self, 'osx_url') and not hasattr(self, 'windows_url') and not hasattr(self, 'unix_url') and not hasattr(self, 'linux_url'): + if not hasattr(self, 'macos_url') and not hasattr(self, 'windows_url') and not hasattr(self, 'unix_url') and not hasattr(self, 'linux_url'): return True - if OSX and hasattr(self, 'osx_url') and self.compatible_with_this_arch(): + if MACOS and hasattr(self, 'macos_url') and self.compatible_with_this_arch(): return True if LINUX and hasattr(self, 'linux_url') and self.compatible_with_this_arch(): @@ -1661,14 +1800,34 @@ def compatible_with_this_os(self): return hasattr(self, 'url') - def is_installed(self): + # the "version file" is a file inside install dirs that indicates the + # version installed there. this helps disambiguate when there is more than + # one version that may be installed to the same directory (which is used + # to avoid accumulating builds over time in some cases, with new builds + # overwriting the old) + def get_version_file_path(self): + return os.path.join(self.installation_path(), '.emsdk_version') + + def is_installed_version(self): + version_file_path = self.get_version_file_path() + if os.path.isfile(version_file_path): + with open(version_file_path, 'r') as version_file: + return version_file.read().strip() == self.name + return False + + def update_installed_version(self): + with open(self.get_version_file_path(), 'w') as version_file: + version_file.write(self.name + '\n') + return None + + def is_installed(self, skip_version_check=False): # If this tool/sdk depends on other tools, require that all dependencies are # installed for this tool to count as being installed. if hasattr(self, 'uses'): for tool_name in self.uses: tool = find_tool(tool_name) if tool is None: - print("Manifest error: No tool by name '" + tool_name + "' found! This may indicate an internal SDK error!") + errlog("Manifest error: No tool by name '" + tool_name + "' found! This may indicate an internal SDK error!") return False if not tool.is_installed(): return False @@ -1677,27 +1836,17 @@ def is_installed(self): # This tool does not contain downloadable elements, so it is installed by default. return True + content_exists = os.path.exists(self.installation_path()) and (os.path.isfile(self.installation_path()) or num_files_in_directory(self.installation_path()) > 0) + # For e.g. fastcomp clang from git repo, the activated PATH is the # directory where the compiler is built to, and installation_path is # the directory where the source tree exists. To distinguish between - # multiple packages sharing the same source (clang-master-32bit, - # clang-master-64bit, clang-master-32bit and clang-master-64bit each + # multiple packages sharing the same source (clang-main-32bit, + # clang-main-64bit, clang-main-32bit and clang-main-64bit each # share the same git repo), require that in addition to the installation # directory, each item in the activated PATH must exist. - if hasattr(self, 'activated_path'): - activated_path = self.expand_vars(self.activated_path).split(';') - else: - activated_path = [self.installation_path()] - - def each_path_exists(pathlist): - return all(os.path.exists(p) for p in pathlist) - - content_exists = os.path.exists(self.installation_path()) and each_path_exists(activated_path) and (os.path.isfile(self.installation_path()) or num_files_in_directory(self.installation_path()) > 0) - - # vs-tool is a special tool since all versions must be installed to the - # same dir, so dir name will not differentiate the version. - if self.id == 'vs-tool': - return content_exists and get_installed_vstool_version(self.installation_path()) == self.version + if hasattr(self, 'activated_path') and not os.path.exists(self.expand_vars(self.activated_path)): + content_exists = False if hasattr(self, 'custom_is_installed_script'): if self.custom_is_installed_script == 'is_optimizer_installed': @@ -1707,7 +1856,7 @@ def each_path_exists(pathlist): else: raise Exception('Unknown custom_is_installed_script directive "' + self.custom_is_installed_script + '"!') - return content_exists + return content_exists and (skip_version_check or self.is_installed_version()) def is_active(self): if not self.is_installed(): @@ -1733,8 +1882,8 @@ def is_active(self): debug_print(str(self) + ' is not active, because key="' + key + '" does not exist in .emscripten') return False - # If running in embedded mode, all paths are stored dynamically relative - # to the emsdk root, so normalize those first. + # all paths are stored dynamically relative to the emsdk root, so + # normalize those first. dot_emscripten_key = dot_emscripten[key].replace("emsdk_path + '", "'" + emsdk_path()) dot_emscripten_key = dot_emscripten_key.strip("'") if dot_emscripten_key != value: @@ -1752,8 +1901,7 @@ def is_env_active(self): return False if hasattr(self, 'activated_path'): - path = self.expand_vars(self.activated_path).replace('\\', '/') - path = path.split(ENVPATH_SEPARATOR) + path = to_unix_path(self.expand_vars(self.activated_path)) for p in path: path_items = os.environ['PATH'].replace('\\', '/').split(ENVPATH_SEPARATOR) if not normalized_contains(path_items, p): @@ -1761,19 +1909,6 @@ def is_env_active(self): return False return True - def win_activate_env_vars(self, permanently_activate): - if WINDOWS: - envs = self.activated_environment() - for env in envs: - key, value = parse_key_value(env) - - if permanently_activate: - # If there is an env var for the LOCAL USER with same name, it will - # hide the system var, so must remove that first. - win_delete_environment_variable(key, False) - - win_set_environment_variable(key, value, permanently_activate) - # If this tool can be installed on this system, this function returns True. # Otherwise, this function returns a string that describes the reason why this # tool is not available. @@ -1784,7 +1919,7 @@ def can_be_installed(self): if self.id == 'vs-tool': msbuild_dir = find_msbuild_dir() - if len(msbuild_dir) > 0: + if msbuild_dir: return True else: return "Visual Studio was not found!" @@ -1794,8 +1929,8 @@ def can_be_installed(self): def download_url(self): if WINDOWS and hasattr(self, 'windows_url'): return self.windows_url - elif OSX and hasattr(self, 'osx_url'): - return self.osx_url + elif MACOS and hasattr(self, 'macos_url'): + return self.macos_url elif LINUX and hasattr(self, 'linux_url'): return self.linux_url elif UNIX and hasattr(self, 'unix_url'): @@ -1806,9 +1941,11 @@ def download_url(self): return None def install(self): + """Returns True if the Tool was installed of False if was skipped due to + already being installed. + """ if self.can_be_installed() is not True: - print("The tool '" + str(self) + "' is not available due to the reason: " + self.can_be_installed()) - return False + exit_with_error("The tool '" + str(self) + "' is not available due to the reason: " + self.can_be_installed()) if self.id == 'sdk': return self.install_sdk() @@ -1816,51 +1953,58 @@ def install(self): return self.install_tool() def install_sdk(self): + """Returns True if any SDK component was installed of False all componented + were already installed. + """ print("Installing SDK '" + str(self) + "'..") + installed = False for tool_name in self.uses: tool = find_tool(tool_name) if tool is None: - print("Manifest error: No tool by name '" + tool_name + "' found! This may indicate an internal SDK error!") - success = tool.install() - if not success: - return False + exit_with_error("Manifest error: No tool by name '" + tool_name + "' found! This may indicate an internal SDK error!") + installed |= tool.install() + + if not installed: + print("All SDK components already installed: '" + str(self) + "'.") + return False + if getattr(self, 'custom_install_script', None) == 'emscripten_npm_install': # upstream tools have hardcoded paths that are not stored in emsdk_manifest.json registry install_path = 'upstream' if 'releases-upstream' in self.version else 'fastcomp' - success = emscripten_npm_install(self, os.path.join(emsdk_path(), install_path, 'emscripten')) - if not success: - return False + emscripten_dir = os.path.join(emsdk_path(), install_path, 'emscripten') + # Older versions of the sdk did not include the node_modules directory + # and require `npm ci` to be run post-install + if not os.path.exists(os.path.join(emscripten_dir, 'node_modules')): + if not emscripten_npm_install(self, emscripten_dir): + exit_with_error('post-install step failed: emscripten_npm_install') print("Done installing SDK '" + str(self) + "'.") return True def install_tool(self): - # We should not force reinstallation of python if it already exists, since that very python - # may be interpreting the current emsdk.py script we are executing. On Windows this would - # lead to a failure to uncompress the python zip file as the python executable files are in use. - # TODO: Refactor codebase to avoid needing this kind of special case check by being more - # careful about reinstallation, see https://github.com/emscripten-core/emsdk/pull/394#issuecomment-559386468 - # for a scheme that would work. - if self.id == 'python' and self.is_installed(): + """Returns True if the SDK was installed of False if was skipped due to + already being installed. + """ + # Avoid doing a redundant reinstall of the tool, if it has already been installed. + # However all tools that are sourced directly from git branches do need to be + # installed every time when requested, since the install step is then used to git + # pull the tool to a newer version. + if self.is_installed() and not hasattr(self, 'git_branch'): print("Skipped installing " + self.name + ", already installed.") - return True - - version_id = self.name - version_file_path = os.path.join(self.installation_path(), '.emsdk_version') - if os.path.isfile(version_file_path): - with open(version_file_path, 'r') as version_file: - if version_id == version_file.read(): - print("Skipped installing " + self.name + ", already installed.") - return True + return False print("Installing tool '" + str(self) + "'..") url = self.download_url() if hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_fastcomp': - success = build_llvm_fastcomp(self) - elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_llvm_monorepo': - success = build_llvm_monorepo(self) + success = build_fastcomp(self) + elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_llvm': + success = build_llvm(self) + elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_ninja': + success = build_ninja(self) + elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_ccache': + success = build_ccache(self) elif hasattr(self, 'git_branch'): success = git_clone_checkout_and_pull(url, self.installation_path(), self.git_branch) elif url.endswith(ARCHIVE_SUFFIXES): @@ -1880,46 +2024,44 @@ def install_tool(self): else: success = False - if success: - if hasattr(self, 'custom_install_script'): - if self.custom_install_script == 'emscripten_post_install': - success = emscripten_post_install(self) - elif self.custom_install_script == 'emscripten_npm_install': - success = emscripten_npm_install(self, self.installation_path()) - elif self.custom_install_script in ('build_fastcomp', 'build_llvm_monorepo'): - # 'build_fastcomp' is a special one that does the download on its - # own, others do the download manually. - pass - elif self.custom_install_script == 'build_binaryen': - success = build_binaryen_tool(self) - else: - raise Exception('Unknown custom_install_script command "' + self.custom_install_script + '"!') - - # Install an emscripten-version.txt file if told to, and if there is one. - # (If this is not an actual release, but some other build, then we do not - # write anything.) - if hasattr(self, 'emscripten_releases_hash'): - emscripten_version_file_path = os.path.join(to_native_path(self.expand_vars(self.activated_path)), 'emscripten-version.txt') - version = get_emscripten_release_version(self.emscripten_releases_hash) - if version: - open(emscripten_version_file_path, 'w').write('"%s"' % version) + if not success: + exit_with_error("Installation failed!") + + if hasattr(self, 'custom_install_script'): + if self.custom_install_script == 'emscripten_post_install': + success = emscripten_post_install(self) + elif self.custom_install_script == 'emscripten_npm_install': + success = emscripten_npm_install(self, self.installation_path()) + elif self.custom_install_script in ('build_fastcomp', 'build_llvm', 'build_ninja', 'build_ccache'): + # 'build_fastcomp' is a special one that does the download on its + # own, others do the download manually. + pass + elif self.custom_install_script == 'build_binaryen': + success = build_binaryen_tool(self) + else: + raise Exception('Unknown custom_install_script command "' + self.custom_install_script + '"!') if not success: - print("Installation failed!") - return False + exit_with_error("Installation failed!") + + # Install an emscripten-version.txt file if told to, and if there is one. + # (If this is not an actual release, but some other build, then we do not + # write anything.) + if hasattr(self, 'emscripten_releases_hash'): + emscripten_version_file_path = os.path.join(to_native_path(self.expand_vars(self.activated_path)), 'emscripten-version.txt') + version = get_emscripten_release_version(self.emscripten_releases_hash) + if version: + open(emscripten_version_file_path, 'w').write('"%s"' % version) print("Done installing tool '" + str(self) + "'.") # Sanity check that the installation succeeded, and if so, remove unneeded # leftover installation files. - if self.is_installed(): - self.cleanup_temp_install_files() - with open(version_file_path, 'w') as version_file: - version_file.write(version_id) - else: - print("Installation of '" + str(self) + "' failed, but no error was detected. Either something went wrong with the installation, or this may indicate an internal emsdk error.") - return False + if not self.is_installed(skip_version_check=True): + exit_with_error("Installation of '" + str(self) + "' failed, but no error was detected. Either something went wrong with the installation, or this may indicate an internal emsdk error.") + self.cleanup_temp_install_files() + self.update_installed_version() return True def cleanup_temp_install_files(self): @@ -1941,12 +2083,8 @@ def uninstall(self): uninstall_binaryen(self) else: raise Exception('Unknown custom_uninstall_script directive "' + self.custom_uninstall_script + '"!') - try: - print("Deleting path '" + self.installation_path() + "'") - remove_tree(self.installation_path()) - os.remove(self.installation_path()) - except: - pass + print("Deleting path '" + self.installation_path() + "'") + remove_tree(self.installation_path()) print("Done uninstalling '" + str(self) + "'.") def dependencies(self): @@ -2028,19 +2166,19 @@ def find_latest_releases_sdk(which): return 'sdk-releases-%s-%s-64bit' % (which, find_latest_releases_hash()) -def find_tot(): - return open(tot_path()).read().strip() +def find_tot_sdk(): + debug_print('Fetching emscripten-releases repository...') + global extra_release_tag + extra_release_tag = get_emscripten_releases_tot() + return 'sdk-releases-upstream-%s-64bit' % (extra_release_tag) -def find_tot_sdk(which): - if not os.path.exists(tot_path()): - print('Tip-of-tree information was not found, run emsdk update-tags') - sys.exit(1) - tot = find_tot() - if not tot: - print('Tip-of-tree build was not found, run emsdk update-tags (however, if there is no recent tip-of-tree build, you may need to wait)') - sys.exit(1) - return 'sdk-releases-%s-%s-64bit' % (which, tot) +def parse_emscripten_version(emscripten_root): + version_file = os.path.join(emscripten_root, 'emscripten-version.txt') + with open(version_file) as f: + version = f.read().strip() + version = version.strip('"').split('.') + return [int(v) for v in version] # Given a git hash in emscripten-releases, find the emscripten @@ -2054,10 +2192,6 @@ def get_emscripten_release_version(emscripten_releases_hash): return None -def tot_path(): - return sdk_path('emscripten-releases-tot.txt') - - # Get the tip-of-tree build identifier. def get_emscripten_releases_tot(): git_clone_checkout_and_pull(emscripten_releases_repo, sdk_path('releases'), 'master') @@ -2076,22 +2210,11 @@ def get_emscripten_releases_tot(): except: continue return release - return '' + exit_with_error('failed to find build of any recent emsdk revision') -# Finds the best-matching python tool for use. -def find_used_python(): - # Find newest tool first - those are always at the end of the list. - for t in reversed(tools): - if t.id == 'python' and t.is_installed() and t.is_active() and t.is_env_active(): - return t - for t in reversed(tools): - if t.id == 'python' and t.is_installed() and t.is_active(): - return t - for t in reversed(tools): - if t.id == 'python' and t.is_installed(): - return t - return None +def get_release_hash(arg, releases_info): + return releases_info.get(arg, None) or releases_info.get('sdk-' + arg + '-64bit') def version_key(ver): @@ -2107,55 +2230,26 @@ def python_2_3_sorted(arr, cmp): return sorted(arr, cmp=cmp) -def fetch_emscripten_tags(): - git = GIT(must_succeed=False) - - if git: - print('Fetching emscripten-releases repository...') - emscripten_releases_tot = get_emscripten_releases_tot() - if emscripten_releases_tot: - open(tot_path(), 'w').write(emscripten_releases_tot) - else: - print('Update complete, however skipped fetching the Emscripten tags, since git was not found, which is necessary for update-tags.') - if WINDOWS: - print("Please install git by typing 'emsdk install git-1.9.4', or alternatively by installing it manually from http://git-scm.com/downloads . If you install git manually, remember to add it to PATH.") - elif OSX: - print("Please install git from http://git-scm.com/ , or by installing XCode and then the XCode Command Line Tools (see http://stackoverflow.com/questions/9329243/xcode-4-4-command-line-tools ).") - elif LINUX: - print("Pease install git using your package manager, see http://git-scm.com/book/en/Getting-Started-Installing-Git .") - else: - print("Please install git.") - return - - def is_emsdk_sourced_from_github(): return os.path.exists(os.path.join(emsdk_path(), '.git')) def update_emsdk(): if is_emsdk_sourced_from_github(): - print('You seem to have bootstrapped Emscripten SDK by cloning from GitHub. In this case, use "git pull" instead of "emsdk update" to update emsdk. (Not doing that automatically in case you have local changes)', file=sys.stderr) - print('Alternatively, use "emsdk update-tags" to refresh the latest list of tags from the different Git repositories.', file=sys.stderr) + errlog('You seem to have bootstrapped Emscripten SDK by cloning from GitHub. In this case, use "git pull" instead of "emsdk update" to update emsdk. (Not doing that automatically in case you have local changes)') sys.exit(1) if not download_and_unzip(emsdk_zip_download_url, emsdk_path(), download_even_if_exists=True, clobber=False): sys.exit(1) - fetch_emscripten_tags() # Lists all legacy (pre-emscripten-releases) tagged versions directly in the Git # repositories. These we can pull and compile from source. def load_legacy_emscripten_tags(): - try: - return open(sdk_path('legacy-emscripten-tags.txt'), 'r').read().split('\n') - except: - return [] + return open(sdk_path('legacy-emscripten-tags.txt'), 'r').read().split('\n') def load_legacy_binaryen_tags(): - try: - return open(sdk_path('legacy-binaryen-tags.txt'), 'r').read().split('\n') - except: - return [] + return open(sdk_path('legacy-binaryen-tags.txt'), 'r').read().split('\n') def remove_prefix(s, prefix): @@ -2174,52 +2268,64 @@ def remove_suffix(s, suffix): # filename should be one of: 'llvm-precompiled-tags-32bit.txt', 'llvm-precompiled-tags-64bit.txt' def load_file_index_list(filename): - try: - items = open(sdk_path(filename), 'r').read().split('\n') - items = map(lambda x: remove_suffix(remove_suffix(remove_prefix(x, 'emscripten-llvm-e'), '.tar.gz'), '.zip').strip(), items) - items = filter(lambda x: 'latest' not in x and len(x) > 0, items) - - # Sort versions from oldest to newest (the default sort would be lexicographic, i.e. '1.37.1 < 1.37.10 < 1.37.2') - items = sorted(items, key=version_key)[::-1] - - return items - except: - return [] - - -def load_llvm_precompiled_tags_32bit(): - return load_file_index_list('llvm-tags-32bit.txt') + items = open(sdk_path(filename)).read().splitlines() + items = [remove_suffix(remove_suffix(remove_prefix(x, 'emscripten-llvm-e'), '.tar.gz'), '.zip').strip() for x in items] + items = [x for x in items if 'latest' not in x and len(x) > 0] - -def load_llvm_precompiled_tags_64bit(): - return load_file_index_list('llvm-tags-64bit.txt') + # Sort versions from oldest to newest (the default sort would be + # lexicographic, i.e. '1.37.1 < 1.37.10 < 1.37.2') + return sorted(items, key=version_key) def exit_with_error(msg): - sys.stdout.write(str(msg) + '\n') + errlog(str(msg)) sys.exit(1) # Load the json info for emscripten-releases. def load_releases_info(): - try: - text = open(sdk_path('emscripten-releases-tags.txt'), 'r').read() - return json.loads(text) - except Exception as e: - print('Error parsing emscripten-releases-tags.txt!') - exit_with_error(str(e)) + if not hasattr(load_releases_info, 'cached_info'): + try: + text = open(sdk_path('emscripten-releases-tags.txt'), 'r').read() + load_releases_info.cached_info = json.loads(text) + except Exception as e: + print('Error parsing emscripten-releases-tags.txt!') + exit_with_error(str(e)) + + return load_releases_info.cached_info + + +def get_installed_sdk_version(): + version_file = sdk_path(os.path.join('upstream', '.emsdk_version')) + if not os.path.exists(version_file): + return None + with open(version_file) as f: + version = f.read() + return version.split('-')[2] # Get a list of tags for emscripten-releases. def load_releases_tags(): + tags = [] + tags_fastcomp = [] info = load_releases_info() - tags = list(info['releases'].values()) - # Add the tip-of-tree, if it exists. - if os.path.exists(tot_path()): - tot = find_tot() - if tot: - tags.append(tot) - return tags + + for version, sha in sorted(info['releases'].items(), key=lambda x: version_key(x[0])): + tags.append(sha) + # Only include versions older than 1.39.0 in fastcomp releases + if version_key(version) < (2, 0, 0): + tags_fastcomp.append(sha) + + if extra_release_tag: + tags.append(extra_release_tag) + + # Explicitly add the currently installed SDK version. This could be a custom + # version (installed explicitly) so it might not be part of the main list loaded above. + installed = get_installed_sdk_version() + if installed and installed not in tags: + tags.append(installed) + + return tags, tags_fastcomp def load_releases_versions(): @@ -2235,7 +2341,6 @@ def is_string(s): def load_sdk_manifest(): - global tools, sdks try: manifest = json.loads(open(sdk_path("emsdk_manifest.json"), "r").read()) except Exception as e: @@ -2244,16 +2349,17 @@ def load_sdk_manifest(): return emscripten_tags = load_legacy_emscripten_tags() - llvm_precompiled_tags_32bit = list(reversed(load_llvm_precompiled_tags_32bit())) - llvm_precompiled_tags_64bit = list(reversed(load_llvm_precompiled_tags_64bit())) + llvm_precompiled_tags_32bit = [] + llvm_precompiled_tags_64bit = load_file_index_list('llvm-tags-64bit.txt') llvm_precompiled_tags = llvm_precompiled_tags_32bit + llvm_precompiled_tags_64bit binaryen_tags = load_legacy_binaryen_tags() - releases_tags = load_releases_tags() + releases_tags, releases_tags_fastcomp = load_releases_tags() def dependencies_exist(sdk): for tool_name in sdk.uses: tool = find_tool(tool_name) if not tool: + debug_print('missing dependency: ' + tool_name) return False return True @@ -2331,6 +2437,8 @@ def expand_category_param(param, category_list, t, is_sdk): expand_category_param('%precompiled_tag64%', llvm_precompiled_tags_64bit, t, is_sdk=False) elif '%binaryen_tag%' in t.version: expand_category_param('%binaryen_tag%', binaryen_tags, t, is_sdk=False) + elif '%releases-tag%' in t.version and 'fastcomp' in t.version: + expand_category_param('%releases-tag%', releases_tags_fastcomp, t, is_sdk=False) elif '%releases-tag%' in t.version: expand_category_param('%releases-tag%', releases_tags, t, is_sdk=False) else: @@ -2351,6 +2459,8 @@ def expand_category_param(param, category_list, t, is_sdk): expand_category_param('%precompiled_tag32%', llvm_precompiled_tags_32bit, sdk, is_sdk=True) elif '%precompiled_tag64%' in sdk.version: expand_category_param('%precompiled_tag64%', llvm_precompiled_tags_64bit, sdk, is_sdk=True) + elif '%releases-tag%' in sdk.version and 'fastcomp' in sdk.version: + expand_category_param('%releases-tag%', releases_tags_fastcomp, sdk, is_sdk=True) elif '%releases-tag%' in sdk.version: expand_category_param('%releases-tag%', releases_tags, sdk, is_sdk=True) else: @@ -2370,7 +2480,7 @@ def remove_nonexisting_tools(tool_list, log_errors=True): tool = tool_list[i] if not tool.is_installed(): if log_errors: - print("Warning: The SDK/tool '" + str(tool) + "' cannot be activated since it is not installed! Skipping this tool...") + errlog("Warning: The SDK/tool '" + str(tool) + "' cannot be activated since it is not installed! Skipping this tool...") tool_list.pop(i) continue i += 1 @@ -2405,97 +2515,38 @@ def process_tool_list(tools_to_activate, log_errors=True): return tools_to_activate -def run_emcc(tools_to_activate): - for tool in tools_to_activate: - activated_path = getattr(tool, 'activated_path', None) - if activated_path and activated_path.endswith('/emscripten'): - activated_path = to_native_path(tool.expand_vars(tool.activated_path)) - emcc_path = os.path.join(activated_path, 'emcc.py') - if os.path.exists(emcc_path): - debug_print('Calling emcc to initialize it') - subprocess.call([sys.executable, emcc_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return - - -# Copy over any emscripten cache contents that were pregenerated. This avoids -# the user needing to immediately build libc etc. on first run. -# This only applies to legacy SDK versions. Anything built after -# https://github.com/WebAssembly/waterfall/pull/644 already has the libraries -# in the correct location. -# TODO(sbc): Remove this code. -def copy_pregenerated_cache(tools_to_activate): - em_cache_dir = None - - # First look through all the tools to find the EMSCRIPTEN_ROOT - for tool in tools_to_activate: - config = tool.activated_config() - if 'EMSCRIPTEN_ROOT' in config: - em_cache_dir = os.path.join(config['EMSCRIPTEN_ROOT'], 'cache') - break - else: - debug_print('Not copying pregenerated libaries (no EMSCRIPTEN_ROOT found)') - return - - # If we found an EMSCRIPTEN_ROOT look for any tools that include - # "pregenerated_cache" and copy those items into the cache. - for tool in tools_to_activate: - pregenerated_cache = getattr(tool, 'pregenerated_cache', None) - if not pregenerated_cache: - continue - for cache_dir in pregenerated_cache: - # Finish the install of an emscripten-releases build. - install_path = to_native_path(sdk_path(tool.expand_vars(tool.install_path))) - in_cache = os.path.join(install_path, 'lib', cache_dir) - if not os.path.exists(in_cache): - continue - out_cache = os.path.join(em_cache_dir, cache_dir) - if not os.path.exists(out_cache): - os.makedirs(out_cache) - for filename in os.listdir(in_cache): - debug_print('Copying %s to cache: %s' % (filename, out_cache)) - shutil.copy2(os.path.join(in_cache, filename), - os.path.join(out_cache, filename)) +def write_set_env_script(env_string): + assert(WINDOWS) + open(EMSDK_SET_ENV, 'w').write(env_string) # Reconfigure .emscripten to choose the currently activated toolset, set PATH # and other environment variables. # Returns the full list of deduced tools that are now active. -def set_active_tools(tools_to_activate, permanently_activate): +def set_active_tools(tools_to_activate, permanently_activate, system): tools_to_activate = process_tool_list(tools_to_activate, log_errors=True) - generate_dot_emscripten(tools_to_activate) - - # Generating .emscripten will cause emcc to clear the cache on first run (emcc - # sees that the file has changed, since we write it here in the emsdk, and it - # never saw it before; so it clears the cache as it assumes a new config file - # means system libraries may need rebuilding). To avoid emcc's clearing wiping - # out the pregenerated cache contents we want to copy in, run emcc here, then - # copy the cache contents. - run_emcc(tools_to_activate) + if tools_to_activate: + tools = [x for x in tools_to_activate if not x.is_sdk] + print('Setting the following tools as active:\n ' + '\n '.join(map(lambda x: str(x), tools))) + print('') - copy_pregenerated_cache(tools_to_activate) + generate_dot_emscripten(tools_to_activate) # Construct a .bat script that will be invoked to set env. vars and PATH + # We only do this on windows since emsdk.bat is able to modify the + # calling shell environment. On other platform `source emsdk_env.sh` is + # required. if WINDOWS: - env_string = construct_env(tools_to_activate, False) - open(EMSDK_SET_ENV, 'w').write(env_string) - - # Apply environment variables to global all users section. - if WINDOWS and permanently_activate: - # Individual env. vars - for tool in tools_to_activate: - tool.win_activate_env_vars(permanently_activate=True) - - # PATH variable - newpath, added_items = adjusted_path(tools_to_activate, system_path_only=True) - # Are there any actual changes? - if newpath != os.environ['PATH']: - win_set_environment_variable('PATH', newpath, system=True) - - if len(tools_to_activate) > 0: - tools = [x for x in tools_to_activate if not x.is_sdk] - print('\nSet the following tools as active:\n ' + '\n '.join(map(lambda x: str(x), tools))) - print('') + # always set local environment variables since permanently activating will only set the registry settings and + # will not affect the current session + env_vars_to_add = get_env_vars_to_add(tools_to_activate, system, user=permanently_activate) + env_string = construct_env_with_vars(env_vars_to_add) + write_set_env_script(env_string) + + if permanently_activate: + win_set_environment_variables(env_vars_to_add, system, user=permanently_activate) + return tools_to_activate @@ -2540,28 +2591,34 @@ def to_msys_path(p): # Looks at the current PATH and adds and removes entries so that the PATH reflects # the set of given active tools. -def adjusted_path(tools_to_activate, log_additions=False, system_path_only=False): +def adjusted_path(tools_to_activate, system=False, user=False): # These directories should be added to PATH path_add = get_required_path(tools_to_activate) # These already exist. if WINDOWS and not MSYS: - existing_path = win_get_environment_variable('PATH', system=True) - if not system_path_only: - current_user_path = win_get_environment_variable('PATH', system=False) - if current_user_path: - existing_path += ENVPATH_SEPARATOR + current_user_path - existing_path = existing_path.split(ENVPATH_SEPARATOR) + existing_path = win_get_environment_variable('PATH', system=system, user=user, fallback=True).split(ENVPATH_SEPARATOR) else: existing_path = os.environ['PATH'].split(ENVPATH_SEPARATOR) emsdk_root_path = to_unix_path(emsdk_path()) - existing_emsdk_tools = [i for i in existing_path if to_unix_path(i).startswith(emsdk_root_path)] - new_emsdk_tools = [i for i in path_add if not normalized_contains(existing_emsdk_tools, i)] + existing_emsdk_tools = [] + existing_nonemsdk_path = [] + for entry in existing_path: + if to_unix_path(entry).startswith(emsdk_root_path): + existing_emsdk_tools.append(entry) + else: + existing_nonemsdk_path.append(entry) + + new_emsdk_tools = [] + kept_emsdk_tools = [] + for entry in path_add: + if not normalized_contains(existing_emsdk_tools, entry): + new_emsdk_tools.append(entry) + else: + kept_emsdk_tools.append(entry) + + whole_path = unique_items(new_emsdk_tools + kept_emsdk_tools + existing_nonemsdk_path) - # Existing non-emsdk tools - existing_path = [i for i in existing_path if not to_unix_path(i).startswith(emsdk_root_path)] - new_path = [i for i in path_add if not normalized_contains(existing_path, i)] - whole_path = unique_items(new_path + existing_path) if MSYS: # XXX Hack: If running native Windows Python in MSYS prompt where PATH # entries look like "/c/Windows/System32", os.environ['PATH'] @@ -2574,105 +2631,210 @@ def adjusted_path(tools_to_activate, log_additions=False, system_path_only=False return (separator.join(whole_path), new_emsdk_tools) -def construct_env(tools_to_activate, permanent): - env_string = '' - newpath, added_path = adjusted_path(tools_to_activate) +def get_env_vars_to_add(tools_to_activate, system, user): + env_vars_to_add = [] - # Dont permanently add to PATH, since this will break the whole system if there - # are more than 1024 chars in PATH. - # (SETX truncates to set only 1024 chars) - # if permanent: - # print('SETX PATH "' + newpath + '"') - # else: + newpath, added_path = adjusted_path(tools_to_activate, system, user) # Don't bother setting the path if there are no changes. if os.environ['PATH'] != newpath: - if POWERSHELL: - env_string += '$env:PATH="' + newpath + '"\n' - elif CMD: - env_string += 'SET PATH=' + newpath + '\n' - elif CSH: - env_string += 'setenv PATH "' + newpath + '"\n' - elif BASH: - env_string += 'export PATH="' + newpath + '"\n' - else: - assert False + env_vars_to_add += [('PATH', newpath)] - if len(added_path) > 0: - print('Adding directories to PATH:') + if added_path: + errlog('Adding directories to PATH:') for item in added_path: - print('PATH += ' + item) - print('') - - env_vars_to_add = [] + errlog('PATH += ' + item) + errlog('') # A core variable EMSDK points to the root of Emscripten SDK directory. env_vars_to_add += [('EMSDK', to_unix_path(emsdk_path()))] - - em_config_path = os.path.normpath(dot_emscripten_path()) - if to_unix_path(os.environ.get('EM_CONFIG', '')) != to_unix_path(em_config_path): - env_vars_to_add += [('EM_CONFIG', em_config_path)] + env_vars_to_add += [('EM_CONFIG', os.path.normpath(dot_emscripten_path()))] for tool in tools_to_activate: config = tool.activated_config() if 'EMSCRIPTEN_ROOT' in config: - # For older emscripten versions that don't use this default we export - # EM_CACHE. - em_cache_dir = os.path.join(config['EMSCRIPTEN_ROOT'], 'cache') - env_vars_to_add += [('EM_CACHE', em_cache_dir)] + # For older emscripten versions that don't use an embedded cache by + # default we need to export EM_CACHE. + # + # Sadly, we can't put this in the config file since those older versions + # also didn't read the `CACHE` key from the config file: + # + # History: + # - 'CACHE' config started being honored in 1.39.16 + # https://github.com/emscripten-core/emscripten/pull/11091 + # - Default to embedded cache also started in 1.39.16 + # https://github.com/emscripten-core/emscripten/pull/11126 + # + # Since setting EM_CACHE in the environment effects the entire machine + # we want to avoid this except when installing these older emscripten + # versions that really need it. + version = parse_emscripten_version(config['EMSCRIPTEN_ROOT']) + if version < [1, 39, 16]: + em_cache_dir = os.path.join(config['EMSCRIPTEN_ROOT'], 'cache') + env_vars_to_add += [('EM_CACHE', em_cache_dir)] + envs = tool.activated_environment() for env in envs: key, value = parse_key_value(env) value = to_native_path(tool.expand_vars(value)) - # Don't set env vars which are already set to the correct value. - if key not in os.environ or to_unix_path(os.environ[key]) != to_unix_path(value): - env_vars_to_add += [(key, value)] + env_vars_to_add += [(key, value)] + + return env_vars_to_add + + +def construct_env(tools_to_activate, system, user): + return construct_env_with_vars(get_env_vars_to_add(tools_to_activate, system, user)) + + +def unset_env(key): + if POWERSHELL: + return 'Remove-Item env:%s\n' % key + if CMD: + return 'set %s=\n' % key + if CSH: + return 'unsetenv %s;\n' % key + if BASH: + return 'unset %s;\n' % key + assert False + + +def construct_env_with_vars(env_vars_to_add): + env_string = '' + if env_vars_to_add: + errlog('Setting environment variables:') - if len(env_vars_to_add) > 0: - print('Setting environment variables:') for key, value in env_vars_to_add: + # Don't set env vars which are already set to the correct value. + if key in os.environ and to_unix_path(os.environ[key]) == to_unix_path(value): + continue + errlog(key + ' = ' + value) if POWERSHELL: env_string += '$env:' + key + '="' + value + '"\n' elif CMD: - if permanent: - env_string += 'SETX ' + key + ' "' + value + '"\n' - else: - env_string += 'SET ' + key + '=' + value + '\n' + env_string += 'SET ' + key + '=' + value + '\n' elif CSH: - env_string += 'setenv ' + key + ' "' + value + '"\n' + env_string += 'setenv ' + key + ' "' + value + '";\n' elif BASH: - env_string += 'export ' + key + '="' + value + '"\n' + env_string += 'export ' + key + '="' + value + '";\n' else: assert False - print(key + ' = ' + value) - print('') - return env_string + if 'EMSDK_PYTHON' in env_vars_to_add: + # When using our bundled python we never want the user's + # PYTHONHOME or PYTHONPATH + # See https://github.com/emscripten-core/emsdk/issues/598 + env_string += unset_env('PYTHONHOME') + env_string += unset_env('PYTHONPATH') + + # Remove any environment variables that might have been set by old or + # inactive tools/sdks. For example, we set EM_CACHE for older versions + # of the SDK but we want to remove that from the current environment + # if no such tool is active. + # Ignore certain keys that are inputs to emsdk itself. + ignore_keys = set(['EMSDK_POWERSHELL', 'EMSDK_CSH', 'EMSDK_CMD', 'EMSDK_BASH', + 'EMSDK_NUM_CORES', 'EMSDK_TTY']) + env_keys_to_add = set(pair[0] for pair in env_vars_to_add) + for key in os.environ: + if key.startswith('EMSDK_') or key.startswith('EM_'): + if key not in env_keys_to_add and key not in ignore_keys: + errlog('Clearing existing environment variable: %s' % key) + env_string += unset_env(key) -def silentremove(filename): - try: - os.remove(filename) - except OSError as e: - if e.errno != errno.ENOENT: - raise + return env_string def error_on_missing_tool(name): if name.endswith('-64bit') and not is_os_64bit(): - print("Error: '%s' is only provided for 64-bit OSes." % name) + errlog("Error: '%s' is only provided for 64-bit OSes." % name) else: - print("Error: No tool or SDK found by name '%s'." % name) + errlog("Error: No tool or SDK found by name '%s'." % name) return 1 -def main(): - global emscripten_config_directory, BUILD_FOR_TESTING, ENABLE_LLVM_ASSERTIONS, TTY_OUTPUT +def exit_with_fastcomp_error(): + exit_with_error('The fastcomp backend is not getting new builds or releases. Please use the upstream llvm backend or use an older version than 2.0.0 (such as 1.40.1).') + + +def expand_sdk_name(name, activating): + if 'upstream-master' in name: + errlog('upstream-master SDK has been renamed upstream-main') + name = name.replace('upstream-master', 'upstream-main') + if name in ('latest-fastcomp', 'latest-releases-fastcomp', 'tot-fastcomp', 'sdk-nightly-latest'): + exit_with_fastcomp_error() + if name in ('latest', 'sdk-latest', 'latest-64bit', 'sdk-latest-64bit'): + # This is effectly the default SDK + return str(find_latest_releases_sdk('upstream')) + elif name in ('latest-upstream', 'latest-clang-upstream', 'latest-releases-upstream'): + return str(find_latest_releases_sdk('upstream')) + elif name in ('tot', 'sdk-tot', 'tot-upstream'): + if activating: + # When we are activating a tot release, assume that the currently + # installed SDK, if any, is the tot release we want to activate. + # Without this `install tot && activate tot` will race with the builders + # that are producing new builds. + installed = get_installed_sdk_version() + if installed: + debug_print('activating currently installed SDK; not updating tot version') + return 'sdk-releases-upstream-%s-64bit' % installed + return str(find_tot_sdk()) + else: + # check if it's a release handled by an emscripten-releases version, + # and if so use that by using the right hash. we support a few notations, + # x.y.z[-(upstream|fastcomp_]) + # sdk-x.y.z[-(upstream|fastcomp_])-64bit + # TODO: support short notation for old builds too? + backend = None + fullname = name + if '-upstream' in fullname: + fullname = name.replace('-upstream', '') + backend = 'upstream' + elif '-fastcomp' in fullname: + fullname = fullname.replace('-fastcomp', '') + backend = 'fastcomp' + version = fullname.replace('sdk-', '').replace('releases-', '').replace('-64bit', '').replace('tag-', '') + releases_info = load_releases_info()['releases'] + release_hash = get_release_hash(version, releases_info) + if release_hash: + # Known release hash + if backend == 'fastcomp' and version_key(version) >= (2, 0, 0): + exit_with_fastcomp_error() + if backend is None: + if version_key(version) >= (1, 39, 0): + backend = 'upstream' + else: + backend = 'fastcomp' + return 'sdk-releases-%s-%s-64bit' % (backend, release_hash) + elif len(version) == 40: + if backend is None: + backend = 'upstream' + global extra_release_tag + extra_release_tag = version + return 'sdk-releases-%s-%s-64bit' % (backend, version) + return name + + +def main(args): - if len(sys.argv) <= 1 or sys.argv[1] == 'help' or sys.argv[1] == '--help': - if len(sys.argv) <= 1: - print(' emsdk: No command given. Please call one of the following:') - else: - print(' emsdk: Available commands:') + if is_emsdk_sourced_from_github(): + # This code only exists on the master branch + errlog('****') + errlog('Error: You appear to be using the `master` branch of emsdk.') + errlog('We recently made the switch to using `main`') + errlog('In order to continue to receive updates you will need to make the switch locally too.') + errlog('For normal clones without any local branches simply running the following command should be enough:') + errlog(' `git checkout main`') + errlog('For more information see https://github.com/emscripten-core/emsdk/issues/805') + errlog('****') + return 1 + + if not args: + errlog("Missing command; Type 'emsdk help' to get a list of commands.") + return 1 + + cmd = args.pop(0) + + if cmd in ('help', '--help', '-h'): + print(' emsdk: Available commands:') print(''' emsdk list [--old] [--uses] - Lists all available SDKs and tools and their @@ -2682,15 +2844,10 @@ def main(): composition of different SDK packages and dependencies. - emsdk update - Updates emsdk to the newest version, and also - runs 'update-tags' (below). If you have + emsdk update - Updates emsdk to the newest version. If you have bootstrapped emsdk via cloning directly from GitHub, call "git pull" instead to update emsdk. - emsdk update-tags - Fetches the most up to date list of available - Emscripten tagged and other releases from the - servers. - emsdk install [options] ... - Downloads and installs given tools or SDKs. Options can contain: @@ -2702,7 +2859,7 @@ def main(): --build=: Controls what kind of build of LLVM to perform. Pass either 'Debug', 'Release', 'MinSizeRel' or 'RelWithDebInfo'. Default: - 'RelWithDebInfo'. + 'Release'. --generator=: Specifies the CMake Generator to be used during the build. Possible values are the @@ -2747,38 +2904,38 @@ def main(): in the environment where the build is invoked. See README.md for details. + --override-repository: Specifies the git URL to use for a given Tool. E.g. + --override-repository emscripten-main@https://github.com//emscripten/tree/ + + emsdk uninstall - Removes the given tool or SDK from disk.''') if WINDOWS: print(''' - emsdk activate [--global] [--embedded] [--build=type] [--vs2017/--vs2019] + emsdk activate [--permanent] [--system] [--build=type] [--vs2017/--vs2019] - Activates the given tool or SDK in the - environment of the current shell. If the - --global option is passed, the registration - is done globally to all users in the system - environment. If the --embedded option is - passed, all Emcripten configuration files as - well as the temp, cache and ports directories - are located inside the Emscripten SDK - directory rather than the user home - directory. If a custom compiler version was - used to override the compiler to use, pass - the same --vs2017/--vs2019 parameter + environment of the current shell. + + - If the `--permanent` option is passed, then the environment + variables are set permanently for the current user. + + - If the `--system` option is passed, the registration + is done for all users of the system. + This needs admin privileges + (uses Machine environment variables). + + - If a custom compiler version was used to override + the compiler to use, pass the same --vs2017/--vs2019 parameter here to choose which version to activate. emcmdprompt.bat - Spawns a new command prompt window with the Emscripten environment active.''') else: - print(''' emsdk activate [--embedded] [--build=type] + print(''' emsdk activate [--build=type] - Activates the given tool or SDK in the - environment of the current shell. If the - --embedded option is passed, all Emcripten - configuration files as well as the temp, cache - and ports directories are located inside the - Emscripten SDK directory rather than the user - home directory.''') + environment of the current shell.''') print(''' Both commands 'install' and 'activate' accept an optional parameter @@ -2789,44 +2946,78 @@ def main(): 'activate' commands and the invocation of 'emsdk_env', or otherwise these commands will default to operating on the default build type which in and RelWithDebInfo.''') - return 1 + return 0 - # Extracts a boolean command line argument from sys.argv and returns True if it was present + # Extracts a boolean command line argument from args and returns True if it was present def extract_bool_arg(name): - old_argv = sys.argv - sys.argv = list(filter(lambda a: a != name, sys.argv)) - return len(sys.argv) != len(old_argv) + if name in args: + args.remove(name) + return True + return False + + def extract_string_arg(name): + for i in range(len(args)): + if args[i] == name: + value = args[i + 1] + del args[i:i + 2] + return value arg_old = extract_bool_arg('--old') arg_uses = extract_bool_arg('--uses') + arg_permanent = extract_bool_arg('--permanent') arg_global = extract_bool_arg('--global') - arg_embedded = extract_bool_arg('--embedded') + arg_system = extract_bool_arg('--system') + if arg_global: + print('--global is deprecated. Use `--system` to set the environment variables for all users') + arg_system = True + if arg_system: + arg_permanent = True + if extract_bool_arg('--embedded'): + errlog('embedded mode is now the only mode available') + if extract_bool_arg('--no-embedded'): + errlog('embedded mode is now the only mode available') + return 1 + arg_notty = extract_bool_arg('--notty') if arg_notty: + global TTY_OUTPUT TTY_OUTPUT = False - cmd = sys.argv[1] - - # On first run when tag list is not present, populate it to bootstrap. - if (cmd == 'install' or cmd == 'list') and not os.path.isfile(sdk_path('llvm-tags-64bit.txt')): - fetch_emscripten_tags() + # Replace meta-packages with the real package names. + if cmd in ('update', 'install', 'activate'): + activating = cmd == 'activate' + args = [expand_sdk_name(a, activating=activating) for a in args] load_dot_emscripten() load_sdk_manifest() + # Apply any overrides to git branch names to clone from. + forked_url = extract_string_arg('--override-repository') + while forked_url: + tool_name, url_and_refspec = forked_url.split('@') + t = find_tool(tool_name) + if not t: + errlog('Failed to find tool ' + tool_name + '!') + return False + else: + t.url, t.git_branch = parse_github_url_and_refspec(url_and_refspec) + debug_print('Reading git repository URL "' + t.url + '" and git branch "' + t.git_branch + '" for Tool "' + tool_name + '".') + + forked_url = extract_string_arg('--override-repository') + # Process global args - for i in range(2, len(sys.argv)): - if sys.argv[i].startswith('--generator='): - build_generator = re.match(r'''^--generator=['"]?([^'"]+)['"]?$''', sys.argv[i]) + for i in range(len(args)): + if args[i].startswith('--generator='): + build_generator = re.match(r'''^--generator=['"]?([^'"]+)['"]?$''', args[i]) if build_generator: global CMAKE_GENERATOR CMAKE_GENERATOR = build_generator.group(1) - sys.argv[i] = '' + args[i] = '' else: - print("Cannot parse CMake generator string: " + sys.argv[i] + ". Try wrapping generator string with quotes", file=sys.stderr) + errlog("Cannot parse CMake generator string: " + args[i] + ". Try wrapping generator string with quotes") return 1 - elif sys.argv[i].startswith('--build='): - build_type = re.match(r'^--build=(.+)$', sys.argv[i]) + elif args[i].startswith('--build='): + build_type = re.match(r'^--build=(.+)$', args[i]) if build_type: global CMAKE_BUILD_TYPE_OVERRIDE build_type = build_type.group(1) @@ -2834,61 +3025,23 @@ def extract_bool_arg(name): try: build_type_index = [x.lower() for x in build_types].index(build_type.lower()) CMAKE_BUILD_TYPE_OVERRIDE = build_types[build_type_index] - sys.argv[i] = '' + args[i] = '' except: - print('Unknown CMake build type "' + build_type + '" specified! Please specify one of ' + str(build_types), file=sys.stderr) + errlog('Unknown CMake build type "' + build_type + '" specified! Please specify one of ' + str(build_types)) return 1 else: - print("Invalid command line parameter " + sys.argv[i] + ' specified!', file=sys.stderr) + errlog("Invalid command line parameter " + args[i] + ' specified!') return 1 - sys.argv = [x for x in sys.argv if not len(x) == 0] - - releases_info = load_releases_info()['releases'] - - # Replace meta-packages with the real package names. - if cmd in ('update', 'install', 'activate'): - for i in range(2, len(sys.argv)): - arg = sys.argv[i] - if arg in ('latest', 'sdk-latest', 'latest-64bit', 'sdk-latest-64bit'): - # This is effectly the default SDK - sys.argv[i] = str(find_latest_releases_sdk('upstream')) - elif arg in ('latest-fastcomp', 'latest-releases-fastcomp'): - sys.argv[i] = str(find_latest_releases_sdk('fastcomp')) - elif arg in ('latest-upstream', 'latest-clang-upstream', 'latest-releases-upstream'): - sys.argv[i] = str(find_latest_releases_sdk('upstream')) - elif arg in ('tot', 'sdk-tot'): - sys.argv[i] = str(find_tot_sdk('upstream')) - elif arg == 'tot-upstream': - sys.argv[i] = str(find_tot_sdk('upstream')) - elif arg in ('tot-fastcomp', 'sdk-nightly-latest'): - sys.argv[i] = str(find_tot_sdk('fastcomp')) - else: - # check if it's a release handled by an emscripten-releases version, - # and if so use that by using the right hash. we support a few notations, - # x.y.z[-(upstream|fastcomp_]) - # sdk-x.y.z[-(upstream|fastcomp_])-64bit - # TODO: support short notation for old builds too? - backend = None - if '-upstream' in arg: - arg = arg.replace('-upstream', '') - backend = 'upstream' - elif '-fastcomp' in arg: - arg = arg.replace('-fastcomp', '') - backend = 'fastcomp' - arg = arg.replace('sdk-', '').replace('-64bit', '').replace('tag-', '') - release_hash = releases_info.get(arg, None) or releases_info.get('sdk-' + arg + '-64bit') - if release_hash: - if backend is None: - if version_key(arg) >= (1, 39, 0): - backend = 'upstream' - else: - backend = 'fastcomp' - sys.argv[i] = 'sdk-releases-%s-%s-64bit' % (backend, release_hash) + args = [x for x in args if x] if cmd == 'list': print('') - if (LINUX or OSX or WINDOWS) and (ARCH == 'x86' or ARCH == 'x86_64'): + def installed_sdk_text(name): + sdk = find_sdk(name) + return 'INSTALLED' if sdk and sdk.is_installed() else '' + + if (LINUX or MACOS or WINDOWS) and (ARCH == 'x86' or ARCH == 'x86_64'): print('The *recommended* precompiled SDK download is %s (%s).' % (find_latest_releases_version(), find_latest_releases_hash())) print() print('To install/activate it, use one of:') @@ -2896,8 +3049,8 @@ def extract_bool_arg(name): print(' latest-fastcomp [legacy (fastcomp) backend]') print('') print('Those are equivalent to installing/activating the following:') - print(' %s' % find_latest_releases_version()) - print(' %s-fastcomp' % find_latest_releases_version()) + print(' %s %s' % (find_latest_releases_version(), installed_sdk_text(find_latest_releases_sdk('upstream')))) + print(' %s-fastcomp %s' % (find_latest_releases_version(), installed_sdk_text(find_latest_releases_sdk('fastcomp')))) print('') else: print('Warning: your platform does not have precompiled SDKs available.') @@ -2905,17 +3058,21 @@ def extract_bool_arg(name): print('') print('All recent (non-legacy) installable versions are:') - releases_versions = sorted(load_releases_versions()) - releases_versions.reverse() + releases_versions = sorted( + load_releases_versions(), + key=lambda x: [int(v) if v.isdigit() else -1 for v in x.split('.')], + reverse=True, + ) + releases_info = load_releases_info()['releases'] for ver in releases_versions: - print(' %s' % ver) + print(' %s %s' % (ver, installed_sdk_text('sdk-releases-upstream-%s-64bit' % get_release_hash(ver, releases_info)))) print() # Use array to work around the lack of being able to mutate from enclosing # function. has_partially_active_tools = [False] - if len(sdks) > 0: + if sdks: def find_sdks(needs_compilation): s = [] for sdk in sdks: @@ -2940,7 +3097,7 @@ def print_sdks(s): print('The following SDKs can be compiled from source:') print_sdks(find_sdks(True)) - if len(tools) > 0: + if tools: def find_tools(needs_compilation): t = [] for tool in tools: @@ -2977,7 +3134,7 @@ def print_tools(t): print_tools(find_tools(needs_compilation=True)) else: if is_emsdk_sourced_from_github(): - print("There are no tools available. Run 'git pull' followed by 'emsdk update-tags' to fetch the latest set of tools.") + print("There are no tools available. Run 'git pull' to fetch the latest set of tools.") else: print("There are no tools available. Run 'emsdk update' to fetch the latest set of tools.") print('') @@ -2985,61 +3142,46 @@ def print_tools(t): print('Items marked with * are activated for the current user.') if has_partially_active_tools[0]: env_cmd = 'emsdk_env.bat' if WINDOWS else 'source ./emsdk_env.sh' - print('Items marked with (*) are selected for use, but your current shell environment is not configured to use them. Type "' + env_cmd + '" to set up your current shell to use them' + (', or call "emsdk activate --global " to permanently activate them.' if WINDOWS else '.')) + print('Items marked with (*) are selected for use, but your current shell environment is not configured to use them. Type "' + env_cmd + '" to set up your current shell to use them' + (', or call "emsdk activate --permanent " to permanently activate them.' if WINDOWS else '.')) if not arg_old: print('') print("To access the historical archived versions, type 'emsdk list --old'") print('') if is_emsdk_sourced_from_github(): - print('Run "git pull" followed by "./emsdk update-tags" to pull in the latest list.') + print('Run "git pull" to pull in the latest list.') else: print('Run "./emsdk update" to pull in the latest list.') return 0 elif cmd == 'construct_env': - if len(sys.argv) == 2: - outfile = EMSDK_SET_ENV - # Clean up old temp file up front, in case of failure later before we get - # to write out the new one. - silentremove(EMSDK_SET_ENV) - else: - outfile = sys.argv[2] + # Clean up old temp file up front, in case of failure later before we get + # to write out the new one. tools_to_activate = currently_active_tools() tools_to_activate = process_tool_list(tools_to_activate, log_errors=True) - env_string = construct_env(tools_to_activate, len(sys.argv) >= 3 and 'perm' in sys.argv[2]) - open(outfile, 'w').write(env_string) - if UNIX: - os.chmod(outfile, 0o755) + env_string = construct_env(tools_to_activate, arg_system, arg_permanent) + if WINDOWS and not BASH: + write_set_env_script(env_string) + else: + sys.stdout.write(env_string) return 0 elif cmd == 'update': update_emsdk() - # Clean up litter after old emsdk update which may have left this temp file - # around. - silentremove(sdk_path(EMSDK_SET_ENV)) + if WINDOWS: + # Clean up litter after old emsdk update which may have left this temp + # file around. + rmfile(sdk_path(EMSDK_SET_ENV)) return 0 elif cmd == 'update-tags': - fetch_emscripten_tags() + errlog('`update-tags` is not longer needed. To install the latest tot release just run `install tot`') return 0 elif cmd == 'activate': - if arg_global: - print('Registering active Emscripten environment globally for all users.') + if arg_permanent: + print('Registering active Emscripten environment permanently') print('') - if arg_embedded: - # Activating the emsdk tools locally relative to Emscripten SDK directory. - emscripten_config_directory = emsdk_path() - print('Writing .emscripten configuration file to Emscripten SDK directory ' + emscripten_config_directory) - else: - print('Writing .emscripten configuration file to user home directory ' + emscripten_config_directory) - # Remove .emscripten from emsdk dir, since its presence is used to detect - # whether emsdk is activate in embedded mode or not. - try: - os.remove(os.path.join(emsdk_path(), ".emscripten")) - except: - pass tools_to_activate = currently_active_tools() - args = [x for x in sys.argv[2:] if not x.startswith('--')] + args = [x for x in args if not x.startswith('--')] for arg in args: tool = find_tool(arg) if tool is None: @@ -3047,69 +3189,68 @@ def print_tools(t): if tool is None: return error_on_missing_tool(arg) tools_to_activate += [tool] - if len(tools_to_activate) == 0: - print('No tools/SDKs specified to activate! Usage:\n emsdk activate tool/sdk1 [tool/sdk2] [...]') + if not tools_to_activate: + errlog('No tools/SDKs specified to activate! Usage:\n emsdk activate tool/sdk1 [tool/sdk2] [...]') return 1 - tools_to_activate = set_active_tools(tools_to_activate, permanently_activate=arg_global) - if len(tools_to_activate) == 0: - print('No tools/SDKs found to activate! Usage:\n emsdk activate tool/sdk1 [tool/sdk2] [...]') + active_tools = set_active_tools(tools_to_activate, permanently_activate=arg_permanent, system=arg_system) + if not active_tools: + errlog('No tools/SDKs found to activate! Usage:\n emsdk activate tool/sdk1 [tool/sdk2] [...]') return 1 - if WINDOWS and not arg_global: - print('The changes made to environment variables only apply to the currently running shell instance. Use the \'emsdk_env.bat\' to re-enter this environment later, or if you\'d like to permanently register this environment globally to all users in Windows Registry, rerun this command with the option --global.') + if WINDOWS and not arg_permanent: + errlog('The changes made to environment variables only apply to the currently running shell instance. Use the \'emsdk_env.bat\' to re-enter this environment later, or if you\'d like to permanently register this environment permanently, rerun this command with the option --permanent.') return 0 elif cmd == 'install': + global BUILD_FOR_TESTING, ENABLE_LLVM_ASSERTIONS, CPU_CORES, GIT_CLONE_SHALLOW + # Process args - for i in range(2, len(sys.argv)): - if sys.argv[i].startswith('-j'): - multicore = re.match(r'^-j(\d+)$', sys.argv[i]) + for i in range(len(args)): + if args[i].startswith('-j'): + multicore = re.match(r'^-j(\d+)$', args[i]) if multicore: - global CPU_CORES CPU_CORES = int(multicore.group(1)) - sys.argv[i] = '' + args[i] = '' else: - print("Invalid command line parameter " + sys.argv[i] + ' specified!', file=sys.stderr) + errlog("Invalid command line parameter " + args[i] + ' specified!') return 1 - elif sys.argv[i] == '--shallow': - global GIT_CLONE_SHALLOW + elif args[i] == '--shallow': GIT_CLONE_SHALLOW = True - sys.argv[i] = '' - elif sys.argv[i] == '--build-tests': + args[i] = '' + elif args[i] == '--build-tests': BUILD_FOR_TESTING = True - sys.argv[i] = '' - elif sys.argv[i] == '--enable-assertions': + args[i] = '' + elif args[i] == '--enable-assertions': ENABLE_LLVM_ASSERTIONS = 'ON' - sys.argv[i] = '' - elif sys.argv[i] == '--disable-assertions': + args[i] = '' + elif args[i] == '--disable-assertions': ENABLE_LLVM_ASSERTIONS = 'OFF' - sys.argv[i] = '' - sys.argv = [x for x in sys.argv if not len(x) == 0] - if len(sys.argv) <= 2: - print("Missing parameter. Type 'emsdk install ' to install a tool or an SDK. Type 'emsdk list' to obtain a list of available tools. Type 'emsdk install latest' to automatically install the newest version of the SDK.") + args[i] = '' + args = [x for x in args if x] + if not args: + errlog("Missing parameter. Type 'emsdk install ' to install a tool or an SDK. Type 'emsdk list' to obtain a list of available tools. Type 'emsdk install latest' to automatically install the newest version of the SDK.") return 1 - for t in sys.argv[2:]: + + for t in args: tool = find_tool(t) if tool is None: tool = find_sdk(t) if tool is None: return error_on_missing_tool(t) - success = tool.install() - if not success: - return 1 + tool.install() return 0 elif cmd == 'uninstall': - if len(sys.argv) <= 2: - print("Syntax error. Call 'emsdk uninstall '. Call 'emsdk list' to obtain a list of available tools.") + if not args: + errlog("Syntax error. Call 'emsdk uninstall '. Call 'emsdk list' to obtain a list of available tools.") return 1 - tool = find_tool(sys.argv[2]) + tool = find_tool(args[0]) if tool is None: - print("Error: Tool by name '" + sys.argv[2] + "' was not found.") + errlog("Error: Tool by name '" + args[0] + "' was not found.") return 1 tool.uninstall() return 0 - print("Unknown command '" + cmd + "' given! Type 'emsdk help' to get a list of commands.") + errlog("Unknown command '" + cmd + "' given! Type 'emsdk help' to get a list of commands.") return 1 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main(sys.argv[1:])) diff --git a/emsdk_env.bat b/emsdk_env.bat index 1f8d1650bb..c793321c98 100644 --- a/emsdk_env.bat +++ b/emsdk_env.bat @@ -1 +1 @@ -@call "%~dp0emsdk" construct_env %* +@call "%~dp0emsdk" construct_env diff --git a/emsdk_env.csh b/emsdk_env.csh old mode 100755 new mode 100644 index 54cdf0e902..187d4c5438 --- a/emsdk_env.csh +++ b/emsdk_env.csh @@ -5,8 +5,6 @@ # because it won't have any effect then. # That is, always run this script with # -# . ./emsdk_env.csh -# or # source ./emsdk_env.csh # # instead of just plainly running with @@ -18,19 +16,15 @@ set SRC=($_) if ("$SRC" == "") then set SRC="$0" else - set SRC="$SRC[2]" + set SRC="$SRC[1]" endif set CURDIR=`pwd` -cd `dirname "$SRC"` +setenv DIR `dirname "$SRC"` unset SRC setenv EMSDK_CSH 1 -set tmpfile=`mktemp` || exit 1 -./emsdk construct_env $tmpfile -source $tmpfile -rm -f $tmpfile +eval `$DIR/emsdk construct_env` +unsetenv DIR unsetenv EMSDK_CSH - -cd "$CURDIR" diff --git a/emsdk_env.fish b/emsdk_env.fish old mode 100755 new mode 100644 index 99119b73eb..29f6f4647b --- a/emsdk_env.fish +++ b/emsdk_env.fish @@ -6,12 +6,7 @@ set -l script (status -f) set -l dir (dirname $script) -pushd $dir > /dev/null - -./emsdk construct_env -. ./emsdk_set_env.sh +eval ($dir/emsdk construct_env) set -e -l script set -e -l dir - -popd > /dev/null diff --git a/emsdk_env.ps1 b/emsdk_env.ps1 index fc7f2c7692..ab5fc4dc45 100644 --- a/emsdk_env.ps1 +++ b/emsdk_env.ps1 @@ -1,2 +1,2 @@ $ScriptDirectory = Split-Path -parent $PSCommandPath -& "$ScriptDirectory/emsdk.ps1" construct_env $args +& "$ScriptDirectory/emsdk.ps1" construct_env diff --git a/emsdk_env.sh b/emsdk_env.sh old mode 100755 new mode 100644 index 1e972a3541..8f76de7522 --- a/emsdk_env.sh +++ b/emsdk_env.sh @@ -1,32 +1,73 @@ -#!/bin/bash # This script is sourced by the user and uses -# their shell. Try not to use bashisms. - +# their shell. +# +# This script tries to find its location but +# this does not work in every shell. +# +# It is known to work in bash, zsh and ksh +# # Do not execute this script without sourcing, # because it won't have any effect then. # That is, always run this script with # -# . ./emsdk_env.sh +# . /path/to/emsdk_env.sh +# # or -# source ./emsdk_env.sh +# +# source /path/to/emsdk_env.sh # # instead of just plainly running with # # ./emsdk_env.sh # # which won't have any effect. -SRC="$BASH_SOURCE" -if [ "$SRC" = "" ]; then - SRC="$0" + +CURRENT_SCRIPT= +DIR="." + +# use shell specific method to get the path +# to the current file being source'd. +# +# To add a shell, add another conditional below, +# then add tests to scripts/test_source_env.sh + +if [ -n "${BASH_SOURCE-}" ]; then + CURRENT_SCRIPT="$BASH_SOURCE" +elif [ -n "${ZSH_VERSION-}" ]; then + CURRENT_SCRIPT="${(%):-%x}" +elif [ -n "${KSH_VERSION-}" ]; then + CURRENT_SCRIPT=${.sh.file} fi -CURDIR="$(pwd)" -cd "$(dirname "$SRC")" -unset SRC -tmpfile=`mktemp` || exit 1 -# Force emsdk to use bash syntax so that this works in windows + bash too -EMSDK_BASH=1 ./emsdk construct_env $tmpfile -. $tmpfile -rm -f $tmpfile +if [ -n "${CURRENT_SCRIPT-}" ]; then + DIR=$(dirname "$CURRENT_SCRIPT") + if [ -h "$CURRENT_SCRIPT" ]; then + # Now work out actual DIR since this is part of a symlink. + # Since we can't be sure that readlink or realpath + # are available, use tools more likely to be installed. + # (This will still fail if sed is not available.) + SYMDIR=$(dirname "$(ls -l "$CURRENT_SCRIPT" | sed -n "s/.*-> //p")") + if [ -z "$SYMDIR" ]; then + SYMDIR="." + fi + FULLDIR="$DIR/$SYMDIR" + DIR=$(cd "$FULLDIR" > /dev/null 2>&1; /bin/pwd) + unset SYMDIR + unset FULLDIR + fi +fi +unset CURRENT_SCRIPT -cd "$CURDIR" +if [ ! -f "$DIR/emsdk.py" ]; then + echo "Error: unable to determine 'emsdk' directory. Perhaps you are using a shell or" 1>&2 + echo " environment that this script does not support." 1>&2 + echo 1>&2 + echo "A possible solution is to source this script while in the 'emsdk' directory." 1>&2 + echo 1>&2 + unset DIR + return +fi + +# Force emsdk to use bash syntax so that this works in windows + bash too +eval `EMSDK_BASH=1 $DIR/emsdk construct_env` +unset DIR diff --git a/emsdk_manifest.json b/emsdk_manifest.json index 2814d12972..21b4508c2c 100644 --- a/emsdk_manifest.json +++ b/emsdk_manifest.json @@ -2,12 +2,12 @@ "tools": [ { "id": "llvm-git", - "version": "master", + "version": "main", "bitness": 32, "install_path": "llvm/git", - "git_branch": "master", + "git_branch": "main", "url": "https://github.com/llvm/llvm-project.git", - "custom_install_script": "build_llvm_monorepo", + "custom_install_script": "build_llvm", "only_supports_wasm": true, "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", @@ -16,12 +16,12 @@ }, { "id": "llvm-git", - "version": "master", + "version": "main", "bitness": 64, "install_path": "llvm/git", - "git_branch": "master", + "git_branch": "main", "url": "https://github.com/llvm/llvm-project.git", - "custom_install_script": "build_llvm_monorepo", + "custom_install_script": "build_llvm", "only_supports_wasm": true, "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", @@ -34,10 +34,10 @@ "version": "tag-e%tag%", "bitness": 32, "append_bitness": false, - "windows_url": "https://github.com/kripken/emscripten-fastcomp/archive/%tag%.zip", - "unix_url": "https://github.com/kripken/emscripten-fastcomp/archive/%tag%.tar.gz", - "windows_clang_url": "https://github.com/kripken/emscripten-fastcomp-clang/archive/%tag%.zip", - "unix_clang_url": "https://github.com/kripken/emscripten-fastcomp-clang/archive/%tag%.tar.gz", + "windows_url": "https://github.com/emscripten-core/emscripten-fastcomp/archive/%tag%.zip", + "unix_url": "https://github.com/emscripten-core/emscripten-fastcomp/archive/%tag%.tar.gz", + "windows_clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang/archive/%tag%.zip", + "unix_clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang/archive/%tag%.tar.gz", "custom_install_script": "build_fastcomp", "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", @@ -49,10 +49,10 @@ "version": "tag-e%tag%", "bitness": 64, "append_bitness": false, - "windows_url": "https://github.com/kripken/emscripten-fastcomp/archive/%tag%.zip", - "unix_url": "https://github.com/kripken/emscripten-fastcomp/archive/%tag%.tar.gz", - "windows_clang_url": "https://github.com/kripken/emscripten-fastcomp-clang/archive/%tag%.zip", - "unix_clang_url": "https://github.com/kripken/emscripten-fastcomp-clang/archive/%tag%.tar.gz", + "windows_url": "https://github.com/emscripten-core/emscripten-fastcomp/archive/%tag%.zip", + "unix_url": "https://github.com/emscripten-core/emscripten-fastcomp/archive/%tag%.tar.gz", + "windows_clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang/archive/%tag%.zip", + "unix_clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang/archive/%tag%.tar.gz", "custom_install_script": "build_fastcomp", "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", @@ -64,8 +64,8 @@ "version": "master", "bitness": 32, "install_path": "clang/fastcomp", - "url": "https://github.com/kripken/emscripten-fastcomp.git", - "clang_url": "https://github.com/kripken/emscripten-fastcomp-clang.git", + "url": "https://github.com/emscripten-core/emscripten-fastcomp.git", + "clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang.git", "git_branch": "master", "custom_install_script": "build_fastcomp", "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", @@ -79,8 +79,8 @@ "bitness": 64, "install_path": "clang/fastcomp", "git_branch": "master", - "url": "https://github.com/kripken/emscripten-fastcomp.git", - "clang_url": "https://github.com/kripken/emscripten-fastcomp-clang.git", + "url": "https://github.com/emscripten-core/emscripten-fastcomp.git", + "clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang.git", "custom_install_script": "build_fastcomp", "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", @@ -94,14 +94,13 @@ "bitness": 64, "arch": "x86_64", "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2", - "osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2", + "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2", "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/win/%releases-tag%/wasm-binaries.zip", "zipfile_prefix": "%releases-tag%-", "install_path": "upstream", "activated_path": "%installation_dir%/emscripten", "activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%';EMSCRIPTEN_ROOT='%installation_dir%/emscripten'", - "emscripten_releases_hash": "%releases-tag%", - "pregenerated_cache": ["asmjs", "wasm", "wasm-obj", "wasm-bc"] + "emscripten_releases_hash": "%releases-tag%" }, { "id": "releases", @@ -109,14 +108,13 @@ "bitness": 64, "arch": "x86_64", "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2", - "osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2", + "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2", "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/win/%releases-tag%/wasm-binaries.zip", "zipfile_prefix": "%releases-tag%-", "install_path": "fastcomp", "activated_path": "%installation_dir%/emscripten", "activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%';EMSCRIPTEN_ROOT='%installation_dir%/emscripten';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%/bin/optimizer%.exe%'", - "emscripten_releases_hash": "%releases-tag%", - "pregenerated_cache": ["asmjs", "wasm", "wasm-obj", "wasm-bc"] + "emscripten_releases_hash": "%releases-tag%" }, { @@ -125,7 +123,7 @@ "bitness": 32, "arch": "x86", "windows_url": "llvm/tag/win_64bit/emscripten-llvm-e%precompiled_tag32%.zip", - "osx_url": "llvm/tag/osx_64bit/emscripten-llvm-e%precompiled_tag32%.tar.gz", + "macos_url": "llvm/tag/osx_64bit/emscripten-llvm-e%precompiled_tag32%.tar.gz", "linux_url": "llvm/tag/linux_64bit/emscripten-llvm-e%precompiled_tag32%.tar.gz", "activated_path": "%installation_dir%", "activated_cfg": "LLVM_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%/optimizer%.exe%';BINARYEN_ROOT='%installation_dir%/binaryen'", @@ -137,7 +135,7 @@ "bitness": 64, "arch": "x86_64", "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/old/win/emscripten-llvm-e%precompiled_tag64%.zip", - "osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/old/mac/emscripten-llvm-e%precompiled_tag64%.tar.gz", + "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/old/mac/emscripten-llvm-e%precompiled_tag64%.tar.gz", "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/old/linux/emscripten-llvm-e%precompiled_tag64%.tar.gz", "activated_path": "%installation_dir%", "activated_cfg": "LLVM_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%/optimizer%.exe%';BINARYEN_ROOT='%installation_dir%/binaryen'", @@ -152,7 +150,8 @@ "linux_url": "node-v8.9.1-linux-x86.tar.xz", "activated_path": "%installation_dir%/bin", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", - "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%", + "is_old": true }, { "id": "node", @@ -162,19 +161,21 @@ "linux_url": "https://nodejs.org/dist/v8.9.1/node-v8.9.1-linux-armv7l.tar.xz", "activated_path": "%installation_dir%/bin", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", - "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%", + "is_old": true }, { "id": "node", "version": "8.9.1", "bitness": 64, "arch": "x86_64", - "osx_url": "node-v8.9.1-darwin-x64.tar.gz", + "macos_url": "node-v8.9.1-darwin-x64.tar.gz", "windows_url": "node-v8.9.1-win-x64.zip", "linux_url": "node-v8.9.1-linux-x64.tar.xz", "activated_path": "%installation_dir%/bin", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", - "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%", + "is_old": true }, { "id": "node", @@ -184,46 +185,48 @@ "linux_url": "node-v8.9.1-linux-arm64.tar.xz", "activated_path": "%installation_dir%/bin", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", - "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%", + "is_old": true }, { "id": "node", - "version": "12.9.1", + "version": "14.15.5", "bitness": 32, "arch": "x86", - "windows_url": "node-v12.9.1-win-x86.zip", + "windows_url": "node-v14.15.5-win-x86.zip", "activated_path": "%installation_dir%/bin", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, { "id": "node", - "version": "12.9.1", + "version": "14.15.5", "arch": "arm", "bitness": 32, - "linux_url": "node-v12.9.1-linux-armv7l.tar.xz", + "linux_url": "node-v14.15.5-linux-armv7l.tar.xz", "activated_path": "%installation_dir%/bin", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, { "id": "node", - "version": "12.9.1", + "version": "14.15.5", "bitness": 64, "arch": "x86_64", - "osx_url": "node-v12.9.1-darwin-x64.tar.gz", - "windows_url": "node-v12.9.1-win-x64.zip", - "linux_url": "node-v12.9.1-linux-x64.tar.xz", + "macos_url": "node-v14.15.5-darwin-x64.tar.gz", + "windows_url": "node-v14.15.5-win-x64.zip", + "linux_url": "node-v14.15.5-linux-x64.tar.xz", "activated_path": "%installation_dir%/bin", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, { "id": "node", - "version": "12.9.1", + "version": "14.15.5", "arch": "aarch64", "bitness": 64, - "linux_url": "node-v12.9.1-linux-arm64.tar.xz", + "macos_url": "node-v14.15.5-darwin-x64.tar.gz", + "linux_url": "node-v14.15.5-linux-arm64.tar.xz", "activated_path": "%installation_dir%/bin", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" @@ -234,9 +237,9 @@ "bitness": 32, "arch": "x86", "windows_url": "WinPython-32bit-2.7.13.1Zero.zip", - "activated_path": "%installation_dir%/python-2.7.13", "activated_cfg": "PYTHON='%installation_dir%/python-2.7.13/python%.exe%'", - "activated_env": "EMSDK_PYTHON=%installation_dir%/python-2.7.13/python%.exe%" + "activated_env": "EMSDK_PYTHON=%installation_dir%/python-2.7.13/python%.exe%", + "is_old": true }, { "id": "python", @@ -244,9 +247,9 @@ "bitness": 64, "arch": "x86_64", "windows_url": "WinPython-64bit-2.7.13.1Zero.zip", - "activated_path": "%installation_dir%/python-2.7.13.amd64", "activated_cfg": "PYTHON='%installation_dir%/python-2.7.13.amd64/python%.exe%'", - "activated_env": "EMSDK_PYTHON=%installation_dir%/python-2.7.13.amd64/python%.exe%" + "activated_env": "EMSDK_PYTHON=%installation_dir%/python-2.7.13.amd64/python%.exe%", + "is_old": true }, { "id": "python", @@ -254,9 +257,9 @@ "bitness": 32, "arch": "x86", "windows_url": "python-3.7.4-embed-win32-patched.zip", - "activated_path": "%installation_dir%", "activated_cfg": "PYTHON='%installation_dir%/python.exe'", - "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe" + "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe", + "is_old": true }, { "id": "python", @@ -264,9 +267,9 @@ "bitness": 64, "arch": "x86_64", "windows_url": "python-3.7.4-embed-amd64-patched.zip", - "activated_path": "%installation_dir%", "activated_cfg": "PYTHON='%installation_dir%/python.exe'", - "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe" + "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe", + "is_old": true }, { "id": "python", @@ -274,9 +277,9 @@ "bitness": 32, "arch": "x86", "windows_url": "python-3.7.4-embed-win32+pywin32.zip", - "activated_path": "%installation_dir%", "activated_cfg": "PYTHON='%installation_dir%/python.exe'", - "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe" + "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe", + "is_old": true }, { "id": "python", @@ -284,17 +287,53 @@ "bitness": 64, "arch": "x86_64", "windows_url": "python-3.7.4-embed-amd64+pywin32.zip", - "activated_path": "%installation_dir%", + "activated_cfg": "PYTHON='%installation_dir%/python.exe'", + "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe", + "is_old": true + }, + { + "id": "python", + "version": "3.7.4-2", + "bitness": 64, + "arch": "x86_64", + "macos_url": "python-3.7.4-2-macos.tar.gz", + "activated_cfg": "PYTHON='%installation_dir%/bin/python3'", + "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3;SSL_CERT_FILE=%installation_dir%/lib/python3.7/site-packages/certifi/cacert.pem", + "is_old": true + }, + { + "id": "python", + "version": "3.9.2-1", + "bitness": 64, + "arch": "x86_64", + "windows_url": "python-3.9.2-1-embed-amd64+pywin32.zip", "activated_cfg": "PYTHON='%installation_dir%/python.exe'", "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe" }, + { + "id": "python", + "version": "3.9.2-1", + "bitness": 64, + "arch": "x86_64", + "macos_url": "python-3.9.2-1-macos-x86_64.tar.gz", + "activated_cfg": "PYTHON='%installation_dir%/bin/python3'", + "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3;SSL_CERT_FILE=%installation_dir%/lib/python3.9/site-packages/certifi/cacert.pem" + }, + { + "id": "python", + "version": "3.9.2-1", + "bitness": 64, + "arch": "aarch64", + "macos_url": "python-3.9.2-1-macos-arm64.tar.gz", + "activated_cfg": "PYTHON='%installation_dir%/bin/python3'", + "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3;SSL_CERT_FILE=%installation_dir%/lib/python3.9/site-packages/certifi/cacert.pem" + }, { "id": "java", "version": "8.152", "bitness": 32, "arch": "x86", "windows_url": "portable_jre_8_update_152_32bit.zip", - "activated_path": "%installation_dir%/bin", "activated_env": "JAVA_HOME=%installation_dir%", "activated_cfg": "JAVA='%installation_dir%/bin/java%.exe%'" }, @@ -304,7 +343,6 @@ "bitness": 64, "arch": "x86_64", "windows_url": "portable_jre_8_update_152_64bit.zip", - "activated_path": "%installation_dir%/bin", "activated_env": "JAVA_HOME=%installation_dir%", "activated_cfg": "JAVA='%installation_dir%/bin/java%.exe%'" }, @@ -313,8 +351,8 @@ "version": "tag-%tag%", "bitness": 32, "append_bitness": false, - "windows_url": "https://github.com/kripken/emscripten/archive/%tag%.zip", - "unix_url": "https://github.com/kripken/emscripten/archive/%tag%.tar.gz", + "windows_url": "https://github.com/emscripten-core/emscripten/archive/%tag%.zip", + "unix_url": "https://github.com/emscripten-core/emscripten/archive/%tag%.tar.gz", "zipfile_prefix": "emscripten-e", "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%%generator_prefix%_32bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%'", "activated_path": "%installation_dir%", @@ -329,8 +367,8 @@ "version": "tag-%tag%", "bitness": 64, "append_bitness": false, - "windows_url": "https://github.com/kripken/emscripten/archive/%tag%.zip", - "unix_url": "https://github.com/kripken/emscripten/archive/%tag%.tar.gz", + "windows_url": "https://github.com/emscripten-core/emscripten/archive/%tag%.zip", + "unix_url": "https://github.com/emscripten-core/emscripten/archive/%tag%.tar.gz", "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%%generator_prefix%_64bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%'", "activated_path": "%installation_dir%", "activated_env": "EMSCRIPTEN=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%%generator_prefix%_64bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%", @@ -342,8 +380,8 @@ { "id": "emscripten", "version": "%precompiled_tag%", - "windows_url": "https://github.com/kripken/emscripten/archive/%precompiled_tag%.zip", - "unix_url": "https://github.com/kripken/emscripten/archive/%precompiled_tag%.tar.gz", + "windows_url": "https://github.com/emscripten-core/emscripten/archive/%precompiled_tag%.zip", + "unix_url": "https://github.com/emscripten-core/emscripten/archive/%precompiled_tag%.tar.gz", "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%'", "activated_path": "%installation_dir%", "activated_env": "EMSCRIPTEN=%installation_dir%" @@ -382,41 +420,37 @@ }, { "id": "emscripten", - "version": "master", + "version": "main", "bitness": 32, "append_bitness": false, "url": "https://github.com/emscripten-core/emscripten.git", - "git_branch": "master", - "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%%generator_prefix%_32bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%'", + "git_branch": "main", + "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%'", "activated_path": "%installation_dir%", - "activated_env": "EMSCRIPTEN=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%%generator_prefix%_32bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%", + "activated_env": "EMSCRIPTEN=%installation_dir%", "cmake_build_type": "Release", - "custom_install_script": "emscripten_post_install", - "custom_is_installed_script": "is_optimizer_installed", - "custom_uninstall_script": "uninstall_optimizer" + "custom_install_script": "emscripten_npm_install" }, { "id": "emscripten", - "version": "master", + "version": "main", "bitness": 64, "append_bitness": false, "url": "https://github.com/emscripten-core/emscripten.git", - "git_branch": "master", - "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%%generator_prefix%_64bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%'", + "git_branch": "main", + "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%'", "activated_path": "%installation_dir%", - "activated_env": "EMSCRIPTEN=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%%generator_prefix%_64bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%", + "activated_env": "EMSCRIPTEN=%installation_dir%", "cmake_build_type": "Release", - "custom_install_script": "emscripten_post_install", - "custom_is_installed_script": "is_optimizer_installed", - "custom_uninstall_script": "uninstall_optimizer" + "custom_install_script": "emscripten_npm_install" }, { "id": "binaryen", - "version": "master", + "version": "main", "bitness": 32, "append_bitness": false, "url": "https://github.com/WebAssembly/binaryen.git", - "git_branch": "master", + "git_branch": "main", "activated_cfg": "BINARYEN_ROOT='%installation_dir%%generator_prefix%_32bit_binaryen'", "activated_path": "%installation_dir%%generator_prefix%_32bit_binaryen/bin", "activated_env": "BINARYEN_ROOT=%installation_dir%%generator_prefix%_32bit_binaryen", @@ -427,11 +461,11 @@ }, { "id": "binaryen", - "version": "master", + "version": "main", "bitness": 64, "append_bitness": false, "url": "https://github.com/WebAssembly/binaryen.git", - "git_branch": "master", + "git_branch": "main", "activated_cfg": "BINARYEN_ROOT='%installation_dir%%generator_prefix%_64bit_binaryen'", "activated_path": "%installation_dir%%generator_prefix%_64bit_binaryen/bin", "activated_env": "BINARYEN_ROOT=%installation_dir%%generator_prefix%_64bit_binaryen", @@ -462,62 +496,54 @@ "windows_url": "mingw_7.1.0_64bit.zip", "activated_cfg": "MINGW_ROOT='%installation_dir%'", "activated_path": "%installation_dir%/bin" - } - ], - - "sdks": [ - { - "version": "upstream-master", - "bitness": 64, - "uses": ["llvm-git-master-64bit", "node-12.9.1-64bit", "python-3.7.4-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], - "os": "win" }, { - "version": "upstream-master", + "id": "ninja", + "version": "git-release", "bitness": 64, - "uses": ["llvm-git-master-64bit", "node-12.9.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], - "os": "osx" + "url": "https://github.com/ninja-build/ninja.git", + "git_branch": "release", + "activated_cfg": "NINJA=%installation_dir%/bin", + "activated_path": "%installation_dir%/bin", + "cmake_build_type": "Release", + "custom_install_script": "build_ninja" }, { - "version": "upstream-master", + "id": "ccache", + "version": "git-emscripten", "bitness": 64, - "uses": ["llvm-git-master-64bit", "node-12.9.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], - "os": "linux" - }, - { - "version": "upstream-master", - "bitness": 32, - "uses": ["llvm-git-master-32bit", "emscripten-master-32bit", "binaryen-master-32bit"], - "os": "linux" - }, - { - "version": "fastcomp-master", - "bitness": 32, - "uses": ["fastcomp-clang-master-32bit", "node-12.9.1-32bit", "python-3.7.4-32bit", "java-8.152-32bit", "emscripten-master-32bit", "binaryen-master-32bit"], - "os": "win" - }, + "url": "https://github.com/juj/ccache.git", + "git_branch": "emscripten", + "activated_path": "%installation_dir%/bin", + "activated_env": "_EMCC_CCACHE=1;CCACHE_CONFIGPATH=%installation_dir%/emcc_ccache.conf", + "cmake_build_type": "Release", + "custom_install_script": "build_ccache" + } + ], + + "sdks": [ { - "version": "fastcomp-master", + "version": "upstream-main", "bitness": 64, - "uses": ["fastcomp-clang-master-64bit", "node-12.9.1-64bit", "python-3.7.4-64bit", "java-8.152-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], + "uses": ["python-3.9.2-1-64bit", "llvm-git-main-64bit", "node-14.15.5-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], "os": "win" }, { - "version": "fastcomp-master", + "version": "upstream-main", "bitness": 64, - "uses": ["fastcomp-clang-master-64bit", "node-12.9.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], - "os": "osx" + "uses": ["python-3.9.2-1-64bit", "llvm-git-main-64bit", "node-14.15.5-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], + "os": "macos" }, { - "version": "fastcomp-master", - "bitness": 32, - "uses": ["fastcomp-clang-master-32bit", "node-12.9.1-32bit", "emscripten-master-32bit", "binaryen-master-32bit"], + "version": "upstream-main", + "bitness": 64, + "uses": ["llvm-git-main-64bit", "node-14.15.5-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], "os": "linux" }, { - "version": "fastcomp-master", - "bitness": 64, - "uses": ["fastcomp-clang-master-64bit", "node-12.9.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], + "version": "upstream-main", + "bitness": 32, + "uses": ["llvm-git-main-32bit", "emscripten-main-32bit", "binaryen-main-32bit"], "os": "linux" }, { @@ -555,46 +581,48 @@ "version_filter": [ ["%tag%", ">", "1.37.22"] ] - }, + }, { "version": "releases-upstream-%releases-tag%", "bitness": 64, - "uses": ["node-12.9.1-64bit", "releases-upstream-%releases-tag%-64bit"], + "uses": ["node-14.15.5-64bit", "releases-upstream-%releases-tag%-64bit"], "os": "linux", "custom_install_script": "emscripten_npm_install" }, { "version": "releases-upstream-%releases-tag%", "bitness": 64, - "uses": ["node-12.9.1-64bit", "releases-upstream-%releases-tag%-64bit"], - "os": "osx", + "uses": ["node-14.15.5-64bit", "python-3.9.2-1-64bit", "releases-upstream-%releases-tag%-64bit"], + "os": "macos", + "arch": "x86_64", "custom_install_script": "emscripten_npm_install" }, { "version": "releases-upstream-%releases-tag%", "bitness": 64, - "uses": ["node-12.9.1-64bit", "python-3.7.4-pywin32-64bit", "java-8.152-64bit", "releases-upstream-%releases-tag%-64bit"], + "uses": ["node-14.15.5-64bit", "python-3.9.2-1-64bit", "java-8.152-64bit", "releases-upstream-%releases-tag%-64bit"], "os": "win", "custom_install_script": "emscripten_npm_install" }, { "version": "releases-fastcomp-%releases-tag%", "bitness": 64, - "uses": ["node-12.9.1-64bit", "releases-fastcomp-%releases-tag%-64bit"], + "uses": ["node-14.15.5-64bit", "releases-fastcomp-%releases-tag%-64bit"], "os": "linux", "custom_install_script": "emscripten_npm_install" }, { "version": "releases-fastcomp-%releases-tag%", "bitness": 64, - "uses": ["node-12.9.1-64bit", "releases-fastcomp-%releases-tag%-64bit"], - "os": "osx", + "uses": ["node-14.15.5-64bit", "python-3.7.4-2-64bit", "releases-fastcomp-%releases-tag%-64bit"], + "os": "macos", + "arch": "x86_64", "custom_install_script": "emscripten_npm_install" }, { "version": "releases-fastcomp-%releases-tag%", "bitness": 64, - "uses": ["node-12.9.1-64bit", "python-3.7.4-pywin32-64bit", "java-8.152-64bit", "releases-fastcomp-%releases-tag%-64bit"], + "uses": ["node-14.15.5-64bit", "python-3.7.4-pywin32-64bit", "java-8.152-64bit", "releases-fastcomp-%releases-tag%-64bit"], "os": "win", "custom_install_script": "emscripten_npm_install" }, @@ -637,8 +665,9 @@ { "version": "fastcomp-%precompiled_tag32%", "bitness": 32, - "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "emscripten-%precompiled_tag32%"], - "os": "osx", + "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "python-3.7.4-2-64bit", "emscripten-%precompiled_tag32%"], + "os": "macos", + "arch": "x86_64", "version_filter": [ ["%precompiled_tag32%", ">", "1.37.22"] ] @@ -646,8 +675,9 @@ { "version": "fastcomp-%precompiled_tag64%", "bitness": 64, - "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "emscripten-%precompiled_tag64%"], - "os": "osx", + "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "python-3.7.4-2-64bit", "emscripten-%precompiled_tag64%"], + "os": "macos", + "arch": "x86_64", "version_filter": [ ["%precompiled_tag64%", ">", "1.37.22"] ] diff --git a/llvm-tags-32bit.txt b/llvm-tags-32bit.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/llvm-tags-64bit.txt b/llvm-tags-64bit.txt index 0bc8cd8116..7920f38fdc 100644 --- a/llvm-tags-64bit.txt +++ b/llvm-tags-64bit.txt @@ -72,4 +72,4 @@ emscripten-llvm-e1.37.10.tar.gz emscripten-llvm-e1.37.1.tar.gz emscripten-llvm-e1.37.0.tar.gz emscripten-llvm-e1.36.14.tar.gz -emscripten-llvm-e1.36.13.tar.gz \ No newline at end of file +emscripten-llvm-e1.36.13.tar.gz diff --git a/scripts/create_release.py b/scripts/create_release.py new file mode 100755 index 0000000000..b84a43284f --- /dev/null +++ b/scripts/create_release.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import json +import os +import subprocess +import sys +from collections import OrderedDict + +script_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.dirname(script_dir) +sys.path.append(root_dir) + +import emsdk # noqa + + +def version_to_list(version_string): + return [int(part) for part in version_string.split('.')] + + +def main(args): + if subprocess.check_output(['git', 'status', '--porcelain'], cwd=root_dir).strip(): + print('tree is not clean') + sys.exit(1) + + release_info = emsdk.load_releases_info() + new_version = version_to_list(release_info['latest']) + new_version[-1] += 1 + branch_name = 'version_%s' % '_'.join(str(part) for part in new_version) + + # Create a new git branch + subprocess.check_call(['git', 'checkout', '-b', branch_name], cwd=root_dir) + + new_version = '.'.join(str(part) for part in new_version) + new_hash = emsdk.get_emscripten_releases_tot() + print('Creating new release: %s -> %s' % (new_version, new_hash)) + release_info['releases'][new_version] = new_hash + releases = [(k, v) for k, v in release_info['releases'].items()] + releases.sort(key=lambda pair: version_to_list(pair[0])) + + release_info['releases'] = OrderedDict(reversed(releases)) + release_info['latest'] = new_version + + with open(os.path.join(root_dir, 'emscripten-releases-tags.txt'), 'w') as f: + f.write(json.dumps(release_info, indent=2)) + f.write('\n') + + subprocess.check_call(os.path.join(script_dir, 'update_bazel_workspace.sh'), cwd=root_dir) + + # Create auto-generated changes to the new git branch + subprocess.check_call(['git', 'add', '-u', '.'], cwd=root_dir) + subprocess.check_call(['git', 'commit', '-m', new_version], cwd=root_dir) + + print('New relase created in branch: `%s`' % branch_name) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test.bat b/scripts/test.bat deleted file mode 100755 index 9dd53328e3..0000000000 --- a/scripts/test.bat +++ /dev/null @@ -1,7 +0,0 @@ -:: equivilent of test.sh as windows bat file -set PATH=%PATH%;%PYTHON_BIN% -@CALL emsdk install latest -@CALL emsdk activate latest -@CALL emsdk_env.bat --build=Release -@CALL python -c "import sys; print(sys.executable)" -@CALL emcc.bat -v diff --git a/scripts/test.py b/scripts/test.py deleted file mode 100755 index e1194d0de9..0000000000 --- a/scripts/test.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -import shutil -import subprocess -import sys -import tempfile - -WINDOWS = sys.platform.startswith('win') -MACOS = sys.platform == 'darwin' - -assert 'EM_CONFIG' in os.environ, "emsdk should be activated before running this script" - -emconfig = os.environ['EM_CONFIG'] -upstream_emcc = os.path.join('upstream', 'emscripten', 'emcc') -fastcomp_emcc = os.path.join('fastcomp', 'emscripten', 'emcc') -emsdk = './emsdk' -if WINDOWS: - upstream_emcc += '.bat' - fastcomp_emcc += '.bat' - emsdk = 'emsdk.bat' -else: - emsdk = './emsdk' - -# Utilities - - -def listify(x): - if type(x) == list or type(x) == tuple: - return x - return [x] - - -def check_call(cmd, **args): - if type(cmd) != list: - cmd = cmd.split() - print('running: %s' % cmd) - args['universal_newlines'] = True - subprocess.check_call(cmd, **args) - - -def checked_call_with_output(cmd, expected=None, unexpected=None, stderr=None): - cmd = cmd.split(' ') - print('running: %s' % cmd) - stdout = subprocess.check_output(cmd, stderr=stderr, universal_newlines=True) - if expected: - for x in listify(expected): - assert x in stdout, 'call had the right output: ' + stdout + '\n[[[' + x + ']]]' - if unexpected: - for x in listify(unexpected): - assert x not in stdout, 'call had the wrong output: ' + stdout + '\n[[[' + x + ']]]' - - -def failing_call_with_output(cmd, expected): - proc = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE, universal_newlines=True) - stdout, stderr = proc.communicate() - assert proc.returncode, 'call must have failed' - assert expected in stdout, 'call did not have the right output' - - -def hack_emsdk(marker, replacement): - src = open('emsdk.py').read() - assert marker in src - src = src.replace(marker, replacement) - name = '__test_emsdk' - open(name, 'w').write(src) - return name - - -# Set up - -open('hello_world.c', 'w').write('''\ -#include - -int main() { - printf("Hello, world!\\n"); - return 0; -} -''') - -TAGS = json.loads(open('emscripten-releases-tags.txt').read()) - -LIBC = os.environ['EM_CACHE'] + '/wasm/libc.a' - -# Tests - -print('test .emscripten contents (latest was installed/activated in test.sh)') -assert 'fastcomp' not in open(emconfig).read() -assert 'upstream' in open(emconfig).read() - -# Test we don't re-download unnecessarily -checked_call_with_output(emsdk + ' install latest', expected='already installed', unexpected='Downloading:') - -print('building proper system libraries') - - -def test_lib_building(emcc, use_asmjs_optimizer): - def test_build(args, expected=None, unexpected=None): - checked_call_with_output(emcc + ' hello_world.c' + args, - expected=expected, - unexpected=unexpected, - stderr=subprocess.STDOUT) - - # by default we ship libc, struct_info, and the asm.js optimizer, as they - # are important for various reasons (libc takes a long time to build; - # struct_info is a bootstrap product so if the user's setup is broken it's - # confusing; the asm.js optimizer is a native application so it needs a - # working native local build environment). otherwise we don't ship every - # single lib, so some building is expected on first run. - - unexpected_system_libs = ['generating system library: libc.', - 'generating system asset: optimizer'] - if use_asmjs_optimizer: - unexpected_system_libs += ['generating system asset: generated_struct_info.json'] - - first_time_system_libs = ['generating system library: libdlmalloc.'] - - test_build('', expected=first_time_system_libs, - unexpected=unexpected_system_libs) - test_build(' -O2', unexpected=unexpected_system_libs + first_time_system_libs) - test_build(' -s WASM=0', unexpected=unexpected_system_libs + first_time_system_libs) - test_build(' -O2 -s WASM=0', unexpected=unexpected_system_libs + first_time_system_libs) - - -def run_emsdk(cmd): - if type(cmd) != list: - cmd = cmd.split() - check_call([emsdk] + cmd) - - -test_lib_building(upstream_emcc, use_asmjs_optimizer=True) - -print('update') -run_emsdk('update-tags') - -print('test latest-releases-fastcomp') -run_emsdk('install latest-fastcomp') -run_emsdk('activate latest-fastcomp') - -test_lib_building(fastcomp_emcc, use_asmjs_optimizer=False) -assert open(emconfig).read().count('LLVM_ROOT') == 1 -assert 'upstream' not in open(emconfig).read() -assert 'fastcomp' in open(emconfig).read() - -print('verify version') -checked_call_with_output(fastcomp_emcc + ' -v', TAGS['latest'], stderr=subprocess.STDOUT) - -print('clear cache') -check_call(upstream_emcc + ' --clear-cache') -assert not os.path.exists(LIBC) - -# Test the normal tools like node don't re-download on re-install -print('another install must re-download') -checked_call_with_output(emsdk + ' uninstall node-12.9.1-64bit') -checked_call_with_output(emsdk + ' install node-12.9.1-64bit', expected='Downloading:', unexpected='already installed') -checked_call_with_output(emsdk + ' install node-12.9.1-64bit', unexpected='Downloading:', expected='already installed') - -print('test tot-upstream') -run_emsdk('install tot-upstream') -old_config = open(emconfig).read() -run_emsdk('activate tot-upstream') -assert old_config == open(emconfig + '.old').read() -# TODO; test on latest as well -check_call(upstream_emcc + ' hello_world.c') - -print('test tot-fastcomp') -run_emsdk('install tot-fastcomp') -run_emsdk('activate tot-fastcomp') -check_call(fastcomp_emcc + ' hello_world.c') - -print('test specific release (old)') -run_emsdk('install sdk-fastcomp-1.38.31-64bit') -run_emsdk('activate sdk-fastcomp-1.38.31-64bit') - -print('test specific release (new, short name)') -run_emsdk('install 1.38.33') -print('another install, but no need for re-download') -checked_call_with_output(emsdk + ' install 1.38.33', expected='Skipped', unexpected='Downloading:') -run_emsdk('activate 1.38.33') -assert 'upstream' not in open(emconfig).read() -assert 'fastcomp' in open(emconfig).read() - -print('test specific release (new, full name)') -run_emsdk('install sdk-1.38.33-upstream-64bit') -run_emsdk('activate sdk-1.38.33-upstream-64bit') - -print('test specific release (new, full name)') -run_emsdk('install sdk-tag-1.38.33-64bit') -run_emsdk('activate sdk-tag-1.38.33-64bit') - -print('test binaryen source build') -run_emsdk(['install', '--build=Release', '--generator=Unix Makefiles', 'binaryen-master-64bit']) - -print('test 32-bit error') - -failing_call_with_output('python %s install latest' % hack_emsdk('not is_os_64bit()', 'True'), 'this tool is only provided for 64-bit OSes') - -print('test non-git update') - -temp_dir = tempfile.mkdtemp() - -for filename in os.listdir('.'): - if not filename.startswith('.') and not os.path.isdir(filename): - shutil.copy2(filename, os.path.join(temp_dir, filename)) - -os.chdir(temp_dir) - -run_emsdk('update') -print('second time') -run_emsdk('update') - -print('verify downloads exist for all OSes') -latest_hash = TAGS['releases'][TAGS['latest']] -for osname, suffix in [ - ('linux', 'tbz2'), - ('mac', 'tbz2'), - ('win', 'zip') -]: - url = 'https://storage.googleapis.com/webassembly/emscripten-releases-builds/%s/%s/wasm-binaries.%s' % (osname, latest_hash, suffix) - print(' checking url: ' + url), - check_call('curl --fail --head --silent ' + url, stdout=subprocess.PIPE) diff --git a/scripts/test.sh b/scripts/test.sh deleted file mode 100755 index 1a4ae2fd1c..0000000000 --- a/scripts/test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -echo "test the standard workflow (as close as possible to how a user would do it, in the shell)" - -set -x -set -e - -./emsdk install latest -./emsdk activate latest -source ./emsdk_env.sh --build=Release -emcc -v diff --git a/scripts/update_bazel_workspace.sh b/scripts/update_bazel_workspace.sh new file mode 100755 index 0000000000..b9b7bd278a --- /dev/null +++ b/scripts/update_bazel_workspace.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# This script will update emsdk/bazel/WORKSPACE to the latest version of +# emscripten. It reads emsdk/emscripten-releases-tags.txt to get the latest +# version number. Then, it downloads the prebuilts for that version and computes +# the sha256sum for the archive. It then puts all this information into the +# emsdk/bazel/WORKSPACE file. + +ERR=0 +# Attempt to change to the emsdk root directory +cd $(dirname $0)/.. + +# If the previous command succeeded. We are in the emsdk root. Check to make +# sure the files and directories we need are present. +if [[ $? = 0 ]]; then + if [[ ! -f emscripten-releases-tags.txt ]]; then + echo "Cannot find emscripten-releases-tags.txt." + ERR=1 + fi + + if [[ ! -d bazel ]]; then + echo "Cannot find the bazel directory." + ERR=1 + elif [[ ! -f bazel/WORKSPACE ]]; then + echo "Cannot find bazel/WORKSPACE." + ERR=1 + fi +else + ERR=1 +fi + +if [[ $ERR = 1 ]]; then + echo "Unable to cd into the emsdk root directory." + exit 1 +fi + +URL1=https://storage.googleapis.com/webassembly/emscripten-releases-builds/ +URL2=/wasm-binaries.tbz2 + +# Get commit hash for $1 version +get_hash () { + echo $(grep "$1" emscripten-releases-tags.txt | grep -v latest | cut -f4 -d\") +} + +# Get sha256 for $1 os $2 hash +get_sha () { + echo $(curl "${URL1}$1/$2${URL2}" 2>/dev/null | sha256sum | awk '{print $1}') +} + +# Assemble dictionary line +revisions_item () { + hash=$(get_hash $1) + echo \ + "\ \"$1\": struct(\n" \ + "\ hash = \"$(get_hash ${hash})\",\n" \ + "\ sha_linux = \"$(get_sha linux ${hash})\",\n" \ + "\ sha_mac = \"$(get_sha mac ${hash})\",\n" \ + "\ sha_win = \"$(get_sha win ${hash})\",\n" \ + "\ )," +} + +append_revision () { + sed -i "5 i $(revisions_item $1)" bazel/revisions.bzl +} + +# Get the latest version number from emscripten-releases-tag.txt. +VER=$(grep -oP '(?<=latest\": \")([\d\.]+)(?=\")' \ + emscripten-releases-tags.txt \ + | sed --expression "s/\./\\\./g") + +append_revision ${VER} + +echo "Done!" diff --git a/scripts/update_node.py b/scripts/update_node.py index b0e778d370..45adce9d5d 100755 --- a/scripts/update_node.py +++ b/scripts/update_node.py @@ -16,8 +16,8 @@ import os import shutil -version = '12.16.3' -base = 'https://nodejs.org/dist/latest-v12.x/' +version = '14.15.5' +base = 'https://nodejs.org/dist/latest-v14.x/' upload_base = 'gs://webassembly/emscripten-releases-builds/deps/' suffixes = [ @@ -40,6 +40,7 @@ shutil.move(dirname, 'bin') os.mkdir(dirname) shutil.move('bin', dirname) + os.remove(filename) subprocess.check_call(['zip', '-rq', filename, dirname]) shutil.rmtree(dirname) diff --git a/scripts/update_python.py b/scripts/update_python.py index eed07efd4e..390aa71521 100755 --- a/scripts/update_python.py +++ b/scripts/update_python.py @@ -7,25 +7,34 @@ """Updates the python binaries that we cache store at http://storage.google.com/webassembly. -Currently this is windows only and we rely on the system python on other -platforms. +We only supply binaries for windows and macOS, but we do it very different ways for those two OSes. -We currently bundle a version of python for windows use the following -recipe: +Windows recipe: 1. Download the "embeddable zip file" version of python from python.org 2. Remove .pth file to work around https://bugs.python.org/issue34841 3. Download and install pywin32 in the `site-packages` directory 4. Re-zip and upload to storage.google.com + +macOS recipe: + 1. Clone cpython + 2. Use homebrew to install and configure openssl (for static linking!) + 3. Build cpython from source and use `make install` to create archive. """ -import urllib.request -import subprocess +import glob +import multiprocessing import os +import platform +import urllib.request import shutil +import subprocess import sys +from subprocess import check_call -version = '3.7.4' +version = '3.9.2' +major_minor_version = '.'.join(version.split('.')[:2]) # e.g. '3.9.2' -> '3.9' base = 'https://www.python.org/ftp/python/%s/' % version +revision = '1' pywin32_version = '227' pywin32_base = 'https://github.com/mhammond/pywin32/releases/download/b%s/' % pywin32_version @@ -33,13 +42,31 @@ upload_base = 'gs://webassembly/emscripten-releases-builds/deps/' +def unzip_cmd(): + # Use 7-Zip if available (https://www.7-zip.org/) + sevenzip = os.path.join(os.getenv('ProgramFiles'), '7-Zip', '7z.exe') + if os.path.isfile(sevenzip): + return [sevenzip, 'x'] + # Fall back to 'unzip' tool + return ['unzip', '-q'] + + +def zip_cmd(): + # Use 7-Zip if available (https://www.7-zip.org/) + sevenzip = os.path.join(os.getenv('ProgramFiles'), '7-Zip', '7z.exe') + if os.path.isfile(sevenzip): + return [sevenzip, 'a', '-mx9'] + # Fall back to 'zip' tool + return ['zip', '-rq'] + + def make_python_patch(arch): if arch == 'amd64': - pywin32_filename = 'pywin32-%s.win-%s-py3.7.exe' % (pywin32_version, arch) + pywin32_filename = 'pywin32-%s.win-%s-py%s.exe' % (pywin32_version, arch, major_minor_version) else: - pywin32_filename = 'pywin32-%s.%s-py3.7.exe' % (pywin32_version, arch) + pywin32_filename = 'pywin32-%s.%s-py%s.exe' % (pywin32_version, arch, major_minor_version) filename = 'python-%s-embed-%s.zip' % (version, arch) - out_filename = 'python-%s-embed-%s+pywin32.zip' % (version, arch) + out_filename = 'python-%s-%s-embed-%s+pywin32.zip' % (version, revision, arch) if not os.path.exists(pywin32_filename): download_url = pywin32_base + pywin32_filename print('Downloading pywin32: ' + download_url) @@ -51,33 +78,94 @@ def make_python_patch(arch): urllib.request.urlretrieve(download_url, filename) os.mkdir('python-embed') - subprocess.check_call(['unzip', '-q', os.path.abspath(filename)], cwd='python-embed') - os.remove(os.path.join('python-embed', 'python37._pth')) + check_call(unzip_cmd() + [os.path.abspath(filename)], cwd='python-embed') + os.remove(os.path.join('python-embed', 'python%s._pth' % major_minor_version.replace('.', ''))) os.mkdir('pywin32') - rtn = subprocess.call(['unzip', '-q', os.path.abspath(pywin32_filename)], cwd='pywin32') + rtn = subprocess.call(unzip_cmd() + [os.path.abspath(pywin32_filename)], cwd='pywin32') assert rtn in [0, 1] os.mkdir(os.path.join('python-embed', 'lib')) shutil.move(os.path.join('pywin32', 'PLATLIB'), os.path.join('python-embed', 'lib', 'site-packages')) - subprocess.check_call(['zip', '-rq', os.path.join('..', out_filename), '.'], cwd='python-embed') + check_call(zip_cmd() + [os.path.join('..', out_filename), '.'], cwd='python-embed') + + # cleanup if everything went fine + shutil.rmtree('python-embed') + shutil.rmtree('pywin32') upload_url = upload_base + out_filename print('Uploading: ' + upload_url) cmd = ['gsutil', 'cp', '-n', out_filename, upload_url] print(' '.join(cmd)) - subprocess.check_call(cmd) - - # cleanup if everything went fine - shutil.rmtree('python-embed') - shutil.rmtree('pywin32') + check_call(cmd) + + +def build_python(): + if sys.platform.startswith('darwin'): + osname = 'macos' + # Take some rather drastic steps to link openssl statically + check_call(['brew', 'install', 'openssl', 'pkg-config']) + if platform.machine() == 'x86_64': + prefix = '/usr/local' + min_macos_version = '10.13' + elif platform.machine() == 'arm64': + prefix = '/opt/homebrew' + min_macos_version = '11.0' + + osname += '-' + platform.machine() # Append '-x86_64' or '-arm64' depending on current arch. (TODO: Do this for Linux too, move this below?) + + try: + os.remove(os.path.join(prefix, 'opt', 'openssl', 'lib', 'libssl.dylib')) + os.remove(os.path.join(prefix, 'opt', 'openssl', 'lib', 'libcrypto.dylib')) + except Exception: + pass + os.environ['PKG_CONFIG_PATH'] = os.path.join(prefix, 'opt', 'openssl', 'lib', 'pkgconfig') + else: + osname = 'linux' + + src_dir = 'cpython' + if not os.path.exists(src_dir): + check_call(['git', 'clone', 'https://github.com/python/cpython']) + check_call(['git', 'checkout', 'v' + version], cwd=src_dir) + + min_macos_version_line = '-mmacosx-version-min=' + min_macos_version # Specify the min OS version we want the build to work on + build_flags = min_macos_version_line + ' -Werror=partial-availability' # Build against latest SDK, but issue an error if using any API that would not work on the min OS version + env = os.environ.copy() + env['MACOSX_DEPLOYMENT_TARGET'] = min_macos_version + check_call(['./configure', 'CFLAGS=' + build_flags, 'CXXFLAGS=' + build_flags, 'LDFLAGS=' + min_macos_version_line], cwd=src_dir, env=env) + check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir, env=env) + check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir, env=env) + + install_dir = os.path.join(src_dir, 'install') + + # Install requests module. This is needed in particualr on macOS to ensure + # SSL certificates are available (certifi in installed and used by requests). + pybin = os.path.join(src_dir, 'install', 'usr', 'local', 'bin', 'python3') + pip = os.path.join(src_dir, 'install', 'usr', 'local', 'bin', 'pip3') + check_call([pybin, pip, 'install', 'requests']) + + dirname = 'python-%s-%s' % (version, revision) + if os.path.isdir(dirname): + print('Erasing old build directory ' + dirname) + shutil.rmtree(dirname) + os.rename(os.path.join(install_dir, 'usr', 'local'), dirname) + tarball = 'python-%s-%s-%s.tar.gz' % (version, revision, osname) + shutil.rmtree(os.path.join(dirname, 'lib', 'python' + major_minor_version, 'test')) + shutil.rmtree(os.path.join(dirname, 'include')) + for lib in glob.glob(os.path.join(dirname, 'lib', 'lib*.a')): + os.remove(lib) + check_call(['tar', 'zcvf', tarball, dirname]) + print('Uploading: ' + upload_base + tarball) + check_call(['gsutil', 'cp', '-n', tarball, upload_base + tarball]) def main(): - for arch in ('amd64', 'win32'): - make_python_patch(arch) - + if sys.platform.startswith('win'): + for arch in ('amd64', 'win32'): + make_python_patch(arch) + else: + build_python() return 0 diff --git a/test/test.bat b/test/test.bat new file mode 100755 index 0000000000..9fea0c1f43 --- /dev/null +++ b/test/test.bat @@ -0,0 +1,7 @@ +:: equivilent of test.sh as windows bat file +set PATH=%PATH%;%PYTHON_BIN% +CALL emsdk install latest +CALL emsdk activate latest +CALL emsdk_env.bat +CALL python -c "import sys; print(sys.executable)" +CALL emcc.bat -v diff --git a/test/test.py b/test/test.py new file mode 100755 index 0000000000..2824efc66e --- /dev/null +++ b/test/test.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +WINDOWS = sys.platform.startswith('win') +MACOS = sys.platform == 'darwin' + +assert 'EM_CONFIG' in os.environ, "emsdk should be activated before running this script" + +emconfig = os.environ['EM_CONFIG'] +upstream_emcc = os.path.join('upstream', 'emscripten', 'emcc') +fastcomp_emcc = os.path.join('fastcomp', 'emscripten', 'emcc') +emsdk = './emsdk' +if WINDOWS: + upstream_emcc += '.bat' + fastcomp_emcc += '.bat' + emsdk = 'emsdk.bat' +else: + emsdk = './emsdk' + +# Utilities + + +def listify(x): + if type(x) == list or type(x) == tuple: + return x + return [x] + + +def check_call(cmd, **args): + if type(cmd) != list: + cmd = cmd.split() + print('running: %s' % cmd) + args['universal_newlines'] = True + subprocess.check_call(cmd, **args) + + +def checked_call_with_output(cmd, expected=None, unexpected=None, stderr=None): + cmd = cmd.split(' ') + print('running: %s' % cmd) + try: + stdout = subprocess.check_output(cmd, stderr=stderr, universal_newlines=True) + except subprocess.CalledProcessError as e: + print(e.stderr) + print(e.stdout) + raise e + + if expected: + for x in listify(expected): + assert x in stdout, 'call had the right output: ' + stdout + '\n[[[' + x + ']]]' + if unexpected: + for x in listify(unexpected): + assert x not in stdout, 'call had the wrong output: ' + stdout + '\n[[[' + x + ']]]' + + +def failing_call_with_output(cmd, expected): + proc = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + stdout, stderr = proc.communicate() + if WINDOWS: + print('warning: skipping part of failing_call_with_output() due to error codes not being propagated (see #592)') + else: + assert proc.returncode, 'call must have failed: ' + str([stdout, "\n========\n", stderr]) + assert expected in stdout or expected in stderr, 'call did not have the right output' + + +def hack_emsdk(marker, replacement): + with open('emsdk.py') as f: + src = f.read() + assert marker in src + src = src.replace(marker, replacement) + name = '__test_emsdk' + with open(name, 'w') as f: + f.write(src) + return name + + +# Set up + +TAGS = json.loads(open('emscripten-releases-tags.txt').read()) + +# Tests + + +def do_lib_building(emcc): + cache_building_messages = ['generating system library: '] + + def do_build(args, expected): + if expected: + expected = cache_building_messages + unexpected = [] + else: + expected = [] + unexpected = cache_building_messages + checked_call_with_output(emcc + ' hello_world.c' + args, + expected=expected, + unexpected=unexpected, + stderr=subprocess.STDOUT) + + # The emsdk ships all system libraries so we don't expect to see any + # cache population unless we explicly --clear-cache. + do_build('', expected=False) + check_call(emcc + ' --clear-cache') + do_build(' -O2', expected=True) + do_build(' -s WASM=0', expected=False) + do_build(' -O2 -s WASM=0', expected=False) + + +def run_emsdk(cmd): + if type(cmd) != list: + cmd = cmd.split() + check_call([emsdk] + cmd) + + +class Emsdk(unittest.TestCase): + @classmethod + def setUpClass(cls): + with open('hello_world.c', 'w') as f: + f.write('''\ +#include + +int main() { + printf("Hello, world!\\n"); + return 0; +} +''') + + def setUp(self): + run_emsdk('install latest') + run_emsdk('activate latest') + + def test_already_installed(self): + # Test we don't re-download unnecessarily + checked_call_with_output(emsdk + ' install latest', expected='already installed', unexpected='Downloading:') + + def test_list(self): + # Test we report installed tools properly. The latest version should be + # installed, but not some random old one. + checked_call_with_output(emsdk + ' list', expected=TAGS['latest'] + ' INSTALLED', unexpected='1.39.15 INSTALLED:') + + def test_config_contents(self): + print('test .emscripten contents') + with open(emconfig) as f: + config = f.read() + assert 'fastcomp' not in config + assert 'upstream' in config + + def test_lib_building(self): + print('building proper system libraries') + do_lib_building(upstream_emcc) + + def test_fastcomp(self): + print('test the last fastcomp release') + run_emsdk('install 1.40.1-fastcomp') + run_emsdk('activate 1.40.1-fastcomp') + + do_lib_building(fastcomp_emcc) + with open(emconfig) as f: + config = f.read() + assert config.count('LLVM_ROOT') == 1 + assert 'upstream' not in config + assert 'fastcomp' in config + + print('verify latest fastcomp version is fixed at 1.40.1') + checked_call_with_output(fastcomp_emcc + ' -v', '1.40.1', stderr=subprocess.STDOUT) + + def test_fastcomp_missing(self): + print('verify that attempting to use newer fastcomp gives an error') + fastcomp_error = 'The fastcomp backend is not getting new builds or releases. Please use the upstream llvm backend or use an older version than 2.0.0 (such as 1.40.1).' + failing_call_with_output(emsdk + ' install latest-fastcomp', fastcomp_error) + failing_call_with_output(emsdk + ' install tot-fastcomp', fastcomp_error) + failing_call_with_output(emsdk + ' install 2.0.0-fastcomp', fastcomp_error) + + def test_redownload(self): + print('go back to using upstream') + run_emsdk('activate latest') + + # Test the normal tools like node don't re-download on re-install + print('another install must re-download') + checked_call_with_output(emsdk + ' uninstall node-14.15.5-64bit') + checked_call_with_output(emsdk + ' install node-14.15.5-64bit', expected='Downloading:', unexpected='already installed') + checked_call_with_output(emsdk + ' install node-14.15.5-64bit', unexpected='Downloading:', expected='already installed') + + def test_tot_upstream(self): + print('test update-tags') + run_emsdk('update-tags') + print('test tot-upstream') + run_emsdk('install tot-upstream') + with open(emconfig) as f: + config = f.read() + run_emsdk('activate tot-upstream') + with open(emconfig + '.old') as f: + old_config = f.read() + self.assertEqual(config, old_config) + # TODO; test on latest as well + check_call(upstream_emcc + ' hello_world.c') + + def test_closure(self): + # Specificlly test with `--closure` so we know that node_modules is working + check_call(upstream_emcc + ' hello_world.c --closure=1') + + def test_specific_old(self): + print('test specific release (old, using sdk-* notation)') + run_emsdk('install sdk-fastcomp-1.38.31-64bit') + run_emsdk('activate sdk-fastcomp-1.38.31-64bit') + + def test_specific_version(self): + print('test specific release (new, short name)') + run_emsdk('install 1.38.33') + print('another install, but no need for re-download') + checked_call_with_output(emsdk + ' install 1.38.33', expected='Skipped', unexpected='Downloading:') + run_emsdk('activate 1.38.33') + with open(emconfig) as f: + config = f.read() + assert 'upstream' not in config + assert 'fastcomp' in config + + def test_specific_version_full(self): + print('test specific release (new, full name)') + run_emsdk('install sdk-1.38.33-upstream-64bit') + run_emsdk('activate sdk-1.38.33-upstream-64bit') + print('test specific release (new, tag name)') + run_emsdk('install sdk-tag-1.38.33-64bit') + run_emsdk('activate sdk-tag-1.38.33-64bit') + + def test_binaryen_from_source(self): + print('test binaryen source build') + run_emsdk(['install', '--build=Release', '--generator=Unix Makefiles', 'binaryen-main-64bit']) + + def test_no_32bit(self): + print('test 32-bit error') + emsdk_hacked = hack_emsdk('not is_os_64bit()', 'True') + failing_call_with_output('python %s install latest' % emsdk_hacked, 'this tool is only provided for 64-bit OSes') + os.remove(emsdk_hacked) + + def test_update_no_git(self): + print('test non-git update') + + temp_dir = tempfile.mkdtemp() + for filename in os.listdir('.'): + if not filename.startswith('.') and not os.path.isdir(filename): + shutil.copy2(filename, os.path.join(temp_dir, filename)) + + os.chdir(temp_dir) + run_emsdk('update') + + print('second time') + run_emsdk('update') + + def test_install_arbitrary(self): + # Test that its possible to install arbrary emscripten-releases SDKs + run_emsdk('install 5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2') + + # Check that its not re-downloaded + checked_call_with_output(emsdk + ' install 5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2', expected='Skipped', unexpected='Downloading:') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 0000000000..9d8171e79b --- /dev/null +++ b/test/test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +echo "test the standard workflow (as close as possible to how a user would do it, in the shell)" + +set -x +set -e + +# Test that arbitrary (non-released) versions can be installed and +# activated. +./emsdk install sdk-upstream-5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2 +./emsdk activate sdk-upstream-5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2 +source ./emsdk_env.sh +which emcc +emcc -v + +# Install an older version of the SDK that requires EM_CACHE to be +# set in the environment, so that we can test it is later removed +./emsdk install sdk-fastcomp-3b8cff670e9233a6623563add831647e8689a86b +./emsdk activate sdk-fastcomp-3b8cff670e9233a6623563add831647e8689a86b +source ./emsdk_env.sh +which emcc +emcc -v +test -n "$EM_CACHE" + +# Install the latest version of the SDK which is the expected precondition +# of test.py. +./emsdk install latest +./emsdk activate latest +source ./emsdk_env.sh --build=Release +# Test that EM_CACHE was unset +test -z "$EM_CACHE" + +# On mac and windows python3 should be in the path and point to the +# bundled version. +which python3 +which emcc +emcc -v diff --git a/test/test_activation.ps1 b/test/test_activation.ps1 new file mode 100644 index 0000000000..1aa0a97c9b --- /dev/null +++ b/test/test_activation.ps1 @@ -0,0 +1,107 @@ +# This test installs emsdk and activates the latest toolchain using `--system` or `--permanent` flags, +# and checks if the environment variables and PATH are correctly updated. Set $env:SYSTEM_FLAG and $env:PERMANENT_FLAG to test each. +# If no flag is provided the process/shell values are tested. See the CI file for an example. + +refreshenv + +$repo_root = [System.IO.Path]::GetDirectoryName((resolve-path "$PSScriptRoot")) + +$PATH_USER_BEFORE = [System.Environment]::GetEnvironmentVariable("PATH", "User") +$PATH_MACHINE_BEFORE = [System.Environment]::GetEnvironmentVariable("PATH", "Machine") +$PATH_Process_BEFORE = [System.Environment]::GetEnvironmentVariable("PATH", "Process") + + +try { + + & "$repo_root/emsdk.ps1" install latest + + $esc = '--%' + & "$repo_root/emsdk.ps1" activate latest $esc $env:PERMANENT_FLAG $env:SYSTEM_FLAG + + if ($env:SYSTEM_FLAG) { + $env_type = "Machine" + } + elseif ($env:PERMANENT_FLAG) { + $env_type = "User" + } else { + $env_type = "Process" + } + + $EMSDK = [System.Environment]::GetEnvironmentVariable("EMSDK", $env_type) + $EM_CONFIG = [System.Environment]::GetEnvironmentVariable("EM_CONFIG", $env_type) + $EMSDK_NODE = [System.Environment]::GetEnvironmentVariable("EMSDK_NODE", $env_type) + $EMSDK_PYTHON = [System.Environment]::GetEnvironmentVariable("EMSDK_PYTHON", $env_type) + $JAVA_HOME = [System.Environment]::GetEnvironmentVariable("JAVA_HOME", $env_type) + $PATH = [System.Environment]::GetEnvironmentVariable("PATH", $env_type) + + if (!$EMSDK) { + throw "EMSDK is not set for the user" + } + if (!$EM_CONFIG) { + throw "EM_CONFIG is not set for the user" + } + if (!$EMSDK_NODE) { + throw "EMSDK_NODE is not set for the user" + } + if (!$JAVA_HOME) { + throw "JAVA_HOME is not set for the user" + } + if (!$EMSDK_PYTHON) { + throw "EMSDK_PYTHON is not set for the user" + } + + + $path_split = $PATH.Split(';') + + $EMSDK_Path = $path_split | Where-Object { $_ -like "$repo_root*" } + if (!$EMSDK_Path) { + throw "No path is added!" + } + $EMSDK_NODE_Path = $path_split | Where-Object { $_ -like "$repo_root\node*" } + if (!$EMSDK_NODE_Path) { + throw "$repo_root\\node is not added to path." + } + + $EMSDK_UPSTREAM_Path = $path_split | Where-Object { $_ -like "$repo_root\upstream\emscripten*" } + if (!$EMSDK_UPSTREAM_Path) { + throw "$repo_root\\upstream\emscripten is not added to path." + } + + +} +finally { + # Recover pre-split PATH + refreshenv + + [Environment]::SetEnvironmentVariable("Path", $PATH_USER_BEFORE, "User") + try { + [Environment]::SetEnvironmentVariable("Path", $PATH_MACHINE_BEFORE, "Machine") + } + catch {} + + [Environment]::SetEnvironmentVariable("Path", $PATH_Process_BEFORE, "Process") + + # Recover pre activation env variables + [Environment]::SetEnvironmentVariable("EMSDK", $null, "User") + [Environment]::SetEnvironmentVariable("EM_CONFIG", $null, "User") + [Environment]::SetEnvironmentVariable("EMSDK_NODE", $null, "User") + [Environment]::SetEnvironmentVariable("EMSDK_PYTHON", $null, "User") + [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "User") + + try { + [Environment]::SetEnvironmentVariable("EMSDK", $null, "Machine") + [Environment]::SetEnvironmentVariable("EM_CONFIG", $null, "Machine") + [Environment]::SetEnvironmentVariable("EMSDK_NODE", $null, "Machine") + [Environment]::SetEnvironmentVariable("EMSDK_PYTHON", $null, "Machine") + [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine") + } catch {} + + + [Environment]::SetEnvironmentVariable("EMSDK", $null, "Process") + [Environment]::SetEnvironmentVariable("EM_CONFIG", $null, "Process") + [Environment]::SetEnvironmentVariable("EMSDK_NODE", $null, "Process") + [Environment]::SetEnvironmentVariable("EMSDK_PYTHON", $null, "Process") + [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Process") + + refreshenv +} diff --git a/test/test_bazel.sh b/test/test_bazel.sh new file mode 100755 index 0000000000..6ae69f9ba8 --- /dev/null +++ b/test/test_bazel.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +echo "test bazel" + +set -x +set -e + +# Get the latest version number from emscripten-releases-tag.txt. +VER=$(grep -oP '(?<=latest\": \")([\d\.]+)(?=\")' \ + emscripten-releases-tags.txt \ + | sed --expression "s/\./\\\./g") +# Based on the latest version number, get the commit hash for that version. +HASH=$(grep "${VER}" emscripten-releases-tags.txt \ + | grep -v latest \ + | cut -f4 -d\") + +FAILMSG="!!! scripts/update_bazel_toolchain.sh needs to be run !!!" + +# Ensure the WORKSPACE file is up to date with the latest version. +grep ${VER} bazel/revisions.bzl || (echo ${FAILMSG} && false) +grep ${HASH} bazel/revisions.bzl || (echo ${FAILMSG} && false) + +cd bazel +bazel build //hello-world:hello-world-wasm +bazel build //hello-world:hello-world-wasm-simd + +cd test_external +bazel build //:hello-world-wasm diff --git a/test/test_bazel_mac.sh b/test/test_bazel_mac.sh new file mode 100755 index 0000000000..851ec5fdde --- /dev/null +++ b/test/test_bazel_mac.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +echo "test bazel" + +set -x +set -e + +# Get the latest version number from emscripten-releases-tag.txt. +VER=$(ggrep -oP '(?<=latest\": \")([\d\.]+)(?=\")' \ + emscripten-releases-tags.txt \ + | sed "s/\./\\\./g") +# Based on the latest version number, get the commit hash for that version. +HASH=$(grep "${VER}" emscripten-releases-tags.txt \ + | grep -v latest \ + | cut -f4 -d\") + +FAILMSG="!!! scripts/update_bazel_toolchain.sh needs to be run !!!" + +# Ensure the WORKSPACE file is up to date with the latest version. +grep ${VER} bazel/revisions.bzl || (echo ${FAILMSG} && false) +grep ${HASH} bazel/revisions.bzl || (echo ${FAILMSG} && false) + +cd bazel +bazel build //hello-world:hello-world-wasm +bazel build //hello-world:hello-world-wasm-simd + +cd test_external +bazel build //:hello-world-wasm diff --git a/test/test_path_preservation.ps1 b/test/test_path_preservation.ps1 new file mode 100644 index 0000000000..b894954564 --- /dev/null +++ b/test/test_path_preservation.ps1 @@ -0,0 +1,148 @@ +# This test installs emsdk and activates the latest toolchain using `--system` or `--permanent` flags, +# and checks if parts of PATH are lost or overwritten. Set $env:SYSTEM_FLAG and $env:PERMANENT_FLAG to test each. +# If no flag is provided the process/shell values are tested. See the CI file for an example. + +refreshenv + +$repo_root = [System.IO.Path]::GetDirectoryName((resolve-path "$PSScriptRoot")) + +$PATH_USER_BEFORE = [System.Environment]::GetEnvironmentVariable("PATH", "User") +$PATH_MACHINE_BEFORE = [System.Environment]::GetEnvironmentVariable("PATH", "Machine") +$PATH_Process_BEFORE = [System.Environment]::GetEnvironmentVariable("PATH", "Process") + +try { + + + & "$repo_root/emsdk.ps1" install latest + + $esc = '--%' + & "$repo_root/emsdk.ps1" activate latest $esc $env:PERMANENT_FLAG $env:SYSTEM_FLAG + + $PATH_USER = [System.Environment]::GetEnvironmentVariable("PATH", "User") + $PATH_MACHINE = [System.Environment]::GetEnvironmentVariable("PATH", "Machine") + $PATH_Process = [System.Environment]::GetEnvironmentVariable("PATH", "Process") + + if ($env:SYSTEM_FLAG) { + echo "--system test............................." + $path_before_arr = $PATH_MACHINE_BEFORE.Split(';') + $path_arr = $PATH_MACHINE.Split(';') + } + elseif ($env:PERMANENT_FLAG) { + echo "--permanent test.........................." + $path_before_arr = $PATH_USER_BEFORE.Split(';') + $path_arr = $PATH_USER.Split(';') + } else { + echo "no flag test (shell/process).............." + $path_before_arr = $PATH_Process_BEFORE.Split(';') + $path_arr = $PATH_Process.Split(';') + } + + + $EMSDK_Path = $path_arr | Where-Object { $_ -like "$repo_root*" } + $EMSDK_NODE_Path = $path_arr | Where-Object { $_ -like "$repo_root\node*" } + $EMSDK_PYTHON_Path = $path_arr | Where-Object { $_ -like "$repo_root\python*" } + $EMSDK_JAVA_Path = $path_arr | Where-Object { $_ -like "$repo_root\java*" } + $EMSDK_UPSTREAM_Path = $path_arr | Where-Object { $_ -like "$repo_root\upstream\emscripten*" } + + $number_of_items = $path_arr.count + [System.Collections.ArrayList]$rest_of_path = @() + Foreach ($item in $path_arr) { + if ( + ($item -like "$repo_root*") -or + ($item -like "$repo_root\node*") -or + ($item -like "$repo_root\python*") -or + ($item -like "$repo_root\java*") -or + ($item -like "$repo_root\upstream\emscripten*") + ) { + echo "$item is on the PATH" + } + else { + $rest_of_path.add($item) | Out-Null + } + } + + # compare the PATHs before activation and after activation + if (Compare-Object -ReferenceObject $path_before_arr -DifferenceObject $rest_of_path ) { + echo "Old path is ............................." + echo $path_before_arr + echo "Current rest of path is ................." + echo $rest_of_path + throw "some parts of PATH are removed" + } + + # Compare the other untouched PATH + if ($env:SYSTEM_FLAG) { + if (Compare-Object -ReferenceObject $PATH_USER_BEFORE.Split(';') -DifferenceObject $PATH_USER.Split(';') ) { + echo "Old user path is ...................." + echo $PATH_USER_BEFORE + echo "Current user path is ................" + echo $PATH_USER + throw "User PATH are changed while --system had been provided" + } + } + elseif ($env:PERMANENT_FLAG) { + if (Compare-Object -ReferenceObject $PATH_MACHINE_BEFORE.Split(';') -DifferenceObject $PATH_MACHINE.Split(';') ) { + echo "Old machine path is.................." + echo $PATH_MACHINE_BEFORE + echo "Current machine path is.............." + echo $PATH_MACHINE + throw "MACHINE PATH are changed while --system was not provided" + } + } else { + if ( + (Compare-Object -ReferenceObject $PATH_MACHINE_BEFORE.Split(';') -DifferenceObject $PATH_MACHINE.Split(';')) -or + (Compare-Object -ReferenceObject $PATH_MACHINE_BEFORE.Split(';') -DifferenceObject $PATH_MACHINE.Split(';')) + ) { + echo "Old machine path is.................." + echo $PATH_MACHINE_BEFORE + echo "Current machine path is.............." + echo $PATH_MACHINE + echo "Old user path is ...................." + echo $PATH_USER_BEFORE + echo "Current user path is ................" + echo $PATH_USER + throw "MACHINE/USER PATH are changed while no flag was provided" + } + } + + + + +} +finally { + # Recover pre-split PATH + refreshenv + + [Environment]::SetEnvironmentVariable("Path", $PATH_USER_BEFORE, "User") + try { + [Environment]::SetEnvironmentVariable("Path", $PATH_MACHINE_BEFORE, "Machine") + } + catch {} + + [Environment]::SetEnvironmentVariable("Path", $PATH_Process_BEFORE, "Process") + + # Recover pre activation env variables + [Environment]::SetEnvironmentVariable("EMSDK", $null, "User") + [Environment]::SetEnvironmentVariable("EM_CONFIG", $null, "User") + [Environment]::SetEnvironmentVariable("EMSDK_NODE", $null, "User") + [Environment]::SetEnvironmentVariable("EMSDK_PYTHON", $null, "User") + [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "User") + + try { + [Environment]::SetEnvironmentVariable("EMSDK", $null, "Machine") + [Environment]::SetEnvironmentVariable("EM_CONFIG", $null, "Machine") + [Environment]::SetEnvironmentVariable("EMSDK_NODE", $null, "Machine") + [Environment]::SetEnvironmentVariable("EMSDK_PYTHON", $null, "Machine") + [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine") + } catch {} + + + [Environment]::SetEnvironmentVariable("EMSDK", $null, "Process") + [Environment]::SetEnvironmentVariable("EM_CONFIG", $null, "Process") + [Environment]::SetEnvironmentVariable("EMSDK_NODE", $null, "Process") + [Environment]::SetEnvironmentVariable("EMSDK_PYTHON", $null, "Process") + [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Process") + + refreshenv + +} diff --git a/test/test_source_env.sh b/test/test_source_env.sh new file mode 100755 index 0000000000..71a3e4070a --- /dev/null +++ b/test/test_source_env.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +echo "Test ability to source emsdk_env.sh in different shells" + +if [ -n "$EMSDK" ]; then + echo "EMSDK is already defined in this shell. Run tests in a shell without sourcing emsdk_env.sh first" + exit 1 +fi + +DIR=$(dirname "$BASH_SOURCE") + +# setup a symlink relative to the current dir +REL_LINK_DIR="$DIR/tmp" +if [ -d "$REL_LINK_DIR" ]; then + rm -rf "$REL_LINK_DIR" +fi +echo "Creating links in $REL_LINK_DIR" +mkdir -p "$REL_LINK_DIR" +(cd $DIR/.. && ln -s `pwd` "$REL_LINK_DIR/emsdk") +(cd $DIR/.. && ln -s `pwd`/emsdk_env.sh "$REL_LINK_DIR") + +# setup a symlink in an absolute directory +ABS_LINK_DIR="/tmp/emsdk_env_test" +if [ -d "$ABS_LINK_DIR" ]; then + rm -rf "$ABS_LINK_DIR" +fi +echo "Creating links in $ABS_LINK_DIR" +mkdir -p "$ABS_LINK_DIR" +(cd $DIR/.. && ln -s `pwd` "$ABS_LINK_DIR/emsdk") +(cd $DIR/.. && ln -s `pwd`/emsdk_env.sh "$ABS_LINK_DIR") + +PATH1="$DIR/../emsdk_env.sh" +PATH2="$REL_LINK_DIR/emsdk/emsdk_env.sh" +PATH3="$REL_LINK_DIR/emsdk_env.sh" +PATH4="$ABS_LINK_DIR/emsdk/emsdk_env.sh" +PATH5="$ABS_LINK_DIR/emsdk_env.sh" + +assert_emcc() { + current=$1 + cmd=$2 + value=$3 + if [ -z "$value" ] || [ "$value" == "false" ]; then + echo "FAILED: $current" + echo " unable to get EMSDK in $current using '$cmd'" + else + echo "SUCCESS: $current testing $cmd" + echo " -> EMSDK = $value" + fi +} + +test_bash() { + value=$(bash --rcfile <(echo $1)) + assert_emcc bash "$1" "$value" +} + +test_zsh() { + value=$(zsh -d -c "$1") + assert_emcc zsh "$1" "$value" +} + +test_ksh() { + value=$(ksh -c "$1") + assert_emcc ksh "$1" "$value" +} + +it_tests_direct_path() { + TEST_SCRIPT=". ${PATH1}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" + TEST_SCRIPT="source ${PATH1}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" +} + +it_tests_via_relative_dir_symlink() { + TEST_SCRIPT=". ${PATH2}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" + TEST_SCRIPT="source ${PATH2}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" +} + +it_tests_via_relative_file_symlink() { + TEST_SCRIPT=". ${PATH3}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" + TEST_SCRIPT="source ${PATH3}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" +} + +it_tests_via_absolute_dir_symlink() { + TEST_SCRIPT=". ${PATH4}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" + TEST_SCRIPT="source ${PATH4}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" +} + +it_tests_via_absolute_file_symlink() { + TEST_SCRIPT=". ${PATH5}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" + TEST_SCRIPT="source ${PATH5}"' >/dev/null 2>&1; if [ -n "$EMSDK" ]; then echo "$EMSDK"; else echo false; fi ; exit' + test_bash "$TEST_SCRIPT" + test_zsh "$TEST_SCRIPT" + test_ksh "$TEST_SCRIPT" +} + +run_bash_tests() { + it_tests_direct_path + it_tests_via_relative_dir_symlink + it_tests_via_relative_file_symlink + it_tests_via_absolute_dir_symlink + it_tests_via_absolute_file_symlink +} + +run_bash_tests + +rm -rf $REL_LINK_DIR +rm -rf $ABS_LINK_DIR