Merge branch 'main' into build_rules

* main: fix: copy LLVM libraries in toolchain (#22) chore: improve GCC packaging (#30) fix: resolve macos headers (#29) Revert "fix: use resource directory for clang (#27)" (#28) fix: use resource directory for clang (#27) fix: remove duplicate objects (#26) fix: add partition objects to main module (#25) fix: correctly resolve target dependencies (#24) fix: allow module partitions (#23) fix: use github mirror to build gcc (#18)
alexbatashev · Jan 2, 2024 · 143b4c2 · 143b4c2
2 parents 20b0357 + ccf0b48
commit 143b4c2
Show file tree

Hide file tree

Showing 10 changed files with 51 additions and 25 deletions.
diff --git a/.github/workflows/gcc.yaml b/.github/workflows/gcc.yaml
@@ -16,28 +16,33 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          repository: "gcc.gnu.org/git/gcc.git"
-          ref: release/${{ inputs.gcc_release }}
+          repository: "gcc-mirror/gcc"
+          ref: releases/${{ inputs.gcc_release }}
       - name: Build and install
         env:
           CXXFLAGS: -w
           CFLAGS: -w
         run: |
-            mkdir -p install/${{ inputs.gcc_release }}-linux-x86_64
+            mkdir -p install
             ./configure --with-static-standard-libraries \
                 --enable-gold --enable-languages=c,c++,lto \
                 --enable-lto --host=x86_64-pc-linux-gnu \
-                --with-isl --with-zstd --prefix=$PWD/install/${{ inputs.gcc_release }}-linux-x86_64
-            make -j2
-            make install
+                --with-isl --with-zstd --prefix=/
+            make -j4
+            make DESTDIR=$PWD/install/${{ inputs.gcc_release }}-linux-x86_64-stdlib install-strip-target-libstdc++-v3
+            make DESTDIR=$PWD/install/${{ inputs.gcc_release }}-linux-x86_64-stdlib install-strip-target-libgcc
+            make DESTDIR=$PWD/install/${{ inputs.gcc_release }}-linux-x86_64-openmp install-strip-target-libgomp
+            make DESTDIR=$PWD/install/${{ inputs.gcc_release }}-linux-x86_64 install-strip
 
             cd install
 
+            tar --zstd -cf ${{ inputs.gcc_release }}-linux-x86_64-stdlib.tar.zst ${{ inputs.gcc_release }}-linux-x86_64-stdlib
+            tar --zstd -cf ${{ inputs.gcc_release }}-linux-x86_64-openmp.tar.zst ${{ inputs.gcc_release }}-linux-x86_64-openmp
             tar --zstd -cf ${{ inputs.gcc_release }}-linux-x86_64.tar.zst ${{ inputs.gcc_release }}-linux-x86_64
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
-          path: install/${{ inputs.gcc_release }}-linux-x86_64.tar.zst
+          path: install/${{ inputs.gcc_release }}-linux-x86_64*.tar.zst
           name: linux-x86_64-toolchain
 
   publish_release:
@@ -50,6 +55,8 @@ jobs:
           run: |
             touch RELASE.txt
             sha256sum linux-x86_64-toolchain/${{ inputs.gcc_release }}-linux-x86_64.tar.zst >> RELEASE.txt
+            sha256sum linux-x86_64-toolchain/${{ inputs.gcc_release }}-linux-x86_64-stdlib.tar.zst >> RELEASE.txt
+            sha256sum linux-x86_64-toolchain/${{ inputs.gcc_release }}-linux-x86_64-openmp.tar.zst >> RELEASE.txt
         - name: Create release
           uses: softprops/action-gh-release@v1
           with:
@@ -59,4 +66,6 @@ jobs:
             name: GCC Toolchain ${{ inputs.gcc_release }}
             prerelease: true
             files: |
-              linux-x86_64-toolchain/${{ inputs.gcc_release }}-linux-x86_64.tar.zst
+              linux-x86_64-toolchain/${{ inputs.gcc_release }}-linux-x86_64.tar.zst
+              linux-x86_64-toolchain/${{ inputs.gcc_release }}-linux-x86_64-stdlib.tar.zst
+              linux-x86_64-toolchain/${{ inputs.gcc_release }}-linux-x86_64-openmp.tar.zst
diff --git a/.github/workflows/llvm.yaml b/.github/workflows/llvm.yaml
@@ -136,8 +136,8 @@ jobs:
           ninja llvm-dwp
           ninja install-pstl
           cp ./bin/llvm-dwp ./install/${{inputs.llvm_tag}}-macos-x86_64/bin
-          cp ./lib/libclang-*.dylib ./install/${{inputs.llvm_tag}}-macos-x86_64/lib || true
-          cp ./lib/libLLVM-*.dylib ./install/${{inputs.llvm_tag}}-macos-x86_64/lib || true
+          cp ./lib/libclang*.dylib ./install/${{inputs.llvm_tag}}-macos-x86_64/lib || true
+          cp ./lib/libLLVM*.dylib ./install/${{inputs.llvm_tag}}-macos-x86_64/lib || true
 
           cd install
 

diff --git a/cpp/private/actions/compile.bzl b/cpp/private/actions/compile.bzl
@@ -18,6 +18,7 @@ def _cpp_compile_impl(ctx, sources, headers, includes, modules, features, toolch
 
     for m in modules:
         module_files.append(m["file"])
+        module_files.append(m["source"])
         module_vars.append("{name}={file}".format(name = m["name"], file = m["file"].path))
 
     extra_vars["cpp_precompiled_modules"] = module_vars
@@ -31,7 +32,7 @@ def _cpp_compile_impl(ctx, sources, headers, includes, modules, features, toolch
             action_name = action_name,
         )
 
-        outfile = ctx.actions.declare_file("_objs/" + src.basename + ".o")
+        outfile = ctx.actions.declare_file("_objs/" + ctx.label.name + "/" + src.basename + ".o")
         args = get_compile_command_args(
             toolchain,
             source = src.path,

diff --git a/cpp/private/common.bzl b/cpp/private/common.bzl
@@ -6,8 +6,7 @@ def collect_external_headers(deps):
 
     for dep in deps:
         if CcInfo in dep:
-            headers.extend(dep[CcInfo].compilation_context.direct_public_headers)
-            headers.extend(dep[CcInfo].compilation_context.direct_textual_headers)
+            headers.extend(dep[CcInfo].compilation_context.headers.to_list())
 
     return headers
 
@@ -16,6 +15,9 @@ def collect_external_includes(deps):
     for dep in deps:
         if CcInfo in dep:
             includes.extend(dep[CcInfo].compilation_context.includes.to_list())
+            includes.extend(dep[CcInfo].compilation_context.external_includes.to_list())
+            includes.extend(dep[CcInfo].compilation_context.system_includes.to_list())
+            includes.extend(dep[CcInfo].compilation_context.quote_includes.to_list())
     return depset(includes)
 
 def collect_modules(deps):
@@ -26,7 +28,14 @@ def collect_modules(deps):
             modules.append({
                 "name": dep[CppModuleInfo].module_name,
                 "file": dep[CppModuleInfo].pcm,
+                "source": dep[CppModuleInfo].interface_source,
             })
+            for part in dep[CppModuleInfo].partitions.to_list():
+                modules.append({
+                    "name": part[CppModuleInfo].module_name,
+                    "file": part[CppModuleInfo].pcm,
+                    "source": part[CppModuleInfo].interface_source,
+                })
 
     return modules
 
@@ -35,7 +44,7 @@ def collect_module_objects(deps):
 
     for dep in deps:
         if CppModuleInfo in dep:
-            obj_files.extend(dep[CppModuleInfo].objs)
+            obj_files.append(dep[CppModuleInfo].objs)
 
     return obj_files
 

diff --git a/cpp/private/target_rules.bzl b/cpp/private/target_rules.bzl
@@ -52,7 +52,7 @@ def shlib_impl(ctx):
     modules = collect_modules(ctx.attr.deps)
 
     obj_files = cpp_compile(ctx.files.srcs, all_headers, includes, modules, features, toolchain)
-    obj_files = cpp_strip_objects(obj_files, features, toolchain) + collect_module_objects(ctx.attr.deps)
+    obj_files = depset(cpp_strip_objects(obj_files, features, toolchain), transitive = collect_module_objects(ctx.attr.deps))
 
     shlib = ctx.actions.declare_file(ctx.attr.lib_prefix + ctx.attr.name + ctx.attr.lib_suffix)
     compile_output = shlib
@@ -61,7 +61,7 @@ def shlib_impl(ctx):
 
     link_flags, lib_inputs = resolve_linker_arguments(ctx, toolchain, features, compile_output.path, True)
 
-    for obj in obj_files:
+    for obj in obj_files.to_list():
         link_flags.append(obj.path)
 
     linker = cc_common.get_tool_for_action(
@@ -71,7 +71,7 @@ def shlib_impl(ctx):
 
     ctx.actions.run(
         outputs = [compile_output],
-        inputs = depset(obj_files, transitive = [lib_inputs, toolchain.all_files]),
+        inputs = depset(obj_files.to_list(), transitive = [lib_inputs, toolchain.all_files]),
         executable = linker,
         arguments = link_flags,
         mnemonic = "CppLinkSharedLibrary",
@@ -123,7 +123,7 @@ def module_impl(ctx):
 
     headers = collect_external_headers(ctx.attr.deps)
     includes = depset(ctx.attr.includes, transitive = [collect_external_includes(ctx.attr.deps)])
-    modules = collect_modules(ctx.attr.deps)
+    modules = collect_modules(ctx.attr.deps + ctx.attr.partitions)
 
     extra_vars = {
         "cpp_precompiled_modules": [],
@@ -138,7 +138,7 @@ def module_impl(ctx):
 
     extra_vars["cpp_precompiled_modules"] = module_vars
 
-    pcm = ctx.actions.declare_file("_pcm/" + ctx.attr.module_name + "-" + ctx.files.interface[0].basename[:-(len(ctx.files.interface[0].extension) + 1)] + ".pcm")
+    pcm = ctx.actions.declare_file("_pcm/" + ctx.attr.name + "/" + ctx.attr.module_name + "-" + ctx.files.interface[0].basename[:-(len(ctx.files.interface[0].extension) + 1)] + ".pcm")
 
     precompile_args = get_compile_command_args(
         toolchain,
@@ -165,12 +165,14 @@ def module_impl(ctx):
     )
 
     obj_files = cpp_compile(ctx.files.srcs + [pcm], headers, includes, modules, features, toolchain, module_srcs = ctx.files.interface)
-    obj_files = cpp_strip_objects(obj_files, features, toolchain) + collect_module_objects(ctx.attr.deps)
+    obj_files = depset(cpp_strip_objects(obj_files, features, toolchain), transitive = collect_module_objects(ctx.attr.deps + ctx.attr.partitions))
 
     return CppModuleInfo(
         module_name = ctx.attr.module_name,
         pcm = pcm,
         objs = obj_files,
+        interface_source = ctx.files.interface[0],
+        partitions = depset(ctx.attr.partitions),
     )
 
 def binary_impl(ctx):
@@ -195,7 +197,7 @@ def binary_impl(ctx):
     modules = collect_modules(ctx.attr.deps)
 
     obj_files = cpp_compile(ctx.files.srcs, all_headers, includes, modules, features, toolchain)
-    obj_files = cpp_strip_objects(obj_files, features, toolchain) + collect_module_objects(ctx.attr.deps)
+    obj_files = depset(cpp_strip_objects(obj_files, features, toolchain), transitive = collect_module_objects(ctx.attr.deps))
 
     bin = ctx.actions.declare_file(ctx.attr.name + ctx.attr.bin_suffix)
     compile_output = bin
@@ -204,7 +206,7 @@ def binary_impl(ctx):
 
     link_flags, lib_inputs = resolve_linker_arguments(ctx, toolchain, features, compile_output.path, False)
 
-    for obj in obj_files:
+    for obj in obj_files.to_list():
         link_flags.append(obj.path)
 
     linker = cc_common.get_tool_for_action(
@@ -214,7 +216,7 @@ def binary_impl(ctx):
 
     ctx.actions.run(
         outputs = [compile_output],
-        inputs = depset(obj_files, transitive = [lib_inputs, toolchain.all_files]),
+        inputs = depset(obj_files.to_list(), transitive = [lib_inputs, toolchain.all_files]),
         executable = linker,
         arguments = link_flags,
         mnemonic = "CppLinkExecutable",

diff --git a/cpp/private/toolchain.bzl b/cpp/private/toolchain.bzl
@@ -179,6 +179,7 @@ def _get_clang_features(ctx):
             flag_set(
                 actions = all_cpp_compile_actions + [EXTRA_ACTIONS.cpp_module_precompile_interface],
                 flag_groups = [
+                    flag_group(flags = ["-fno-implicit-modules"]),
                     flag_group(
                         iterate_over = "cpp_precompiled_modules",
                         expand_if_available = "cpp_precompiled_modules",

diff --git a/cpp/private/unix_toolchain_features.bzl b/cpp/private/unix_toolchain_features.bzl
@@ -39,7 +39,7 @@ def get_default_flags(std_compile_flags, include_dirs, link_dirs, exec_rpath_pre
                     ACTION_NAMES.cpp_header_parsing,
                     ACTION_NAMES.cpp_module_compile,
                     ACTION_NAMES.cpp_module_codegen,
-                ],
+                ] + [EXTRA_ACTIONS.cpp_module_precompile_interface],
                 flag_groups = [
                     flag_group(
                         flags = ["-isystem" + x for x in include_dirs] + [

diff --git a/cpp/providers.bzl b/cpp/providers.bzl
@@ -3,5 +3,7 @@ CppModuleInfo = provider(
         "module_name": "name of the exported module",
         "pcm": "file containing precompiled module",
         "objs": "compiled object files",
+        "interface_source": "source file for the interface of the module",
+        "partitions": "required dependencies",
     },
 )
diff --git a/cpp/rules.bzl b/cpp/rules.bzl
@@ -47,6 +47,7 @@ _bin_attrs = {
 _module_attrs = {
     "srcs": attr.label_list(allow_files = True),
     "deps": attr.label_list(providers = [[CcInfo], [CppModuleInfo]]),
+    "partitions": attr.label_list(providers = [[CppModuleInfo]]),
     "module_name": attr.string(mandatory = True),
     "interface": attr.label(allow_files = True, mandatory = True),
     "includes": attr.string_list(),

diff --git a/integration_tests/rules/BUILD.bazel b/integration_tests/rules/BUILD.bazel
@@ -1,4 +1,5 @@
-load("@rules_cpp//cpp:rules.bzl", 
+load(
+    "@rules_cpp//cpp:rules.bzl",
     "clang_format",
     "cpp_binary",
     "cpp_module",