[codegen][MLIR] initial support codegen with MLIR

PikachuHy · Jan 7, 2024 · 3a077f7 · 3a077f7
1 parent ce74ecc
commit 3a077f7
Show file tree

Hide file tree

Showing 21 changed files with 1,245 additions and 13 deletions.
diff --git a/.bazeliskrc b/.bazeliskrc
@@ -0,0 +1 @@
+USE_BAZEL_VERSION=6.4.0
diff --git a/.bazelrc b/.bazelrc
@@ -2,4 +2,6 @@ build --incompatible_strict_action_env
 build --verbose_failures
 build --workspace_status_command "python3 workspace_status.py"
 build:macos --define client_type=macos
+build:macos --cxxopt=-std=c++20 --host_cxxopt=-std=c++20
 build:android --define client_type=android
+common --noenable_bzlmod
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -49,6 +49,16 @@ jobs:
         working-directory: ${{github.workspace}}/build/test
         run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure
 
+      - name: Configure CMake with MLIR
+        run: PKG_CONFIG_PATH=/usr/local/opt/icu4c/lib/pkgconfig cmake -B ${{github.workspace}}/build3 -DCMAKE_BUILD_TYPE=${{matrix.mode}} -G Ninja -DUSE_CCACHE=ON -DPSCM_ENABLE_MLIR_CODEGEN=ON
+
+      - name: Build with MLIR
+        run: cmake --build ${{github.workspace}}/build3 --config ${{matrix.mode}} --verbose -j
+
+      - name: Test with MLIR
+        working-directory: ${{github.workspace}}/build3/test
+        run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure
+
       - name: Configure CMake with C++20 Modules
         run: PKG_CONFIG_PATH=/usr/local/opt/icu4c/lib/pkgconfig  CXX=/usr/local/opt/llvm/bin/clang++ CC=/usr/local/opt/llvm/bin/clang cmake -B ${{github.workspace}}/build2 -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja -G Ninja -DUSE_CCACHE=ON -DPSCM_USE_CXX20_MODULES=ON
 

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -1,6 +1,12 @@
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
 load("version.bzl", "gen_pscm_version_info")
 load("test.bzl", "collect_pscm_tests")
+load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library")
+
+config_setting(
+    name = "mlir_codegen",
+    values = {"define": "codegen=mlir"},
+)
 
 py_binary(
     name = "gen_cpp",
@@ -24,7 +30,11 @@ cc_library(
         [
             "src/**/*.cpp",
         ],
-    ) + [
+        exclude = ["src/codegen/**"],
+    ) + select({
+        ":mlir_codegen": glob(["src/codegen/**/*.cpp"]),
+        "//conditions:default": [],
+    }) + [
         ":src/version.cpp",
     ],
     hdrs = glob(
@@ -42,6 +52,9 @@ cc_library(
     defines = select({
         "@platforms//cpu:wasm32": ["WASM_PLATFORM"],
         "//conditions:default": [],
+    }) + select({
+        ":mlir_codegen": ["PSCM_ENABLE_MLIR_CODEGEN"],
+        "//conditions:default": [],
     }),
     features = select({
         # only wasm need add exceptions explict
@@ -74,7 +87,51 @@ cc_library(
         # "@optional",
         # "@string-view-lite",
         # "@variant",
-    ],
+    ] + select({
+        ":mlir_codegen": [
+            ":pscm-ops-inc-gen",
+            "@llvm-project//llvm:Core",
+            "@llvm-project//llvm:OrcJIT",
+            "@llvm-project//llvm:Support",
+            "@llvm-project//mlir:AffineDialect",
+            "@llvm-project//mlir:AffineToStandard",
+            "@llvm-project//mlir:AffineTransforms",
+            "@llvm-project//mlir:AllPassesAndDialects",
+            "@llvm-project//mlir:Analysis",
+            "@llvm-project//mlir:ArithDialect",
+            "@llvm-project//mlir:ArithToLLVM",
+            "@llvm-project//mlir:BuiltinToLLVMIRTranslation",
+            "@llvm-project//mlir:CastInterfaces",
+            "@llvm-project//mlir:ControlFlowToLLVM",
+            "@llvm-project//mlir:ExecutionEngine",
+            "@llvm-project//mlir:ExecutionEngineUtils",
+            "@llvm-project//mlir:FuncDialect",
+            "@llvm-project//mlir:FuncExtensions",
+            "@llvm-project//mlir:FuncToLLVM",
+            "@llvm-project//mlir:IR",
+            "@llvm-project//mlir:LLVMCommonConversion",
+            "@llvm-project//mlir:LLVMDialect",
+            "@llvm-project//mlir:LLVMIRTransforms",
+            "@llvm-project//mlir:LLVMToLLVMIRTranslation",
+            "@llvm-project//mlir:MemRefDialect",
+            "@llvm-project//mlir:MemRefToLLVM",
+            "@llvm-project//mlir:Parser",
+            "@llvm-project//mlir:Pass",
+            "@llvm-project//mlir:SCFDialect",
+            "@llvm-project//mlir:SCFToControlFlow",
+            "@llvm-project//mlir:SideEffectInterfaces",
+            "@llvm-project//mlir:Support",
+            "@llvm-project//mlir:ToLLVMIRTranslation",
+            "@llvm-project//mlir:TransformUtils",
+            "@llvm-project//mlir:Transforms",
+
+            # clang
+            "@llvm-project//clang:driver",
+            "@llvm-project//clang:frontend",
+            "@llvm-project//clang:tooling",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 cc_binary(
@@ -85,3 +142,45 @@ cc_binary(
 )
 
 collect_pscm_tests()
+
+td_library(
+    name = "pscm-ops-td-files",
+    srcs = [
+        "include/pscm/codegen/mlir/Ops.td",
+    ],
+    includes = ["include"],
+    tags = ["manual"],
+    deps = [
+        "@llvm-project//mlir:CallInterfacesTdFiles",
+        "@llvm-project//mlir:CastInterfacesTdFiles",
+        "@llvm-project//mlir:FunctionInterfacesTdFiles",
+        "@llvm-project//mlir:OpBaseTdFiles",
+        "@llvm-project//mlir:SideEffectInterfacesTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "pscm-ops-inc-gen",
+    tags = ["manual"],
+    tbl_outs = [
+        (
+            ["-gen-op-decls"],
+            "Ops.h.inc",
+        ),
+        (
+            ["-gen-op-defs"],
+            "Ops.cpp.inc",
+        ),
+        (
+            ["-gen-dialect-decls"],
+            "Dialect.h.inc",
+        ),
+        (
+            ["-gen-dialect-defs"],
+            "Dialect.cpp.inc",
+        ),
+    ],
+    tblgen = "@llvm-project//mlir:mlir-tblgen",
+    td_file = "include/pscm/codegen/mlir/Ops.td",
+    deps = [":pscm-ops-td-files"],
+)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,18 +1,22 @@
 cmake_minimum_required(VERSION 3.26)
 SET(CMAKE_OSX_DEPLOYMENT_TARGET "13.3" CACHE STRING "Minimum OS X deployment version" FORCE)
-project(pscm VERSION 0.3.0 LANGUAGES CXX)
+project(pscm VERSION 0.3.0 LANGUAGES C CXX)
 option(PSCM_USE_CXX20_MODULES "enable c++20 modules" OFF)
+option(PSCM_ENABLE_MLIR_CODEGEN "enable codegen with MLIR" OFF)
 if (WIN32)
     set(CMAKE_CXX_STANDARD 20)
 else ()
     if (PSCM_USE_CXX20_MODULES)
         if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-            # CMake 3.26
-            # set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "2182bf5c-ef0d-489a-91da-49dbc3090d2a")
-            # CMake 3.27
-            set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "aa1f7df0-828a-4fcd-9afc-2dc80491aca7")
-            # set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "ac01f462-0f5f-432a-86aa-acef252918a6")
-            include(cxx_modules_rules_clang.cmake)
+            if (${CMAKE_MINOR_VERSION} STREQUAL 26)
+                set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "2182bf5c-ef0d-489a-91da-49dbc3090d2a")
+                include(cxx_modules_rules_clang.cmake)
+            elseif (${CMAKE_MINOR_VERSION} STREQUAL 27)
+                set(CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API "aa1f7df0-828a-4fcd-9afc-2dc80491aca7")
+            else ()
+                # https://cmake.org/cmake/help/latest/policy/CMP0155.html#policy:CMP0155
+                cmake_policy(SET CMP0155 NEW)
+            endif ()
             set(CMAKE_CXX_STANDARD 20)
             set(CMAKE_CXX_EXTENSIONS OFF)
         else ()
@@ -27,7 +31,7 @@ endif ()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-add_library(pscm)
+add_library(pscm STATIC)
 if (EMSCRIPTEN)
     target_link_options(pscm PUBLIC "-sUSE_ICU=1")
     target_compile_options(pscm PUBLIC "-sUSE_ICU=1")
@@ -36,7 +40,7 @@ elseif (WIN32)
     target_link_libraries(pscm PUBLIC icu)
 else ()
     find_package(PkgConfig REQUIRED)
-    pkg_check_modules (ICU REQUIRED icu-i18n icu-uc icu-io IMPORTED_TARGET)
+    pkg_check_modules(ICU REQUIRED icu-i18n icu-uc icu-io IMPORTED_TARGET)
     target_link_libraries(pscm PUBLIC PkgConfig::ICU)
 endif ()
 
@@ -97,6 +101,54 @@ endif ()
 
 file(GLOB PSCM_SRCS "src/*.cpp")
 target_sources(pscm PRIVATE ${PSCM_SRCS})
+if (PSCM_ENABLE_MLIR_CODEGEN)
+    message(STATUS "Enable codegen with MLIR")
+    target_compile_definitions(pscm PRIVATE PSCM_ENABLE_MLIR_CODEGEN)
+    file(GLOB CODEGEN_SRCS "src/codegen/*.cpp" src/codegen/mlir/*.cpp)
+    target_sources(pscm PRIVATE ${CODEGEN_SRCS})
+    set(CMAKE_MODULE_PATH /usr/local/opt/llvm/lib/cmake/clang /usr/local/opt/llvm/lib/cmake/lld /usr/local/opt/llvm/lib/cmake/llvm /usr/local/opt/llvm/lib/cmake/mlir)
+    set(CMAKE_PREFIX_PATH /usr/local/opt/llvm/lib/cmake/clang /usr/local/opt/llvm/lib/cmake/lld /usr/local/opt/llvm/lib/cmake/llvm /usr/local/opt/llvm/lib/cmake/mlir)
+    find_package(LLVM REQUIRED)
+    find_package(MLIR REQUIRED)
+    find_package(Clang REQUIRED)
+    include(TableGen)
+    include(AddMLIR)
+    include_directories(/usr/local/opt/llvm/include)
+    set(LLVM_TARGET_DEFINITIONS include/pscm/codegen/mlir/Ops.td)
+    mlir_tablegen(generate/Ops.h.inc -gen-op-decls)
+    mlir_tablegen(generate/Ops.cpp.inc -gen-op-defs)
+    mlir_tablegen(generate/Dialect.h.inc -gen-dialect-decls)
+    mlir_tablegen(generate/Dialect.cpp.inc -gen-dialect-defs)
+    add_public_tablegen_target(pscm-inc-gen)
+    add_dependencies(pscm pscm-inc-gen)
+    get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+    get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
+    get_property(extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS)
+    target_link_libraries(pscm PRIVATE
+            ${dialect_libs}
+            ${conversion_libs}
+            ${extension_libs}
+
+            MLIRAnalysis
+            MLIRBuiltinToLLVMIRTranslation
+            MLIRCastInterfaces
+            MLIRCallInterfaces
+            MLIRExecutionEngine
+            MLIRLLVMToLLVMIRTranslation
+            MLIRMemRefDialect
+            MLIRIR
+            MLIRParser
+            MLIRPass
+            MLIRSideEffectInterfaces
+            MLIRSupport
+            MLIRTargetLLVMIRExport
+            MLIRTransforms
+
+            clangDriver
+            clangTooling
+            clangFrontend
+    )
+endif ()
 target_sources(pscm PRIVATE
         ${CMAKE_CURRENT_BINARY_DIR}/generate/src/version.cpp
         src/logger/Logger.cpp

diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
@@ -2,6 +2,7 @@ workspace(name = "dev_pscm")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
 new_git_repository(
     name = "cpp-linenoise",
@@ -237,3 +238,87 @@ git_repository(
     commit = "1c1933fa1ebadd6954fe9eff18e8bb0d018594ff",
     remote = "https://github.com/PikachuHy/icu.bazel.git",
 )
+
+new_git_repository(
+    name = "llvm-raw",
+    build_file_content = "# empty",
+    commit = "6009708b4367171ccdbf4b5905cb6a803753fe18",
+    remote = "https://github.com/llvm/llvm-project.git"
+)
+
+load("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure")
+
+llvm_configure(name = "llvm-project")
+
+maybe(
+    http_archive,
+    name = "llvm_zlib",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:zlib-ng.BUILD",
+    sha256 = "e36bb346c00472a1f9ff2a0a4643e590a254be6379da7cddd9daeb9a7f296731",
+    strip_prefix = "zlib-ng-2.0.7",
+    urls = [
+        "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.0.7.zip",
+    ],
+)
+
+maybe(
+    http_archive,
+    name = "vulkan_headers",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:vulkan_headers.BUILD",
+    sha256 = "19f491784ef0bc73caff877d11c96a48b946b5a1c805079d9006e3fbaa5c1895",
+    strip_prefix = "Vulkan-Headers-9bd3f561bcee3f01d22912de10bb07ce4e23d378",
+    urls = [
+        "https://github.com/KhronosGroup/Vulkan-Headers/archive/9bd3f561bcee3f01d22912de10bb07ce4e23d378.tar.gz",
+    ],
+)
+
+load("@llvm-raw//utils/bazel:vulkan_sdk.bzl", "vulkan_sdk_setup")
+
+maybe(
+    vulkan_sdk_setup,
+    name = "vulkan_sdk",
+)
+
+maybe(
+    http_archive,
+    name = "gmp",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:gmp.BUILD",
+    sha256 = "fd4829912cddd12f84181c3451cc752be224643e87fac497b69edddadc49b4f2",
+    strip_prefix = "gmp-6.2.1",
+    urls = [
+        "https://gmplib.org/download/gmp/gmp-6.2.1.tar.xz",
+        "https://ftp.gnu.org/gnu/gmp/gmp-6.2.1.tar.xz",
+    ],
+)
+
+# https://www.mpfr.org/mpfr-current/
+#
+# When updating to a newer version, don't use URLs with "mpfr-current" in them.
+# Instead, find a stable URL like the one used currently.
+maybe(
+    http_archive,
+    name = "mpfr",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:mpfr.BUILD",
+    sha256 = "9cbed5d0af0d9ed5e9f8dd013e17838eb15e1db9a6ae0d371d55d35f93a782a7",
+    strip_prefix = "mpfr-4.1.1",
+    urls = ["https://www.mpfr.org/mpfr-4.1.1/mpfr-4.1.1.tar.gz"],
+)
+
+maybe(
+    new_git_repository,
+    name = "pfm",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:pfm.BUILD",
+    remote = "https://git.code.sf.net/p/perfmon2/libpfm4",
+    tag = "v4.12.1",
+)
+
+maybe(
+    http_archive,
+    name = "llvm_zstd",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:zstd.BUILD",
+    sha256 = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
+    strip_prefix = "zstd-1.5.2",
+    urls = [
+        "https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz"
+    ],
+)
diff --git a/build.pscm b/build.pscm
@@ -31,7 +31,7 @@
 (cpp_library
  (name "pscm")
  (srcs
-  (glob "src/**/*.cpp" "src/**/*.cppm")
+  (glob "src/icu/*.cpp" "src/logger/*.cpp" "src/misc/*.cpp" "src/*.cpp" "src/**/*.cppm")
   "build/generate/src/version.cpp"
   )
  (hdrs

diff --git a/include/pscm/Cell.h b/include/pscm/Cell.h
@@ -149,6 +149,7 @@ class Cell {
   };
   UString to_string() const;
   UString pretty_string() const;
+  std::string to_std_string() const;
   void display(Port& port);
 
   static Cell nil() {

diff --git a/include/pscm/Scheme.h b/include/pscm/Scheme.h
@@ -46,6 +46,7 @@ class Scheme {
   std::vector<Module *> module_list_;
   std::unordered_map<Cell, Module *> module_map_;
   bool use_register_machine_;
+  bool use_mlir_ = true;
   bool in_repl_ = false;
   friend class SchemeProxy;
 };

diff --git a/include/pscm/codegen/codegen.h b/include/pscm/codegen/codegen.h
@@ -0,0 +1,7 @@
+#pragma once
+#include "pscm/Cell.h"
+#include <optional>
+
+namespace pscm {
+std::optional<Cell> mlir_codegen_and_run_jit(Cell expr);
+}