diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2f05a26..43ed6e2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,9 +29,6 @@ jobs:
       fail-fast: false # hdiutil: create failed - Resource busy
       matrix:
         arch: [x86_64, arm64]
-        include:
-          - { arch: x86_64, homebrew_prefix: /usr/local }
-          - { arch: arm64, homebrew_prefix: /opt/homebrew }
 
     steps:
       - uses: actions/checkout@v4
@@ -44,17 +41,14 @@ jobs:
         with:
           python-version: '3.12'
 
-      - name: Install dependencies (x86)
-        if: ${{ matrix.arch == 'x86_64' }}
+      - name: Install dependencies
         run: |
           brew install \
-            expat \
-            fmt \
             extra-cmake-modules \
-            libxkbcommon \
+            xkeyboardconfig \
             iso-codes \
-            json-c \
             ninja
+          ./install-deps.sh ${{ matrix.arch }}
           pip install "dmgbuild[badge_icons]"
 
       - name: Setup arm homebrew
@@ -64,26 +58,15 @@ jobs:
       - name: Install dependencies (arm)
         if: ${{ matrix.arch == 'arm64' }}
         run: |
-          brew install \
-            extra-cmake-modules \
-            ninja
-          pip install "dmgbuild[badge_icons]"
           arm-brew-install \
-            expat \
-            fmt \
-            gettext \
             xkeyboardconfig \
-            libxkbcommon \
-            iso-codes \
-            json-c \
-            libuv
-          cp -f /usr/local/bin/msgfmt /opt/homebrew/bin
+            iso-codes
+          cp -f /usr/local/bin/msgfmt /tmp/fcitx5/bin
 
       - name: Build
         run: |
           cmake -B build -G Ninja \
             -DCMAKE_Swift_COMPILER=`which swiftc` \
-            -DCMAKE_FIND_ROOT_PATH=${{ matrix.homebrew_prefix }} \
             -DCMAKE_OSX_ARCHITECTURES=${{ matrix.arch }} \
             -DCMAKE_BUILD_TYPE=Release
           cmake --build build || ./fix-cross-build.sh
@@ -97,6 +80,10 @@ jobs:
           path: |
             Fcitx5-${{ matrix.arch }}.dmg
 
+      - name: Check validity
+        if: ${{ matrix.arch == 'x86_64' }}
+        run: ./check-validity.sh
+
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3
diff --git a/.gitignore b/.gitignore
index 8c30cb1..13c9cab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 .DS_Store
 build
 *.dmg
+*.tar.bz2
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c59e821..85dbd79 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -13,6 +13,7 @@ if(NOT CMAKE_OSX_ARCHITECTURES)
     set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}")
 endif()
 
+# Need HOMEBREW_PATH for iso-codes and xkeyboardconfig
 if(CMAKE_OSX_ARCHITECTURES STREQUAL arm64)
     add_definitions(-target "arm64-apple-macos11")
     set(HOMEBREW_PATH "/opt/homebrew")
@@ -20,7 +21,15 @@ else()
     set(HOMEBREW_PATH "/usr/local")
 endif()
 
-set(ENV{PKG_CONFIG_PATH} "${HOMEBREW_PATH}/lib/pkgconfig:${HOMEBREW_PATH}/share/pkgconfig:${HOMEBREW_PATH}/opt/expat/lib/pkgconfig")
+set(PREBUILT_INSTALL_PATH "/tmp/fcitx5")
+
+set(ENV{PKG_CONFIG_PATH} "${PREBUILT_INSTALL_PATH}/lib/pkgconfig:${HOMEBREW_PATH}/share/pkgconfig")
+
+# For dependencies not to be find via pkg-config
+set(LibIntl_DIR "${PREBUILT_INSTALL_PATH}/lib/cmake")
+find_package(LibIntl)
+set(fmt_DIR "${PREBUILT_INSTALL_PATH}/lib/cmake/fmt")
+find_package(fmt)
 
 option(ENABLE_TEST "" OFF)
 option(ENABLE_COVERAGE "" OFF)
diff --git a/README.md b/README.md
index 92051cb..e4d2791 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,8 @@ Cross build from Intel to Apple Silicon is performed in [CI](.github/workflows/c
 
 ### Install dependencies
 ```sh
-brew install cmake ninja extra-cmake-modules gettext expat fmt libuv libxkbcommon iso-codes json-c
+brew install cmake ninja extra-cmake-modules gettext iso-codes xkeyboardconfig
+./install-deps.sh
 ```
 
 ### Build with CMake
diff --git a/cache/README.md b/cache/README.md
new file mode 100644
index 0000000..d089126
--- /dev/null
+++ b/cache/README.md
@@ -0,0 +1,3 @@
+This directory caches tarball of build dependencies
+(downloaded from [prebuilder](https://github.com/fcitx-contrib/fcitx5-macos-prebuilder/releases)),
+which will be extracted to `/tmp/fcitx5`.
diff --git a/check-validity.sh b/check-validity.sh
new file mode 100755
index 0000000..9336f4e
--- /dev/null
+++ b/check-validity.sh
@@ -0,0 +1,18 @@
+set -e
+
+has_homebrew_deps=0
+
+cd /Library/Input\ Methods/Fcitx5.app/Contents
+libs=(MacOS/Fcitx5)
+libs+=($(ls lib/libFcitx5{Config,Core,Utils}.dylib))
+libs+=($(ls lib/fcitx5/*.so))
+libs+=(lib/fcitx5/libexec/comp-spell-dict)
+
+for lib in "${libs[@]}"; do
+  if otool -L $lib | grep /usr/local; then
+    otool -L $lib
+    has_homebrew_deps=1
+  fi
+done
+
+exit $has_homebrew_deps
diff --git a/install-deps.sh b/install-deps.sh
new file mode 100755
index 0000000..a1cb1ab
--- /dev/null
+++ b/install-deps.sh
@@ -0,0 +1,26 @@
+set -e
+
+if [[ -z $1 ]]; then
+  ARCH=`uname -m`
+else
+  ARCH=$1
+fi
+
+# This is the same with INSTALL_PREFIX of prebuilder
+INSTALL_PREFIX=/tmp/fcitx5
+mkdir -p $INSTALL_PREFIX
+
+deps=(
+  expat
+  fmt
+  libintl
+  json-c
+  libuv
+  libxkbcommon
+)
+
+for dep in "${deps[@]}"; do
+  file=$dep-$ARCH.tar.bz2
+  [[ -f cache/$file ]] || wget -P cache https://github.com/fcitx-contrib/fcitx5-macos-prebuilder/releases/download/latest/$file
+  tar xjvf cache/$file -C $INSTALL_PREFIX
+done