diff --git a/Rakefile b/Rakefile
index e0d033b5..7bd5fed0 100644
--- a/Rakefile
+++ b/Rakefile
@@ -6,12 +6,10 @@ require "minitest/test_task"
 Minitest::TestTask.create
 task default: [:test]
 
-task :diff => [:buildkite_config, :rails] do
+task :diff do
   diff = Buildkite::Config::Diff.compare
-  puts diff.to_s(:color)
-
   annotate = Buildkite::Config::Annotate.new(diff)
-  annotate.perform
+  puts annotate.plan
 end
 
 task :buildkite_config do
diff --git a/buildkite-config-initial-pipeline.yml b/buildkite-config-initial-pipeline.yml
index 8715a3a6..e361abca 100644
--- a/buildkite-config-initial-pipeline.yml
+++ b/buildkite-config-initial-pipeline.yml
@@ -17,24 +17,35 @@ steps:
       git config --global --add safe.directory /workdir
       bundle install
       bundle exec rake test
+    label: ":test_tube: buildkite-config: rake test"
     key: self-test
     plugins:
       - docker#v5.8.0:
           image: "ruby:latest"
-          propagate-environment: true
   - command: |
-      git config --global --add safe.directory /workdir
-      bundle install
-      bundle exec rake diff
+      git clone --depth=1 https://github.com/rails/buildkite-config tmp/buildkite-config
+      git clone --depth=1 https://github.com/rails/rails tmp/rails
+
+      sh -c "$$ANNOTATE_COMMAND" > tmp/annotate.diff
+      if [[ -z $(grep '[^[:space:]]' tmp/annotate.diff) ]]; then
+        echo "annotate.diff is empty"
+      else
+        echo "annotate.diff is not empty"
+        cat tmp/annotate.diff
+        printf '%b\n' "$(cat tmp/annotate.diff)" | buildkite-agent annotate --style warning
+      fi
+    label: ":writing_hand: buildkite-config: annotate diff"
+    env:
+      BUNDLE_FROZEN: true
+      ANNOTATE_COMMAND: >-
+        docker run --rm
+        -v "$$PWD":/app:ro -w /app
+        -v "$$PWD/tmp":/app/tmp:rw
+        -e BUNDLE_FROZEN
+        ruby:latest
+        ./pipeline-annotate
     key: diff
     depends_on: self-test
-    plugins:
-      - docker#v5.8.0:
-          image: "ruby:latest"
-          mount-buildkite-agent: true
-          propagate-environment: true
-          environment:
-            - "GITHUB_TOKEN"
   - trigger: "rails-ci"
     label: ":pipeline: Build Rails main with new config"
     depends_on: diff
diff --git a/lib/buildkite_config/annotate.rb b/lib/buildkite_config/annotate.rb
index 73bcc503..4cf87901 100644
--- a/lib/buildkite_config/annotate.rb
+++ b/lib/buildkite_config/annotate.rb
@@ -4,32 +4,21 @@ def initialize(diff)
       @diff = diff
     end
 
-    def perform
+    def plan
       return if @diff.to_s.empty?
 
-      io = IO.popen("buildkite-agent annotate --style warning '#{plan}'")
-      output = io.read
-      io.close
+      <<~PLAN
+        ### :writing_hand: buildkite-config/plan
 
-      raise output unless $?.success?
+        <details>
+        <summary>Show Output</summary>
 
-      output
-    end
-
-    private
-      def plan
-        <<~PLAN
-          ### :writing_hand: buildkite-config/plan
-
-          <details>
-          <summary>Show Output</summary>
+        ```term
+        #{@diff.to_s(:color)}
+        ```
 
-          ```term
-          #{@diff.to_s(:color)}
-          ```
-
-          </details>
-        PLAN
-      end
+        </details>
+      PLAN
+    end
   end
 end
\ No newline at end of file
diff --git a/pipeline-annotate b/pipeline-annotate
new file mode 100755
index 00000000..6b7d2414
--- /dev/null
+++ b/pipeline-annotate
@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+def run(cmd, value=false)
+  io = IO.popen(cmd)
+  output = io.read
+  io.close
+
+  raise output unless $?.success?
+
+  return output if value
+end
+
+run "bundle install"
+
+annotation = run "bundle exec rake diff", true
+
+puts annotation
\ No newline at end of file