diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml new file mode 100644 index 00000000..18ec04e1 --- /dev/null +++ b/.github/workflows/benchmark_pr.yml @@ -0,0 +1,78 @@ +name: Benchmark a pull request + +on: + pull_request_target: + branches: + - master + +permissions: + pull-requests: write + +jobs: + generate_plots: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: "1.8" + - uses: julia-actions/cache@v1 + - name: Extract Package Name from Project.toml + id: extract-package-name + run: | + PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/') + echo "::set-output name=package_name::$PACKAGE_NAME" + - name: Build AirspeedVelocity + env: + JULIA_NUM_THREADS: 2 + run: | + # Lightweight build step, as sometimes the runner runs out of memory: + julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")' + julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")' + - name: Add ~/.julia/bin to PATH + run: | + echo "$HOME/.julia/bin" >> $GITHUB_PATH + - name: Run benchmarks + run: | + echo $PATH + ls -l ~/.julia/bin + mkdir results + benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune + - name: Create plots from benchmarks + run: | + mkdir -p plots + benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/ + - name: Upload plot as artifact + uses: actions/upload-artifact@v2 + with: + name: plots + path: plots + - name: Create markdown table from benchmarks + run: | + benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md + echo '### Benchmark Results' > body.md + echo '' >> body.md + echo '' >> body.md + cat table.md >> body.md + echo '' >> body.md + echo '' >> body.md + echo '### Benchmark Plots' >> body.md + echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md + echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md + + - name: Find Comment + uses: peter-evans/find-comment@v2 + id: fcbenchmark + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmark Results + + - name: Comment on PR + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ steps.fcbenchmark.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: body.md + edit-mode: replace diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 00000000..ae717853 --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,8 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +BitBasis = "50ba71b6-fa0f-514d-ae9a-0916efc90dcf" +LuxurySparse = "d05aeea4-b7d4-55ac-b691-9e7fabb07ba2" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl new file mode 100644 index 00000000..265a8772 --- /dev/null +++ b/benchmark/benchmarks.jl @@ -0,0 +1,42 @@ +module BenchmarkUtils + """ + Replace instances of, for example, `using YaoAPI` with `using Yao.YaoAPI`. + """ + const libs = (:YaoAPI, :YaoArrayRegister, :YaoBlocks, :YaoSym) + function replace_imports(ex) + return _replace_imports(ex) + end + function _replace_imports(ex) + if isa(ex, Expr) && ex.head == :using + foreach(ex.args) do e + if isa(e, Expr) && first(e.args) in libs + pushfirst!(e.args, :Yao) + end + end + elseif isa(ex, Expr) + map!(_replace_imports, ex.args, ex.args) + end + return ex + end +end + +module YaoArrayRegisterBenchmarks + using ..BenchmarkUtils: replace_imports + using Yao + using Yao.YaoBlocks: sprand_hermitian + include(replace_imports, "../lib/YaoArrayRegister/benchmark/benchmarks.jl") +end + +module YaoBlocksBenchmarks + using ..BenchmarkUtils: replace_imports + using Yao + include(replace_imports, "../lib/YaoBlocks/benchmark/benchmarks.jl") +end + +using BenchmarkTools +import .YaoArrayRegisterBenchmarks: SUITE as yarSUITE +import .YaoBlocksBenchmarks: SUITE as ybSUITE + +const SUITE = BenchmarkGroup() +SUITE["YaoArrayRegister"] = yarSUITE +SUITE["YaoBlocks"] = ybSUITE