Achieve better results with compiler flags, use median for benchmarks (…

…#22)
domsleee · Mar 18, 2024 · 7628043 · 7628043
1 parent 02afc1d
commit 7628043
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 18 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -38,3 +38,4 @@ lto = true
 strip = true
 opt-level = "z"
 codegen-units = 1
+panic = "abort"
diff --git a/benchmark/all.md b/benchmark/all.md
@@ -1,22 +1,22 @@
 # All results
 
-file size: 13297664
-pwsh version: 7.4.0-rc.1
+file size: 9488384
+pwsh version: 7.5.0-preview.2
 ## Init
 
-| Command                                                      |   Mean [ms] | Min [ms] | Max [ms] |    Relative |
-| :----------------------------------------------------------- | ----------: | -------: | -------: | ----------: |
-| `pwsh -NoProfile -File ./../profiles/ProfileBaseline.ps1`    | 207.6 ± 6.7 |    186.8 |    233.2 |        1.00 |
-| `pwsh -NoProfile -File ./../profiles/ProfilePoshGit.ps1`     | 487.7 ± 8.0 |    478.1 |    524.3 | 2.35 ± 0.08 |
-| `pwsh -NoProfile -File ./../profiles/ProfileTabComplete.ps1` | 253.4 ± 6.4 |    247.2 |    284.2 | 1.22 ± 0.05 |
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+|:---|---:|---:|---:|---:|
+| `pwsh -NoProfile -File ./../profiles/ProfileBaseline.ps1` | 204.9 ± 3.8 | 201.7 | 220.4 | 1.00 |
+| `pwsh -NoProfile -File ./../profiles/ProfilePoshGit.ps1` | 473.5 ± 5.1 | 466.0 | 499.5 | 2.31 ± 0.05 |
+| `pwsh -NoProfile -File ./../profiles/ProfileTabComplete.ps1` | 246.5 ± 2.5 | 242.6 | 262.1 | 1.20 ± 0.03 |
 
-posh-tabcomplete: 45ms, posh-git: 280ms (6.22x faster)
+posh-tabcomplete: 41ms, posh-git: 268ms (6.54x faster)
 ## Complete
 
-| Command                                         |    Mean [ms] | Min [ms] | Max [ms] |    Relative |
-| :---------------------------------------------- | -----------: | -------: | -------: | ----------: |
-| `pwsh -NoProfile -File CompleteBaseline.ps1`    | 523.9 ± 11.8 |    510.4 |    584.0 |        1.00 |
-| `pwsh -NoProfile -File CompletePoshGit.ps1`     | 746.5 ± 28.0 |    721.4 |    843.8 | 1.42 ± 0.06 |
-| `pwsh -NoProfile -File CompleteTabComplete.ps1` | 628.0 ± 19.5 |    609.1 |    736.2 | 1.20 ± 0.05 |
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+|:---|---:|---:|---:|---:|
+| `pwsh -NoProfile -File CompleteBaseline.ps1` | 503.4 ± 7.1 | 495.3 | 565.0 | 1.00 |
+| `pwsh -NoProfile -File CompletePoshGit.ps1` | 683.2 ± 5.7 | 675.8 | 708.0 | 1.36 ± 0.02 |
+| `pwsh -NoProfile -File CompleteTabComplete.ps1` | 586.7 ± 4.3 | 580.7 | 607.7 | 1.17 ± 0.02 |
 
-posh-tabcomplete: 104ms, posh-git: 222ms (2.13x faster)
+posh-tabcomplete: 83ms, posh-git: 179ms (2.16x faster)
diff --git a/benchmark/complete/benchmark_complete.ps1 b/benchmark/complete/benchmark_complete.ps1
@@ -1,5 +1,6 @@
 
 . $PSScriptRoot/../setupErrorHandling.ps1
+. $PSScriptRoot/../util.ps1
 $childRepoDir = "$PSScriptRoot/repo"
 if (Test-Path "$childRepoDir") {
     Remove-Item -r -fo "$childRepoDir"
@@ -19,13 +20,12 @@ git commit -m "init"
 Set-Location "$PSScriptRoot"
 hyperfine `
     --warmup 3 `
-    --runs 75 `
+    --runs (GetNumRuns) `
     -L script CompleteBaseline.ps1,CompletePoshGit.ps1,CompleteTabComplete.ps1 `
     "pwsh -NoProfile -File {script}" `
     --export-markdown $PSScriptRoot/complete.md `
     --export-csv $PSScriptRoot/complete.csv
 
-. $PSScriptRoot/../util.ps1
 $csv = "$PSScriptRoot/complete.csv"
 $tabMs = GetMs $csv "CompleteBaseline.ps1" "CompleteTabComplete.ps1";
 $poshMs = GetMs $csv "CompleteBaseline.ps1" "CompletePoshGit.ps1";

diff --git a/benchmark/init/benchmark_init.ps1 b/benchmark/init/benchmark_init.ps1
@@ -3,7 +3,7 @@
 Set-Location $PSScriptRoot
 hyperfine `
     --warmup 3 `
-    --runs 75 `
+    --runs (GetNumRuns) `
     -L profile ./../profiles/ProfileBaseline.ps1,./../profiles/ProfilePoshGit.ps1,./../profiles/ProfileTabComplete.ps1 `
     "pwsh -NoProfile -File {profile}" `
     --export-markdown init.md `

diff --git a/benchmark/util.ps1 b/benchmark/util.ps1
@@ -2,11 +2,15 @@ function GetMs {
     param ([string] $csv, [string] $baselineFile, [string] $profileFile)
     $baselineEntry = Import-Csv $csv | Where-Object { $_.command.Contains($baselineFile) } | Select-Object -First 1
     $entry = Import-Csv $csv | Where-Object { $_.command.Contains($profileFile) } | Select-Object -First 1
-    $seconds = $entry.mean - $baselineEntry.mean
+    $seconds = $entry.median - $baselineEntry.median
     return [math]::Round([timespan]::fromseconds($seconds).Milliseconds, 2)
 }
 
 function GetSummary {
     param ([double] $poshMs, [double] $tabMs)
     return "posh-tabcomplete: ${tabMs}ms, posh-git: ${poshMs}ms ($([math]::Round($poshMs / $tabMs, 2))x faster)"
+}
+
+function GetNumRuns {
+    return 150;
 }