Merge pull request #2 from CharlieTap/kotlinx-benchmark

Kotlinx Benchmark

.github/workflows/benchmark.yml

@@ -0,0 +1,50 @@
+name: Benchmark ⌛
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        type: string
+        description: 'The machine runner the workflow should run on'
+        default: macos-latest
+        required: false
+  workflow_dispatch:
+    inputs:
+      runner:
+        type: string
+        description: 'The machine runner the workflow should run on'
+        default: macos-latest
+        required: true
+
+jobs:
+  build:
+    runs-on: ${{ inputs.runner }}
+    steps:
+
+      - name: Clone Repo
+        uses: actions/checkout@v4
+
+      - name: Set up jdk@21
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'corretto'
+          java-version: '21'
+
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
+      - name: Execute Gradle build
+        run: ./gradlew benchmark
+
+      - name: Move benchmark files
+        run: |
+          mkdir -p benchmark  # This command ensures that the 'benchmark' directory exists
+          find . -type f \( -iname '*benchmark.json' -o -iname '*benchmark.csv' \) -exec mv {} ./benchmark/ \;
+
+      - name: Commit benchmark files
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add benchmark/*.{json,csv}
+          git commit -m "Commit benchmark results" -a
+          git push

.github/workflows/pr.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - main
+    paths-ignore:
+      - 'benchmark/**'
 
 jobs:
   build:

cachemap/build.gradle.kts

@@ -3,14 +3,26 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 plugins {
     alias(libs.plugins.kotlin.multiplatform)
     alias(libs.plugins.kotlin.serialization)
+    alias(libs.plugins.kotlin.allopen)
     alias(libs.plugins.kotlin.atomic.fu)
+    alias(libs.plugins.kotlin.benchmark)
     alias(libs.plugins.kotlinter)
     id("maven-publish")
 }
 
 group = "com.tap.cachemap"
 version = libs.versions.version.name.get()
 
+allOpen {
+    annotation("org.openjdk.jmh.annotations.State")
+}
+
+benchmark {
+    targets {
+        register("jvmBenchmark")
+    }
+}
+
 
 kotlin {
 
@@ -28,7 +40,9 @@ kotlin {
     }
 
     targets {
-        jvm()
+        jvm {
+            val benchmark by compilations.creating
+        }
     }
 
     sourceSets {
@@ -37,6 +51,7 @@ kotlin {
             dependencies {
                 implementation(projects.leftright)
                 implementation(libs.kotlinx.atomic.fu)
+                implementation(libs.kotlinx.benchmark)
             }
         }
 
@@ -51,6 +66,10 @@ kotlin {
 
             }
         }
+
+        val jvmBenchmark by getting {
+            dependsOn(jvmMain)
+        }
     }
 }
 

cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/BenchmarkConfig.kt

@@ -0,0 +1,7 @@
+package com.tap.cachemap.benchmark
+
+object BenchmarkConfig {
+    const val WARMUP_ITERATIONS = 10
+    const val MEASUREMENT_ITERATIONS = 10
+    const val FORKS = 10
+}

cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/CacheMapMultiThreadedBenchmark.kt

@@ -0,0 +1,6 @@
+package com.tap.cachemap.benchmark
+
+import org.openjdk.jmh.annotations.Threads
+
+@Threads(Threads.MAX)
+class CacheMapMultiThreadedBenchmark : CacheMapSingleThreadBenchmark()

cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/CacheMapSingleThreadBenchmark.kt

@@ -0,0 +1,89 @@
+package com.tap.cachemap.benchmark
+import com.tap.cachemap.cacheMapOf
+import org.openjdk.jmh.annotations.Benchmark
+import org.openjdk.jmh.annotations.BenchmarkMode
+import org.openjdk.jmh.annotations.Fork
+import org.openjdk.jmh.annotations.Measurement
+import org.openjdk.jmh.annotations.Mode
+import org.openjdk.jmh.annotations.OutputTimeUnit
+import org.openjdk.jmh.annotations.Scope
+import org.openjdk.jmh.annotations.Setup
+import org.openjdk.jmh.annotations.State
+import org.openjdk.jmh.annotations.TearDown
+import org.openjdk.jmh.annotations.Warmup
+import org.openjdk.jmh.infra.Blackhole
+import java.util.concurrent.TimeUnit
+
+@State(Scope.Benchmark)
+@Fork(value = BenchmarkConfig.FORKS)
+@BenchmarkMode(Mode.AverageTime, Mode.Throughput)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = BenchmarkConfig.WARMUP_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = BenchmarkConfig.MEASUREMENT_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS)
+class CacheMapSingleThreadBenchmark {
+
+    private val cacheMap = cacheMapOf<String, String>()
+
+    @Setup
+    fun setup() {
+        for (i in 1..1000) {
+            cacheMap.put("key$i", "value$i")
+        }
+    }
+
+    @Benchmark
+    fun put(blackhole: Blackhole) {
+        val result = cacheMap.put("Hello", "World")
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun overwrite(blackhole: Blackhole) {
+        val result = cacheMap.put("key1", "value2")
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun putAll(blackhole: Blackhole) {
+        val anotherMap = mapOf("Hello" to "World", "SecondKey" to "SecondValue")
+        val result = cacheMap.putAll(anotherMap)
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun get(blackhole: Blackhole) {
+        val result: String? = cacheMap["key1"]
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun getMiss(blackhole: Blackhole) {
+        val result: String? = cacheMap["Hello"]
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun remove(blackhole: Blackhole) {
+        val result = cacheMap.remove("key1")
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun stressTest(blackhole: Blackhole) {
+        for (i in 1..1000) {
+            val putResult = cacheMap.put("newKey$i", "newValue$i")
+            blackhole.consume(putResult)
+
+            val getResult: String? = cacheMap["key$i"]
+            blackhole.consume(getResult)
+
+            val removeResult = cacheMap.remove("newKey$i")
+            blackhole.consume(removeResult)
+        }
+    }
+
+    @TearDown
+    fun tearDown() {
+        cacheMap.clear()
+    }
+}

cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/ConcurrentHashMapMultiThreadedBenchmark.kt

@@ -0,0 +1,6 @@
+package com.tap.cachemap.benchmark
+
+import org.openjdk.jmh.annotations.Threads
+
+@Threads(Threads.MAX)
+class ConcurrentHashMapMultiThreadedBenchmark : ConcurrentHashMapSingleThreadedBenchmark()

cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/ConcurrentHashMapSingleThreadedBenchmark.kt

@@ -0,0 +1,90 @@
+package com.tap.cachemap.benchmark
+
+import org.openjdk.jmh.annotations.Benchmark
+import org.openjdk.jmh.annotations.BenchmarkMode
+import org.openjdk.jmh.annotations.Fork
+import org.openjdk.jmh.annotations.Measurement
+import org.openjdk.jmh.annotations.Mode
+import org.openjdk.jmh.annotations.OutputTimeUnit
+import org.openjdk.jmh.annotations.Scope
+import org.openjdk.jmh.annotations.Setup
+import org.openjdk.jmh.annotations.State
+import org.openjdk.jmh.annotations.TearDown
+import org.openjdk.jmh.annotations.Warmup
+import org.openjdk.jmh.infra.Blackhole
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.TimeUnit
+
+@State(Scope.Benchmark)
+@Fork(value = BenchmarkConfig.FORKS)
+@BenchmarkMode(Mode.AverageTime, Mode.Throughput)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = BenchmarkConfig.WARMUP_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = BenchmarkConfig.MEASUREMENT_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS)
+class ConcurrentHashMapSingleThreadedBenchmark {
+
+    private val cacheMap = ConcurrentHashMap<String, String>()
+
+    @Setup
+    fun setup() {
+        for (i in 1..1000) {
+            cacheMap["key$i"] = "value$i"
+        }
+    }
+
+    @Benchmark
+    fun put(blackhole: Blackhole) {
+        val result = cacheMap.put("Hello", "World")
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun overwrite(blackhole: Blackhole) {
+        val result = cacheMap.put("key1", "value2")
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun putAll(blackhole: Blackhole) {
+        val anotherMap = mapOf("Hello" to "World", "SecondKey" to "SecondValue")
+        cacheMap.putAll(anotherMap)
+        blackhole.consume(anotherMap)
+    }
+
+    @Benchmark
+    fun get(blackhole: Blackhole) {
+        val result: String? = cacheMap["key1"]
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun getMiss(blackhole: Blackhole) {
+        val result: String? = cacheMap["Hello"]
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun remove(blackhole: Blackhole) {
+        val result = cacheMap.remove("key1")
+        blackhole.consume(result)
+    }
+
+    @Benchmark
+    fun stressTest(blackhole: Blackhole) {
+        for (i in 1..1000) {
+            val putResult = cacheMap.put("newKey$i", "newValue$i")
+            blackhole.consume(putResult)
+
+            val getResult: String? = cacheMap["key$i"]
+            blackhole.consume(getResult)
+
+            val removeResult = cacheMap.remove("newKey$i")
+            blackhole.consume(removeResult)
+        }
+    }
+
+    @TearDown
+    fun tearDown() {
+        cacheMap.clear()
+    }
+}

gradle/libs.versions.toml

@@ -29,6 +29,7 @@ kotlinter = "3.15.0"
 kot-compile-testing = "1.5.0"
 
 kotlinx-atomic-fu = "0.21.0"
+kotlinx-benchmark = "0.4.9"
 kotlinx-coroutines = "1.7.3"
 kotlinx-datetime = "0.4.0"
 kotlinx-serialization = "1.5.1"
@@ -51,7 +52,9 @@ android = { id = "com.android.application", version.ref = "android-build-tools-p
 android-lib = { id = "com.android.library", version.ref = "android-build-tools-plugin" }
 android-test = { id = "com.android.test", version.ref = "android-build-tools-plugin" }
 dependency-analysis = { id = "com.autonomousapps.dependency-analysis", version.ref="dependency-analysis" }
+kotlin-allopen = { id = "org.jetbrains.kotlin.plugin.allopen", version.ref = "kotlin" }
 kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" }
+kotlin-benchmark = { id = "org.jetbrains.kotlinx.benchmark", version.ref = "kotlinx-benchmark" }
 kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" }
 kotlin-kapt = { id = "org.jetbrains.kotlin.kapt", version.ref = "kotlin" }
 kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" }
@@ -84,6 +87,7 @@ kotlin-reflection = { module = "org.jetbrains.kotlin:kotlin-reflect", version.re
 kotlin-symbol-processing-api = { module = "com.google.devtools.ksp:symbol-processing-api", version.ref = "ksp"}
 
 kotlinx-atomic-fu = { module = "org.jetbrains.kotlinx:atomicfu", version.ref = "kotlinx-atomic-fu"}
+kotlinx-benchmark = { module = "org.jetbrains.kotlinx:kotlinx-benchmark-runtime", version.ref = "kotlinx-benchmark"}
 kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinx-datetime"}
 kotlinx-serialization = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization"}
 kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinx-coroutines"}