diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..27943ff --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,50 @@ +name: Benchmark ⌛ + +on: + workflow_call: + inputs: + runner: + type: string + description: 'The machine runner the workflow should run on' + default: macos-latest + required: false + workflow_dispatch: + inputs: + runner: + type: string + description: 'The machine runner the workflow should run on' + default: macos-latest + required: true + +jobs: + build: + runs-on: ${{ inputs.runner }} + steps: + + - name: Clone Repo + uses: actions/checkout@v4 + + - name: Set up jdk@21 + uses: actions/setup-java@v3 + with: + distribution: 'corretto' + java-version: '21' + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Execute Gradle build + run: ./gradlew benchmark + + - name: Move benchmark files + run: | + mkdir -p benchmark # This command ensures that the 'benchmark' directory exists + find . -type f \( -iname '*benchmark.json' -o -iname '*benchmark.csv' \) -exec mv {} ./benchmark/ \; + + - name: Commit benchmark files + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add benchmark/*.{json,csv} + git commit -m "Commit benchmark results" -a + git push diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cefec4b..4fb40d6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -4,6 +4,8 @@ on: pull_request: branches: - main + paths-ignore: + - 'benchmark/**' jobs: build: diff --git a/cachemap/build.gradle.kts b/cachemap/build.gradle.kts index e8cc761..e301457 100644 --- a/cachemap/build.gradle.kts +++ b/cachemap/build.gradle.kts @@ -3,7 +3,9 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { alias(libs.plugins.kotlin.multiplatform) alias(libs.plugins.kotlin.serialization) + alias(libs.plugins.kotlin.allopen) alias(libs.plugins.kotlin.atomic.fu) + alias(libs.plugins.kotlin.benchmark) alias(libs.plugins.kotlinter) id("maven-publish") } @@ -11,6 +13,16 @@ plugins { group = "com.tap.cachemap" version = libs.versions.version.name.get() +allOpen { + annotation("org.openjdk.jmh.annotations.State") +} + +benchmark { + targets { + register("jvmBenchmark") + } +} + kotlin { @@ -28,7 +40,9 @@ kotlin { } targets { - jvm() + jvm { + val benchmark by compilations.creating + } } sourceSets { @@ -37,6 +51,7 @@ kotlin { dependencies { implementation(projects.leftright) implementation(libs.kotlinx.atomic.fu) + implementation(libs.kotlinx.benchmark) } } @@ -51,6 +66,10 @@ kotlin { } } + + val jvmBenchmark by getting { + dependsOn(jvmMain) + } } } diff --git a/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/BenchmarkConfig.kt b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/BenchmarkConfig.kt new file mode 100644 index 0000000..d136fc8 --- /dev/null +++ b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/BenchmarkConfig.kt @@ -0,0 +1,7 @@ +package com.tap.cachemap.benchmark + +object BenchmarkConfig { + const val WARMUP_ITERATIONS = 10 + const val MEASUREMENT_ITERATIONS = 10 + const val FORKS = 10 +} diff --git a/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/CacheMapMultiThreadedBenchmark.kt b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/CacheMapMultiThreadedBenchmark.kt new file mode 100644 index 0000000..0017c33 --- /dev/null +++ b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/CacheMapMultiThreadedBenchmark.kt @@ -0,0 +1,6 @@ +package com.tap.cachemap.benchmark + +import org.openjdk.jmh.annotations.Threads + +@Threads(Threads.MAX) +class CacheMapMultiThreadedBenchmark : CacheMapSingleThreadBenchmark() diff --git a/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/CacheMapSingleThreadBenchmark.kt b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/CacheMapSingleThreadBenchmark.kt new file mode 100644 index 0000000..aa618f8 --- /dev/null +++ b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/CacheMapSingleThreadBenchmark.kt @@ -0,0 +1,89 @@ +package com.tap.cachemap.benchmark +import com.tap.cachemap.cacheMapOf +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.TearDown +import org.openjdk.jmh.annotations.Warmup +import org.openjdk.jmh.infra.Blackhole +import java.util.concurrent.TimeUnit + +@State(Scope.Benchmark) +@Fork(value = BenchmarkConfig.FORKS) +@BenchmarkMode(Mode.AverageTime, Mode.Throughput) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = BenchmarkConfig.WARMUP_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = BenchmarkConfig.MEASUREMENT_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS) +class CacheMapSingleThreadBenchmark { + + private val cacheMap = cacheMapOf() + + @Setup + fun setup() { + for (i in 1..1000) { + cacheMap.put("key$i", "value$i") + } + } + + @Benchmark + fun put(blackhole: Blackhole) { + val result = cacheMap.put("Hello", "World") + blackhole.consume(result) + } + + @Benchmark + fun overwrite(blackhole: Blackhole) { + val result = cacheMap.put("key1", "value2") + blackhole.consume(result) + } + + @Benchmark + fun putAll(blackhole: Blackhole) { + val anotherMap = mapOf("Hello" to "World", "SecondKey" to "SecondValue") + val result = cacheMap.putAll(anotherMap) + blackhole.consume(result) + } + + @Benchmark + fun get(blackhole: Blackhole) { + val result: String? = cacheMap["key1"] + blackhole.consume(result) + } + + @Benchmark + fun getMiss(blackhole: Blackhole) { + val result: String? = cacheMap["Hello"] + blackhole.consume(result) + } + + @Benchmark + fun remove(blackhole: Blackhole) { + val result = cacheMap.remove("key1") + blackhole.consume(result) + } + + @Benchmark + fun stressTest(blackhole: Blackhole) { + for (i in 1..1000) { + val putResult = cacheMap.put("newKey$i", "newValue$i") + blackhole.consume(putResult) + + val getResult: String? = cacheMap["key$i"] + blackhole.consume(getResult) + + val removeResult = cacheMap.remove("newKey$i") + blackhole.consume(removeResult) + } + } + + @TearDown + fun tearDown() { + cacheMap.clear() + } +} diff --git a/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/ConcurrentHashMapMultiThreadedBenchmark.kt b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/ConcurrentHashMapMultiThreadedBenchmark.kt new file mode 100644 index 0000000..d0c3b8b --- /dev/null +++ b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/ConcurrentHashMapMultiThreadedBenchmark.kt @@ -0,0 +1,6 @@ +package com.tap.cachemap.benchmark + +import org.openjdk.jmh.annotations.Threads + +@Threads(Threads.MAX) +class ConcurrentHashMapMultiThreadedBenchmark : ConcurrentHashMapSingleThreadedBenchmark() diff --git a/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/ConcurrentHashMapSingleThreadedBenchmark.kt b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/ConcurrentHashMapSingleThreadedBenchmark.kt new file mode 100644 index 0000000..4848115 --- /dev/null +++ b/cachemap/src/jvmBenchmark/kotlin/com/tap/cachemap/benchmark/ConcurrentHashMapSingleThreadedBenchmark.kt @@ -0,0 +1,90 @@ +package com.tap.cachemap.benchmark + +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.TearDown +import org.openjdk.jmh.annotations.Warmup +import org.openjdk.jmh.infra.Blackhole +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit + +@State(Scope.Benchmark) +@Fork(value = BenchmarkConfig.FORKS) +@BenchmarkMode(Mode.AverageTime, Mode.Throughput) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = BenchmarkConfig.WARMUP_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = BenchmarkConfig.MEASUREMENT_ITERATIONS, time = 1, timeUnit = TimeUnit.SECONDS) +class ConcurrentHashMapSingleThreadedBenchmark { + + private val cacheMap = ConcurrentHashMap() + + @Setup + fun setup() { + for (i in 1..1000) { + cacheMap["key$i"] = "value$i" + } + } + + @Benchmark + fun put(blackhole: Blackhole) { + val result = cacheMap.put("Hello", "World") + blackhole.consume(result) + } + + @Benchmark + fun overwrite(blackhole: Blackhole) { + val result = cacheMap.put("key1", "value2") + blackhole.consume(result) + } + + @Benchmark + fun putAll(blackhole: Blackhole) { + val anotherMap = mapOf("Hello" to "World", "SecondKey" to "SecondValue") + cacheMap.putAll(anotherMap) + blackhole.consume(anotherMap) + } + + @Benchmark + fun get(blackhole: Blackhole) { + val result: String? = cacheMap["key1"] + blackhole.consume(result) + } + + @Benchmark + fun getMiss(blackhole: Blackhole) { + val result: String? = cacheMap["Hello"] + blackhole.consume(result) + } + + @Benchmark + fun remove(blackhole: Blackhole) { + val result = cacheMap.remove("key1") + blackhole.consume(result) + } + + @Benchmark + fun stressTest(blackhole: Blackhole) { + for (i in 1..1000) { + val putResult = cacheMap.put("newKey$i", "newValue$i") + blackhole.consume(putResult) + + val getResult: String? = cacheMap["key$i"] + blackhole.consume(getResult) + + val removeResult = cacheMap.remove("newKey$i") + blackhole.consume(removeResult) + } + } + + @TearDown + fun tearDown() { + cacheMap.clear() + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 53fc544..06ed0c3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,6 +29,7 @@ kotlinter = "3.15.0" kot-compile-testing = "1.5.0" kotlinx-atomic-fu = "0.21.0" +kotlinx-benchmark = "0.4.9" kotlinx-coroutines = "1.7.3" kotlinx-datetime = "0.4.0" kotlinx-serialization = "1.5.1" @@ -51,7 +52,9 @@ android = { id = "com.android.application", version.ref = "android-build-tools-p android-lib = { id = "com.android.library", version.ref = "android-build-tools-plugin" } android-test = { id = "com.android.test", version.ref = "android-build-tools-plugin" } dependency-analysis = { id = "com.autonomousapps.dependency-analysis", version.ref="dependency-analysis" } +kotlin-allopen = { id = "org.jetbrains.kotlin.plugin.allopen", version.ref = "kotlin" } kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } +kotlin-benchmark = { id = "org.jetbrains.kotlinx.benchmark", version.ref = "kotlinx-benchmark" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } kotlin-kapt = { id = "org.jetbrains.kotlin.kapt", version.ref = "kotlin" } kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } @@ -84,6 +87,7 @@ kotlin-reflection = { module = "org.jetbrains.kotlin:kotlin-reflect", version.re kotlin-symbol-processing-api = { module = "com.google.devtools.ksp:symbol-processing-api", version.ref = "ksp"} kotlinx-atomic-fu = { module = "org.jetbrains.kotlinx:atomicfu", version.ref = "kotlinx-atomic-fu"} +kotlinx-benchmark = { module = "org.jetbrains.kotlinx:kotlinx-benchmark-runtime", version.ref = "kotlinx-benchmark"} kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinx-datetime"} kotlinx-serialization = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization"} kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinx-coroutines"}