diff --git a/CHANGELOG.md b/CHANGELOG.md index adcfc672..2c6cf7b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Added +- Allow :kaocha.plugin.randomize/randomize? to be set at any test level. + ## Fixed ## Changed @@ -855,4 +857,4 @@ namespace. - The configuration format has changed, you should now start with the `#kaocha {}` tagged reader literal in `tests.edn` to provide defaults. If you want more control then overwrite `tests.edn` with the output of `--print-config` and - tweak. \ No newline at end of file + tweak. diff --git a/doc/03_configuration.md b/doc/03_configuration.md index e770cb91..30612b78 100644 --- a/doc/03_configuration.md +++ b/doc/03_configuration.md @@ -216,7 +216,7 @@ bin/kaocha --plugin profiling Some plugins are needed for the normal functioning of Kaocha. These are added automatically when using the `#kaocha/v1 {}` reader literal. They are -- `:kaocha.plugin/randomize`: randomize test order +- `:kaocha.plugin/randomize`: randomize test order ([documentation](08_plugins.md#randomize)) - `:kaocha.plugin/filter`: allow filtering and "focusing" of tests - `:kaocha.plugin/capture-output`: implements output capturing during tests diff --git a/doc/08_plugins.md b/doc/08_plugins.md index d297c336..027df62f 100644 --- a/doc/08_plugins.md +++ b/doc/08_plugins.md @@ -180,3 +180,54 @@ To customize which keys to print, use Kaocha's "bindings" functionality, in `tes ``` clojure :kaocha/bindings {kaocha.plugin.debug/*keys* [,,,]} ``` + +## Randomize + +The randomize plugin picks a random seed during the `config` hook and uses that +seed to randomize the order of test suites, namespaces, and test vars during the +`post-load` hook. + +Randomization can be toggled at the following places. + +1. top level config +2. test suite config +3. namespace metadata + +At each level, changing the randomize value will override the previously set +value. For example, you could: + +- randomize by default (top level `:kaocha.plugin.randomize/randomize? true`, + which is the default) +- exclude a specific test suite by setting `:kaocha.plugin.randomize/randomize? false` + on the suite in tests.edn +- but randomize a single namespace within that suite `(ns ^{:kaocha.plugin.randomize/randomize? true} ...)` + +Or the inverse (case 2) + +- turn it off at the top level +- but turn it on for certain tests suites +- with the exception of certain namespaces + +Passing the `--no-randomize` CLI flag will force all randomization to be disabled. + +### Enabling + +The randomize plugin is enabled by default. Enable or disable randomization by +setting the `:kaocha.plugin.randomize/randomize?` key in tests.edn at the top-level, +in a specific test suite, or on the metadata of a namespace. + +### Plugin-specific command line flags + +```shell +--[no-]randomize Run test namespaces and vars in random order. +--seed SEED Provide a seed to determine the random order of tests. +``` + +### Plugin-specific configuration options + +Shown with their default values: + +```clojure +#kaocha/v1 +{:kaocha.plugin.randomize/randomize? true} +``` diff --git a/src/kaocha/plugin/randomize.clj b/src/kaocha/plugin/randomize.clj index 1ee611ff..af7a809e 100644 --- a/src/kaocha/plugin/randomize.clj +++ b/src/kaocha/plugin/randomize.clj @@ -7,25 +7,41 @@ (let [rng (java.util.Random. seed)] (fn [& _] (.nextInt rng)))) -(defn straight-sort [test-plan] - (if-let [tests (:kaocha.test-plan/tests test-plan)] - (assoc test-plan - :kaocha.test-plan/tests - (->> tests - (sort-by :kaocha.testable/id) - (map straight-sort))) - test-plan)) +(defmacro or-some + ([] nil) + ([x] x) + ([x & next] + `(let [or# ~x] + (if (some? or#) or# (or-some ~@next))))) + +(defn get-randomize + "Returns the randomize? value for the test plan. Metadata takes precedence + over test-plan." + [test-plan] + (or-some + (get-in test-plan [:kaocha.testable/meta ::randomize?]) + (::randomize? test-plan))) -(defn rng-sort [rng test-plan] +(defn rng-sort [rng parent-randomize? test-plan] (if-let [tests (:kaocha.test-plan/tests test-plan)] (assoc test-plan :kaocha.test-plan/tests - (->> tests - (map #(assoc % ::sort-key (rng))) - (sort-by ::sort-key) - (map (partial rng-sort rng)))) + (let [randomize? (or-some (get-randomize test-plan) parent-randomize?)] + (map (partial rng-sort rng randomize?) + (if randomize? + (->> tests + (map #(assoc % ::sort-key (rng))) + (sort-by ::sort-key)) + tests)))) test-plan)) +(defn uses-randomize? + "Returns true if the test-plan uses ::randomize? at any test level." + [test-plan] + (if (get-randomize test-plan) + true + (some uses-randomize? (:kaocha.test-plan/tests test-plan)))) + (defplugin kaocha.plugin/randomize (cli-options [opts] (conj opts @@ -34,25 +50,24 @@ :parse-fn #(Integer/parseInt %)])) (config [config] - (let [randomize? (get-in config [:kaocha/cli-options :randomize]) - seed (get-in config [:kaocha/cli-options :seed]) - config (merge {::randomize? true} - config - (when (some? randomize?) - {::randomize? randomize?}))] - (if (::randomize? config) + (let [randomize? (get-in config [:kaocha/cli-options :randomize]) + no-randomize? (= false randomize?) + seed (get-in config [:kaocha/cli-options :seed]) + config (merge {::randomize? true} + config + (when (some? randomize?) + {::randomize? randomize?}))] + (if (not no-randomize?) (merge {::seed (or seed (rand-int Integer/MAX_VALUE))} config) config))) (post-load [test-plan] - (if (::randomize? test-plan) - (let [rng (rng (::seed test-plan))] - (->> test-plan - straight-sort - (rng-sort rng))) + (if-let [seed (::seed test-plan)] + (let [rng (rng seed)] + (rng-sort rng (get-randomize test-plan) test-plan)) test-plan)) (post-run [test-plan] - (if (and (::randomize? test-plan) (result/failed? test-plan)) + (if (and (::seed test-plan) (uses-randomize? test-plan) (result/failed? test-plan)) (print "\nRandomized with --seed" (::seed test-plan))) test-plan)) diff --git a/test/features/command_line/print_config.feature b/test/features/command_line/print_config.feature index 902c7a63..36e2dbdc 100644 --- a/test/features/command_line/print_config.feature +++ b/test/features/command_line/print_config.feature @@ -13,18 +13,17 @@ Feature: CLI: Print the Kaocha configuration When I run `bin/kaocha --print-config` Then the output should contain: """ clojure - {:kaocha.plugin.randomize/randomize? false, - :kaocha/reporter [kaocha.report/dots], + :kaocha.plugin.randomize/randomize? false, + """ + And the output should contain: + """ clojure + :kaocha/reporter [kaocha.report/dots] + """ + And the output should contain: + """ clojure :kaocha/color? false, - :kaocha/fail-fast? false, """ And the output should contain: """ clojure - :kaocha/tests - [{:kaocha.testable/type :kaocha.type/clojure.test, - :kaocha.testable/id :unit, - :kaocha/ns-patterns ["-test$"], - :kaocha/source-paths ["src"], - :kaocha/test-paths ["test"], - :kaocha.filter/skip-meta [:kaocha/skip]}], + :kaocha/fail-fast? false, """ diff --git a/test/unit/kaocha/plugin/randomize_test.clj b/test/unit/kaocha/plugin/randomize_test.clj index 6dd6bc3c..1cec12fb 100644 --- a/test/unit/kaocha/plugin/randomize_test.clj +++ b/test/unit/kaocha/plugin/randomize_test.clj @@ -2,7 +2,8 @@ (:require [clojure.test :refer :all] [kaocha.test-helper :refer :all] [kaocha.plugin :as plugin] - [kaocha.testable :as testable])) + [kaocha.testable :as testable] + [kaocha.plugin.randomize :as randomize])) (def plugin-chain (plugin/register :kaocha.plugin/randomize [])) @@ -14,6 +15,26 @@ :kaocha/test-paths ["fixtures/c-tests"] :kaocha/ns-patterns [".*"]}) +(deftest rng-sort-test + (let [rng (randomize/rng 123)] + (is (= {} + (randomize/rng-sort rng true + {}))) + (is (= {:kaocha.test-plan/tests [{:kaocha.plugin.randomize/sort-key -1188957731}]} + (randomize/rng-sort rng true + {:kaocha.test-plan/tests [{}]}))) + (is (= {:kaocha.test-plan/tests [{}]} + (randomize/rng-sort rng false + {:kaocha.test-plan/tests [{}]}))) + (is (match? {:kaocha.test-plan/tests [{}]} + (randomize/rng-sort rng true + {:kaocha.testable/meta {:kaocha.plugin.randomize/randomize? false} + :kaocha.test-plan/tests [{}]}))) + (is (match? {:kaocha.test-plan/tests [{}]} + (randomize/rng-sort rng true + {:kaocha.plugin.randomize/randomize? false + :kaocha.test-plan/tests [{}]}))))) + (deftest randomize-test (plugin/with-plugins plugin-chain (is (match? {:kaocha.plugin.randomize/randomize? true