Allow :kaocha.plugin.randomize/randomize? to be set at any test level

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Added
 
+- Allow :kaocha.plugin.randomize/randomize? to be set at any test level.
+
 ## Fixed
 
 ## Changed
@@ -855,4 +857,4 @@ namespace.
 - The configuration format has changed, you should now start with the `#kaocha
   {}` tagged reader literal in `tests.edn` to provide defaults. If you want more
   control then overwrite `tests.edn` with the output of `--print-config` and
-  tweak.
+  tweak.

doc/03_configuration.md

@@ -216,7 +216,7 @@ bin/kaocha --plugin profiling
 Some plugins are needed for the normal functioning of Kaocha. These are added
 automatically when using the `#kaocha/v1 {}` reader literal. They are
 
-- `:kaocha.plugin/randomize`: randomize test order
+- `:kaocha.plugin/randomize`: randomize test order ([documentation](08_plugins.md#randomize))
 - `:kaocha.plugin/filter`: allow filtering and "focusing" of tests
 - `:kaocha.plugin/capture-output`: implements output capturing during tests
 

doc/08_plugins.md

@@ -180,3 +180,54 @@ To customize which keys to print, use Kaocha's "bindings" functionality, in `tes
 ``` clojure
 :kaocha/bindings {kaocha.plugin.debug/*keys* [,,,]}
 ```
+
+## Randomize
+
+The randomize plugin picks a random seed during the `config` hook and uses that
+seed to randomize the order of test suites, namespaces, and test vars during the
+`post-load` hook. 
+
+Randomization can be toggled at the following places.
+
+1. top level config
+2. test suite config
+3. namespace metadata
+
+At each level, changing the randomize value will override the previously set 
+value. For example, you could: 
+
+- randomize by default (top level `:kaocha.plugin.randomize/randomize? true`, 
+  which is the default)
+- exclude a specific test suite by setting `:kaocha.plugin.randomize/randomize? false`
+  on the suite in tests.edn
+- but randomize a single namespace within that suite `(ns ^{:kaocha.plugin.randomize/randomize? true} ...)`
+
+Or the inverse (case 2)
+
+- turn it off at the top level
+- but turn it on for certain tests suites
+- with the exception of certain namespaces
+
+Passing the `--no-randomize` CLI flag will force all randomization to be disabled. 
+
+### Enabling
+
+The randomize plugin is enabled by default. Enable or disable randomization by
+setting the `:kaocha.plugin.randomize/randomize?` key in tests.edn at the top-level,
+in a specific test suite, or on the metadata of a namespace.
+
+### Plugin-specific command line flags
+
+```shell
+--[no-]randomize                  Run test namespaces and vars in random order.
+--seed SEED                       Provide a seed to determine the random order of tests.
+```
+
+### Plugin-specific configuration options
+
+Shown with their default values:
+
+```clojure
+#kaocha/v1
+{:kaocha.plugin.randomize/randomize? true}
+```

src/kaocha/plugin/randomize.clj

@@ -7,25 +7,41 @@
   (let [rng (java.util.Random. seed)]
     (fn [& _] (.nextInt rng))))
 
-(defn straight-sort [test-plan]
-  (if-let [tests (:kaocha.test-plan/tests test-plan)]
-    (assoc test-plan
-           :kaocha.test-plan/tests
-           (->> tests
-                (sort-by :kaocha.testable/id)
-                (map straight-sort)))
-    test-plan))
+(defmacro or-some
+  ([] nil)
+  ([x] x)
+  ([x & next]
+   `(let [or# ~x]
+      (if (some? or#) or# (or-some ~@next)))))
+
+(defn get-randomize
+  "Returns the randomize? value for the test plan. Metadata takes precedence
+  over test-plan."
+  [test-plan]
+  (or-some
+    (get-in test-plan [:kaocha.testable/meta ::randomize?])
+    (::randomize? test-plan)))
 
-(defn rng-sort [rng test-plan]
+(defn rng-sort [rng parent-randomize? test-plan]
   (if-let [tests (:kaocha.test-plan/tests test-plan)]
     (assoc test-plan
            :kaocha.test-plan/tests
-           (->> tests
-                (map #(assoc % ::sort-key (rng)))
-                (sort-by ::sort-key)
-                (map (partial rng-sort rng))))
+           (let [randomize? (or-some (get-randomize test-plan) parent-randomize?)]
+             (map (partial rng-sort rng randomize?)
+                  (if randomize?
+                    (->> tests
+                         (map #(assoc % ::sort-key (rng)))
+                         (sort-by ::sort-key))
+                    tests))))
     test-plan))
 
+(defn uses-randomize?
+  "Returns true if the test-plan uses ::randomize? at any test level."
+  [test-plan]
+  (if (get-randomize test-plan)
+    true
+    (some uses-randomize? (:kaocha.test-plan/tests test-plan))))
+
 (defplugin kaocha.plugin/randomize
   (cli-options [opts]
     (conj opts
@@ -34,25 +50,24 @@
            :parse-fn #(Integer/parseInt %)]))
 
   (config [config]
-    (let [randomize? (get-in config [:kaocha/cli-options :randomize])
-          seed       (get-in config [:kaocha/cli-options :seed])
-          config     (merge {::randomize? true}
-                            config
-                            (when (some? randomize?)
-                              {::randomize? randomize?}))]
-      (if (::randomize? config)
+    (let [randomize?    (get-in config [:kaocha/cli-options :randomize])
+          no-randomize? (= false randomize?)
+          seed          (get-in config [:kaocha/cli-options :seed])
+          config        (merge {::randomize? true}
+                               config
+                               (when (some? randomize?)
+                                 {::randomize? randomize?}))]
+      (if (not no-randomize?)
         (merge {::seed (or seed (rand-int Integer/MAX_VALUE))} config)
         config)))
 
   (post-load [test-plan]
-    (if (::randomize? test-plan)
-      (let [rng (rng (::seed test-plan))]
-        (->> test-plan
-             straight-sort
-             (rng-sort rng)))
+    (if-let [seed (::seed test-plan)]
+      (let [rng (rng seed)]
+        (rng-sort rng (get-randomize test-plan) test-plan))
       test-plan))
 
   (post-run [test-plan]
-    (if (and (::randomize? test-plan) (result/failed? test-plan))
+    (if (and (::seed test-plan) (uses-randomize? test-plan) (result/failed? test-plan))
       (print "\nRandomized with --seed" (::seed test-plan)))
     test-plan))

test/features/command_line/print_config.feature

@@ -13,18 +13,17 @@ Feature: CLI: Print the Kaocha configuration
     When I run `bin/kaocha --print-config`
     Then the output should contain:
       """ clojure
-      {:kaocha.plugin.randomize/randomize? false,
-       :kaocha/reporter [kaocha.report/dots],
+       :kaocha.plugin.randomize/randomize? false,
+      """
+    And the output should contain:
+      """ clojure
+       :kaocha/reporter [kaocha.report/dots]
+      """
+    And the output should contain:
+      """ clojure
        :kaocha/color? false,
-       :kaocha/fail-fast? false,
       """
     And the output should contain:
       """ clojure
-       :kaocha/tests
-       [{:kaocha.testable/type :kaocha.type/clojure.test,
-         :kaocha.testable/id :unit,
-         :kaocha/ns-patterns ["-test$"],
-         :kaocha/source-paths ["src"],
-         :kaocha/test-paths ["test"],
-         :kaocha.filter/skip-meta [:kaocha/skip]}],
+       :kaocha/fail-fast? false,
       """

test/unit/kaocha/plugin/randomize_test.clj

@@ -2,7 +2,8 @@
   (:require [clojure.test :refer :all]
             [kaocha.test-helper :refer :all]
             [kaocha.plugin :as plugin]
-            [kaocha.testable :as testable]))
+            [kaocha.testable :as testable]
+            [kaocha.plugin.randomize :as randomize]))
 
 
 (def plugin-chain (plugin/register :kaocha.plugin/randomize []))
@@ -14,6 +15,26 @@
                  :kaocha/test-paths    ["fixtures/c-tests"]
                  :kaocha/ns-patterns   [".*"]})
 
+(deftest rng-sort-test
+  (let [rng (randomize/rng 123)]
+    (is (= {}
+          (randomize/rng-sort rng true
+            {})))
+    (is (= {:kaocha.test-plan/tests [{:kaocha.plugin.randomize/sort-key -1188957731}]}
+          (randomize/rng-sort rng true
+            {:kaocha.test-plan/tests [{}]})))
+    (is (= {:kaocha.test-plan/tests [{}]}
+          (randomize/rng-sort rng false
+            {:kaocha.test-plan/tests [{}]})))
+    (is (match? {:kaocha.test-plan/tests [{}]}
+          (randomize/rng-sort rng true
+            {:kaocha.testable/meta   {:kaocha.plugin.randomize/randomize? false}
+             :kaocha.test-plan/tests [{}]})))
+    (is (match? {:kaocha.test-plan/tests [{}]}
+          (randomize/rng-sort rng true
+            {:kaocha.plugin.randomize/randomize? false
+             :kaocha.test-plan/tests             [{}]})))))
+
 (deftest randomize-test
   (plugin/with-plugins plugin-chain
     (is (match? {:kaocha.plugin.randomize/randomize? true