From 32e8eed3e15d8c50ec515a4849f35148e04470c5 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 1 Jun 2021 23:13:27 -0500 Subject: [PATCH 01/21] Add data structure foundation. --- src/kaocha/testable.clj | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 1afd16bd..dcda9b62 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -10,7 +10,10 @@ [kaocha.plugin :as plugin] [kaocha.result :as result] [kaocha.specs :refer [assert-spec]] - [kaocha.util :as util])) + [kaocha.util :as util] + [kaocha.hierarchy :as hierarchy]) + (:import [clojure.lang Compiler$CompilerException] + [java.util.concurrent ArrayBlockingQueue BlockingQueue])) (def ^:dynamic *fail-fast?* "Should testing terminate immediately upon failure or error?" @@ -211,11 +214,33 @@ (run % test-plan) (plugin/run-hook :kaocha.hooks/post-test % test-plan))))) +(defn f [acc value] + (if (instance? BlockingQueue value) + (.drainTo value acc) + (.put acc value)) + acc) + +(defn f [acc value] (doto acc (.put value))) + +(def q (ArrayBlockingQueue. 1024)) +(def r (ArrayBlockingQueue. 1024)) + +(.put r 5) + + +(reduce f [q 1 2 r]) + (defn run-testables "Run a collection of testables, returning a result collection." [testables test-plan] - (let [load-error? (some ::load-error testables)] - (loop [result [] + (let [load-error? (some ::load-error testables) + ;; results (watch/make-queue) + put-return (fn [acc value] + (if (instance? BlockingQueue value) + (.drainTo value acc) + (.put acc value)) + acc)] + (loop [result (ArrayBlockingQueue. 1024) [test & testables] testables] (if test (let [test (cond-> test @@ -223,8 +248,8 @@ (assoc ::skip true)) r (run-testable test test-plan)] (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - (reduce into result [[r] testables]) - (recur (conj result r) testables))) + (reduce put-return result [[r] testables]) + (recur (doto result (.put r)) testables))) result)))) (defn test-seq [testable] From e0f03008994e790ccb1bc991f5548470cfffdb17 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 23 Jun 2021 10:25:17 -0500 Subject: [PATCH 02/21] Work from pairing session with Arne. --- src/kaocha/testable.clj | 35 +++++++++++++++++++++++----- src/kaocha/type/ns.clj | 2 +- test/unit/kaocha/type/ns_test.clj | 38 ++++++++++++++++++++----------- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index dcda9b62..f5a77f98 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -233,14 +233,33 @@ (defn run-testables "Run a collection of testables, returning a result collection." [testables test-plan] - (let [load-error? (some ::load-error testables) + (let [load-error? (some ::load-error testables)] + (loop [result [] + [test & testables] testables] + (if test + (let [test (cond-> test + (and load-error? (not (::load-error test))) + (assoc ::skip true)) + r (run-testable test test-plan)] + (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) + (reduce into result [[r] testables]) + (recur (conj result r) testables))) + result)))) + + +(defn run-testables-parallel + "Run a collection of testables, returning a result collection." + [testables test-plan] +(let [load-error? (some ::load-error testables) ;; results (watch/make-queue) put-return (fn [acc value] (if (instance? BlockingQueue value) (.drainTo value acc) (.put acc value)) - acc)] - (loop [result (ArrayBlockingQueue. 1024) + acc) + futures (map #(future (run-testable % test-plan)) testables)] + (println "Running in parallel!") + (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] (if test (let [test (cond-> test @@ -248,9 +267,13 @@ (assoc ::skip true)) r (run-testable test test-plan)] (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - (reduce put-return result [[r] testables]) - (recur (doto result (.put r)) testables))) - result)))) + ;(reduce put-return result [[r] testables]) + (reduce into result [[r] testables]) + ;(recur (doto result (.put r)) testables) + (recur (conj result r) testables))) + result))) + (map deref futures) + )) (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index 470489c9..5fe060d0 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -17,7 +17,7 @@ ;; It's not guaranteed the the fixture-fn returns the result of calling the ;; tests function, so we need to put it in a box for reference. (let [result (atom (:kaocha.test-plan/tests testable))] - (fixture-fn #(swap! result testable/run-testables test-plan)) + (fixture-fn #(swap! result testable/run-testables-parallel test-plan)) @result)) (defmethod testable/-load :kaocha.type/ns [testable] diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index fd9ca575..077a334d 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -66,27 +66,39 @@ :kaocha.result/fail 0}]} (:result (with-test-ctx {:fail-fast? true} - (testable/run testable testable)))))) + (testable/run testable testable))))))) - (let [testable (testable/load {:kaocha.testable/type :kaocha.type/ns - :kaocha.testable/id :baz.qux-test - :kaocha.testable/desc "baz.qux-test" - :kaocha.ns/name 'baz.qux-test})] +(require '[kaocha.config :as config]) + +(deftest run-test-parallel ;both tests currently test the parallel version but later... + (classpath/add-classpath "fixtures/f-tests") + + (let [testable (testable/load {:kaocha.testable/type :kaocha.type/clojure.test + :kaocha.testable/id :unit + :kaocha/ns-patterns ["-test$"] + :kaocha/source-paths ["src"] + :kaocha/test-paths ["fixtures/d-tests"] + :kaocha.filter/skip-meta [:kaocha/skip]}) + + #_(testable/load {:kaocha.testable/type :kaocha.type/ns + :kaocha.testable/id :foo.bar-test + :kaocha.testable/desc "foo.bar-test" + :kaocha.ns/name 'foo.bar-test})] (is (match? {:kaocha.testable/type :kaocha.type/ns - :kaocha.testable/id :baz.qux-test - :kaocha.ns/name 'baz.qux-test + :kaocha.testable/id :foo.bar-test + :kaocha.ns/name 'foo.bar-test :kaocha.ns/ns ns? :kaocha.result/tests [{:kaocha.testable/type :kaocha.type/var - :kaocha.testable/id :baz.qux-test/nested-test - :kaocha.testable/desc "nested-test" - :kaocha.var/name 'baz.qux-test/nested-test + :kaocha.testable/id :foo.bar-test/a-test + :kaocha.testable/desc "a-test" + :kaocha.var/name 'foo.bar-test/a-test :kaocha.var/var var? :kaocha.var/test fn? :kaocha.result/count 1 :kaocha.result/pass 1 - :kaocha.result/error 1 + :kaocha.result/error 0 :kaocha.result/pending 0 :kaocha.result/fail 0}]} (:result - (with-test-ctx {} - (testable/run testable testable))))))) + (with-test-ctx {:fail-fast? true} + (testable/run testable testable))))))) From f128717646323a9752b690448f16c9e6f24b9d22 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 21 Jul 2021 17:20:24 -0500 Subject: [PATCH 03/21] Add semi-functional version (with :parallel) as the flag. --- src/kaocha/runner.clj | 1 + src/kaocha/test_suite.clj | 20 ++++++++++++++++++-- src/kaocha/testable.clj | 22 +++++++++++++--------- src/kaocha/type/ns.clj | 12 ++++++++++-- test/unit/kaocha/type/ns_test.clj | 8 +++++++- 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/kaocha/runner.clj b/src/kaocha/runner.clj index 4c934459..5c1a9fcc 100644 --- a/src/kaocha/runner.clj +++ b/src/kaocha/runner.clj @@ -175,6 +175,7 @@ (try+ (System/exit (apply -main* args)) (catch :kaocha/early-exit {exit-code :kaocha/early-exit} + (shutdown-agents) (System/exit exit-code)))) (defn exec-fn diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index bfe45881..3c712357 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -2,12 +2,28 @@ (:require [clojure.test :as t] [kaocha.testable :as testable])) +(defn deref-recur [testables] + (cond (future? testables) (deref testables) + (vector? testables) (doall (mapv deref-recur testables)) + (seq? testables) (deref-recur (into [] (doall testables))) + (contains? testables :kaocha.test-plan/tests) + (update testables :kaocha.test-plan/tests deref-recur) + (contains? testables :kaocha.result/tests) + (update testables :kaocha.result/tests deref-recur) + :else testables)) + (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) - (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) + (let [results (deref-recur (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) ) + ;; _ (println "Done derefing") + ;; __ (println (class results)) + ;; __ (println (class (last results))) + ;; (doall (map #(if (future? %) (deref %) %) + ;; (testable/run-testables (:kaocha.test-plan/tests testable) test-plan))) testable (-> testable (dissoc :kaocha.test-plan/tests) (assoc :kaocha.result/tests results))] (t/do-report {:type :end-test-suite :kaocha/testable testable}) - testable)) + (doto testable tap>) + )) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index f5a77f98..88740d2c 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -233,6 +233,10 @@ (defn run-testables "Run a collection of testables, returning a result collection." [testables test-plan] + (print "run-testables got a collection of size" (count testables) + " the first of which is " + (:kaocha.testable/type (first testables)) + ) (let [load-error? (some ::load-error testables)] (loop [result [] [test & testables] testables] @@ -250,15 +254,16 @@ (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] + + (print "run-testables-parallel got a collection of size" (count testables)) (let [load-error? (some ::load-error testables) ;; results (watch/make-queue) - put-return (fn [acc value] - (if (instance? BlockingQueue value) - (.drainTo value acc) - (.put acc value)) - acc) - futures (map #(future (run-testable % test-plan)) testables)] - (println "Running in parallel!") + ;; put-return (fn [acc value] + ;; (if (instance? BlockingQueue value) + ;; (.drainTo value acc) + ;; (.put acc value)) + ;; acc) + futures (doall (map #(future (do (println "Firing off future!" (Thread/currentThread)) (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] (if test @@ -272,8 +277,7 @@ ;(recur (doto result (.put r)) testables) (recur (conj result r) testables))) result))) - (map deref futures) - )) + futures)) (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index 5fe060d0..faa51c2b 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -16,8 +16,16 @@ (defn run-tests [testable test-plan fixture-fn] ;; It's not guaranteed the the fixture-fn returns the result of calling the ;; tests function, so we need to put it in a box for reference. - (let [result (atom (:kaocha.test-plan/tests testable))] - (fixture-fn #(swap! result testable/run-testables-parallel test-plan)) + ;; (println (keys test-plan)) + ;; (println (:kaocha.testable/meta test-plan)) + (let [result (atom (:kaocha.test-plan/tests testable)) + run-testables-fn (if (:parallel testable/*config*) + testable/run-testables-parallel + testable/run-testables) + #_testable/run-testables + + ] + (fixture-fn #(swap! result run-testables-fn test-plan)) @result)) (defmethod testable/-load :kaocha.type/ns [testable] diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index 077a334d..c789476d 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -77,7 +77,7 @@ :kaocha.testable/id :unit :kaocha/ns-patterns ["-test$"] :kaocha/source-paths ["src"] - :kaocha/test-paths ["fixtures/d-tests"] + :kaocha/test-paths ["fixtures/f-tests"] :kaocha.filter/skip-meta [:kaocha/skip]}) #_(testable/load {:kaocha.testable/type :kaocha.type/ns @@ -101,4 +101,10 @@ :kaocha.result/fail 0}]} (:result (with-test-ctx {:fail-fast? true} + (testable/run testable testable))))) + (is (not (nil? (:result + (binding [testable/*config* (assoc testable/*config* :parallel true)] + (with-test-ctx {:fail-fast? true + :parallel true } (testable/run testable testable))))))) + )) From 33821987daf01f938d5d0b608af3e50b62beee7f Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Thu, 26 Aug 2021 23:23:37 -0500 Subject: [PATCH 04/21] Add some debugging code for now. --- src/kaocha/testable.clj | 25 +++++++++++++++++++------ test/unit/kaocha/type/ns_test.clj | 4 +--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 88740d2c..14efcb0f 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -58,7 +58,12 @@ (assert-spec :kaocha/testable testable) (let [type (::type testable)] (try-load-third-party-lib type) - (assert-spec type testable))) + (try + (assert-spec type testable) + (catch Exception e + (prn e) + ) + ))) (defmulti -load "Given a testable, load the specified tests, producing a test-plan." @@ -126,8 +131,10 @@ Also performs validation, and lazy loading of the testable type's implementation." [testable test-plan] - (load-type+validate testable) + ;; (println (class testable)) (binding [*current-testable* testable] + ;; (println (:kaocha.testable/id *current-testable*)) + (load-type+validate testable) (let [run (plugin/run-hook :kaocha.hooks/wrap-run -run test-plan) result (run testable test-plan)] (if-let [history history/*history*] @@ -233,7 +240,8 @@ (defn run-testables "Run a collection of testables, returning a result collection." [testables test-plan] - (print "run-testables got a collection of size" (count testables) + (doall testables) + #_(print "run-testables got a collection of size" (count testables) " the first of which is " (:kaocha.testable/type (first testables)) ) @@ -254,8 +262,8 @@ (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] - - (print "run-testables-parallel got a collection of size" (count testables)) + (doall testables) + ;; (print "run-testables-parallel got a collection of size" (count testables)) (let [load-error? (some ::load-error testables) ;; results (watch/make-queue) ;; put-return (fn [acc value] @@ -263,7 +271,12 @@ ;; (.drainTo value acc) ;; (.put acc value)) ;; acc) - futures (doall (map #(future (do (println "Firing off future!" (Thread/currentThread)) (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) testables))] + futures (doall (map #(do + (println (:parallel *config*) \space (.getName (Thread/currentThread))) + (future + ;(do #_(println "Firing off future!" (Thread/currentThread)) ) + (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) + testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] (if test diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index c789476d..605239cb 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -68,9 +68,7 @@ (with-test-ctx {:fail-fast? true} (testable/run testable testable))))))) -(require '[kaocha.config :as config]) - -(deftest run-test-parallel ;both tests currently test the parallel version but later... +(deftest run-test-parallel (classpath/add-classpath "fixtures/f-tests") (let [testable (testable/load {:kaocha.testable/type :kaocha.type/clojure.test From 6a35e4642dae8309d97ddb081b4addd8286256e2 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Fri, 27 Aug 2021 00:10:41 -0500 Subject: [PATCH 05/21] Add workarounds for now. --- src/kaocha/testable.clj | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 14efcb0f..8357b703 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -46,6 +46,13 @@ (try-require (symbol (namespace type)))) (try-require (symbol (name type))))) + +(defn- try-assert-spec [type testable n] + (let [ result (try (assert-spec type testable) (catch Exception _e false))] + (if (or result (<= n 1)) result + (try-assert-spec type testable (dec n))) ;otherwise, retry + )) + (defn- load-type+validate "Try to load a testable type, and validate it both to be a valid generic testable, and a valid instance given the type. @@ -58,12 +65,10 @@ (assert-spec :kaocha/testable testable) (let [type (::type testable)] (try-load-third-party-lib type) - (try - (assert-spec type testable) + (try + (try-assert-spec type testable 3) (catch Exception e - (prn e) - ) - ))) + (output/warn (format "Could not load %s. This is a known bug in parallelization.\n%s" type e)))))) (defmulti -load "Given a testable, load the specified tests, producing a test-plan." @@ -131,10 +136,8 @@ Also performs validation, and lazy loading of the testable type's implementation." [testable test-plan] - ;; (println (class testable)) + (load-type+validate testable) (binding [*current-testable* testable] - ;; (println (:kaocha.testable/id *current-testable*)) - (load-type+validate testable) (let [run (plugin/run-hook :kaocha.hooks/wrap-run -run test-plan) result (run testable test-plan)] (if-let [history history/*history*] @@ -221,6 +224,12 @@ (run % test-plan) (plugin/run-hook :kaocha.hooks/post-test % test-plan))))) +(defn try-run-testable [test test-plan n] + (let [ result (try (run-testable test test-plan) (catch Exception _e false))] + (if (or result (> n 1)) result ;success or last try, return + (try-run-testable test test-plan (dec n))) ;otherwise retry + )) + (defn f [acc value] (if (instance? BlockingQueue value) (.drainTo value acc) @@ -271,11 +280,11 @@ ;; (.drainTo value acc) ;; (.put acc value)) ;; acc) - futures (doall (map #(do + futures (doall (map #(do (println (:parallel *config*) \space (.getName (Thread/currentThread))) - (future + (future ;(do #_(println "Firing off future!" (Thread/currentThread)) ) - (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) + (binding [*config* (dissoc *config* :parallel)] (try-run-testable % test-plan 3)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] From a5d2f7f8f8f5ca1fbec25aad3f3a21d9fd1111bf Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Fri, 27 Aug 2021 00:22:17 -0500 Subject: [PATCH 06/21] Replace workarounds by locking require. --- src/kaocha/testable.clj | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 8357b703..1ce49c58 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -29,13 +29,17 @@ and `:line`." nil) + +(def REQUIRE_LOCK (Object.)) + (defn add-desc [testable description] (assoc testable ::desc (str (name (::id testable)) " (" description ")"))) (defn- try-require [n] (try - (require n) + (locking REQUIRE_LOCK + (require n)) true (catch java.io.FileNotFoundException e false))) @@ -47,10 +51,10 @@ (try-require (symbol (name type))))) -(defn- try-assert-spec [type testable n] +(defn- retry-assert-spec [type testable n] (let [ result (try (assert-spec type testable) (catch Exception _e false))] (if (or result (<= n 1)) result - (try-assert-spec type testable (dec n))) ;otherwise, retry + (retry-assert-spec type testable (dec n))) ;otherwise, retry )) (defn- load-type+validate @@ -66,7 +70,7 @@ (let [type (::type testable)] (try-load-third-party-lib type) (try - (try-assert-spec type testable 3) + (assert-spec type testable) (catch Exception e (output/warn (format "Could not load %s. This is a known bug in parallelization.\n%s" type e)))))) @@ -284,7 +288,7 @@ (println (:parallel *config*) \space (.getName (Thread/currentThread))) (future ;(do #_(println "Firing off future!" (Thread/currentThread)) ) - (binding [*config* (dissoc *config* :parallel)] (try-run-testable % test-plan 3)))) + (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] From 20e2812212179bccbc4f9513237388bacc6fbf67 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 19 Oct 2021 16:15:01 -0500 Subject: [PATCH 07/21] Allow parallelization to happen at any level. --- src/kaocha/result.clj | 1 + src/kaocha/test_suite.clj | 12 ++---------- src/kaocha/testable.clj | 38 ++++++++++++++++++++++++++++++++------ src/kaocha/type/ns.clj | 10 ++-------- 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/kaocha/result.clj b/src/kaocha/result.clj index 755bec2c..b42bdd98 100644 --- a/src/kaocha/result.clj +++ b/src/kaocha/result.clj @@ -34,6 +34,7 @@ (defn ^:no-gen testable-totals "Return a map of summed up results for a testable, including descendants." [testable] + ;; (prn "TESTABLE: " testable) (if-let [testables (::tests testable)] (merge testable (totals testables)) (merge (sum) testable))) diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index 3c712357..eb3e66a8 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -2,19 +2,11 @@ (:require [clojure.test :as t] [kaocha.testable :as testable])) -(defn deref-recur [testables] - (cond (future? testables) (deref testables) - (vector? testables) (doall (mapv deref-recur testables)) - (seq? testables) (deref-recur (into [] (doall testables))) - (contains? testables :kaocha.test-plan/tests) - (update testables :kaocha.test-plan/tests deref-recur) - (contains? testables :kaocha.result/tests) - (update testables :kaocha.result/tests deref-recur) - :else testables)) + (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) - (let [results (deref-recur (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) ) + (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) ;; _ (println "Done derefing") ;; __ (println (class results)) ;; __ (println (class (last results))) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 1ce49c58..dc1ed211 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -57,6 +57,16 @@ (retry-assert-spec type testable (dec n))) ;otherwise, retry )) +(defn deref-recur [testables] + (cond (future? testables) (deref testables) + (vector? testables) (doall (mapv deref-recur testables)) + (seq? testables) (deref-recur (into [] (doall testables))) + (contains? testables :kaocha.test-plan/tests) + (update testables :kaocha.test-plan/tests deref-recur) + (contains? testables :kaocha.result/tests) + (update testables :kaocha.result/tests deref-recur) + :else testables)) + (defn- load-type+validate "Try to load a testable type, and validate it both to be a valid generic testable, and a valid instance given the type. @@ -180,6 +190,7 @@ [file line] (util/compiler-exception-file-and-line error) file (::load-error-file test file) line (::load-error-line test line) + thread (.getName (Thread/currentThread)) m (if-let [message (::load-error-message test)] {:type :error :message message @@ -191,7 +202,8 @@ :kaocha/testable test}) m (cond-> m file (assoc :file file) - line (assoc :line line))] + line (assoc :line line) + thread (assoc :thread thread))] (t/do-report (assoc m :type :kaocha/begin-suite)) (binding [*fail-fast?* false] (t/do-report m)) @@ -250,7 +262,7 @@ (reduce f [q 1 2 r]) -(defn run-testables +(defn run-testables-serial "Run a collection of testables, returning a result collection." [testables test-plan] (doall testables) @@ -284,11 +296,17 @@ ;; (.drainTo value acc) ;; (.put acc value)) ;; acc) + ;; _ (println "nested tests?" (:parallel-test-level *config* false)) + type(:kaocha.testable/type test-plan) + _ (println type) futures (doall (map #(do - (println (:parallel *config*) \space (.getName (Thread/currentThread))) (future - ;(do #_(println "Firing off future!" (Thread/currentThread)) ) - (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) + (binding [;*config* (if (:parallel-test-level *config* false) + ; (dissoc *config* :parallel) *config* ) + + *config* (update *config* :levels (fn [x] (if (nil? x) 1 (inc x)))) ] + (prn (:levels *config* 0)) + (run-testable % test-plan)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] @@ -303,7 +321,15 @@ ;(recur (doto result (.put r)) testables) (recur (conj result r) testables))) result))) - futures)) + (deref-recur futures))) + + +(defn run-testables + [testables test-plan] + (if (:parallel *config*) + (doall (run-testables-parallel testables test-plan)) + (run-testables-serial testables test-plan))) + (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index faa51c2b..97b6378a 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -18,14 +18,8 @@ ;; tests function, so we need to put it in a box for reference. ;; (println (keys test-plan)) ;; (println (:kaocha.testable/meta test-plan)) - (let [result (atom (:kaocha.test-plan/tests testable)) - run-testables-fn (if (:parallel testable/*config*) - testable/run-testables-parallel - testable/run-testables) - #_testable/run-testables - - ] - (fixture-fn #(swap! result run-testables-fn test-plan)) + (let [result (atom (:kaocha.test-plan/tests testable))] + (fixture-fn #(swap! result testable/run-testables test-plan)) @result)) (defmethod testable/-load :kaocha.type/ns [testable] From cb0912975253bd1da8a9d76eabdeaf713c0a06d0 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 19 Oct 2021 16:34:41 -0500 Subject: [PATCH 08/21] Clean up. --- src/kaocha/testable.clj | 48 ++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index dc1ed211..e7e3483e 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -288,40 +288,30 @@ "Run a collection of testables, returning a result collection." [testables test-plan] (doall testables) - ;; (print "run-testables-parallel got a collection of size" (count testables)) -(let [load-error? (some ::load-error testables) - ;; results (watch/make-queue) - ;; put-return (fn [acc value] - ;; (if (instance? BlockingQueue value) - ;; (.drainTo value acc) - ;; (.put acc value)) - ;; acc) - ;; _ (println "nested tests?" (:parallel-test-level *config* false)) - type(:kaocha.testable/type test-plan) - _ (println type) + (let [load-error? (some ::load-error testables) + types (set (:parallel-children-exclude *config*)) futures (doall (map #(do (future - (binding [;*config* (if (:parallel-test-level *config* false) - ; (dissoc *config* :parallel) *config* ) - - *config* (update *config* :levels (fn [x] (if (nil? x) 1 (inc x)))) ] - (prn (:levels *config* 0)) + (binding [*config* + (cond-> *config* + (contains? types (:kaocha.testable/type %)) (dissoc :parallel) + true (update :levels (fn [x] (if (nil? x) 1 (inc x))))) ] (run-testable % test-plan)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) - [test & testables] testables] - (if test - (let [test (cond-> test - (and load-error? (not (::load-error test))) - (assoc ::skip true)) - r (run-testable test test-plan)] - (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - ;(reduce put-return result [[r] testables]) - (reduce into result [[r] testables]) - ;(recur (doto result (.put r)) testables) - (recur (conj result r) testables))) - result))) - (deref-recur futures))) + [test & testables] testables] + (if test + (let [test (cond-> test + (and load-error? (not (::load-error test))) + (assoc ::skip true)) + r (run-testable test test-plan)] + (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) + ;(reduce put-return result [[r] testables]) + (reduce into result [[r] testables]) + ;(recur (doto result (.put r)) testables) + (recur (conj result r) testables))) + result))) + (deref-recur futures))) (defn run-testables From 503bfe7f6104cc90bf1afe92174cdaaa4796dfcf Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 19 Oct 2021 17:12:50 -0500 Subject: [PATCH 09/21] Clean up. --- src/kaocha/testable.clj | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index e7e3483e..2e23fb7e 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -290,14 +290,14 @@ (doall testables) (let [load-error? (some ::load-error testables) types (set (:parallel-children-exclude *config*)) - futures (doall (map #(do - (future - (binding [*config* - (cond-> *config* - (contains? types (:kaocha.testable/type %)) (dissoc :parallel) - true (update :levels (fn [x] (if (nil? x) 1 (inc x))))) ] - (run-testable % test-plan)))) - testables))] + futures (map #(do + (future + (binding [*config* + (cond-> *config* + (contains? types (:kaocha.testable/type %)) (dissoc :parallel) + true (update :levels (fn [x] (if (nil? x) 1 (inc x))))) ] + (run-testable % test-plan)))) + testables)] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] (if test From 544a3d30fe5a47b66b0093af11ba7fe44710e963 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Thu, 21 Oct 2021 01:27:11 -0500 Subject: [PATCH 10/21] Add benchmarking. --- repl_sessions/benchmark.clj | 97 +++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 repl_sessions/benchmark.clj diff --git a/repl_sessions/benchmark.clj b/repl_sessions/benchmark.clj new file mode 100644 index 00000000..4a205481 --- /dev/null +++ b/repl_sessions/benchmark.clj @@ -0,0 +1,97 @@ + +(ns benchmark + (:require [criterium.core :as c]) + (:import [java.util.concurrent Executors ]) + ) + +(def thread-pool (Executors/newFixedThreadPool 10)) + +(defn math-direct [] + (+ 1 1)) + +(defn math-future [] + (deref + (future (+ 1 1)))) + +(defn math-thread [] + (let [result (atom nil)] + (doto (Thread. (fn [] (reset! result (+ 1 1)))) + (.start) + (.join)) + @result)) + +(defn math-threadpool [] + (let [result (atom nil)] + (.get (.submit thread-pool (fn [] (reset! result (+ 1 1))) )) + @result)) + +(defn math-threadpool-no-atom [] + (.get (.submit thread-pool (fn [] (+ 1 1)) ))) + + +(c/bench (math-direct) ) +; (out) Evaluation count : 6215391600 in 60 samples of 103589860 calls. +; (out) Execution time mean : 2,015262 ns +; (out) Execution time std-deviation : 0,497743 ns +; (out) Execution time lower quantile : 1,442374 ns ( 2,5%) +; (out) Execution time upper quantile : 3,392990 ns (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 5 outliers in 60 samples (8,3333 %) +; (out) low-severe 3 (5,0000 %) +; (out) low-mild 2 (3,3333 %) +; (out) Variance from outliers : 94,6147 % Variance is severely inflated by outliers + +(c/bench (math-future) ) +; (out) Evaluation count : 3735420 in 60 samples of 62257 calls. +; (out) Execution time mean : 16,635809 µs +; (out) Execution time std-deviation : 1,104338 µs +; (out) Execution time lower quantile : 15,397518 µs ( 2,5%) +; (out) Execution time upper quantile : 19,751883 µs (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 6 outliers in 60 samples (10,0000 %) +; (out) low-severe 3 (5,0000 %) +; (out) low-mild 3 (5,0000 %) +; (out) Variance from outliers : 50,0892 % Variance is severely inflated by outliers + +(c/bench (math-thread)) + +; (out) Evaluation count : 774420 in 60 samples of 12907 calls. +; (out) Execution time mean : 82,513236 µs +; (out) Execution time std-deviation : 5,706987 µs +; (out) Execution time lower quantile : 75,772237 µs ( 2,5%) +; (out) Execution time upper quantile : 91,971212 µs (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 1 outliers in 60 samples (1,6667 %) +; (out) low-severe 1 (1,6667 %) +; (out) Variance from outliers : 51,7849 % Variance is severely inflated by outliers + +(c/bench (math-threadpool)) +; (out) Evaluation count : 3815100 in 60 samples of 63585 calls. +; (out) Execution time mean : 16,910124 µs +; (out) Execution time std-deviation : 2,443261 µs +; (out) Execution time lower quantile : 14,670118 µs ( 2,5%) +; (out) Execution time upper quantile : 23,743868 µs (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 3 outliers in 60 samples (5,0000 %) +; (out) low-severe 2 (3,3333 %) +; (out) low-mild 1 (1,6667 %) +; (out) Variance from outliers : 82,4670 % Variance is severely inflated by outliers + + +(c/bench (math-threadpool-no-atom)) + +; (out) Evaluation count : 3794940 in 60 samples of 63249 calls. +; (out) Execution time mean : 16,182655 µs +; (out) Execution time std-deviation : 1,215451 µs +; (out) Execution time lower quantile : 14,729393 µs ( 2,5%) +; (out) Execution time upper quantile : 18,549902 µs (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 3 outliers in 60 samples (5,0000 %) +; (out) low-severe 2 (3,3333 %) +; (out) low-mild 1 (1,6667 %) +; (out) Variance from outliers : 56,7625 % Variance is severely inflated by outliers From 141683e98663b80c1755b88945734ce82487234f Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 8 Mar 2022 13:58:24 -0600 Subject: [PATCH 11/21] Add warning. --- src/kaocha/api.clj | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/kaocha/api.clj b/src/kaocha/api.clj index 9b2d194c..ad154b6e 100644 --- a/src/kaocha/api.clj +++ b/src/kaocha/api.clj @@ -123,6 +123,11 @@ " To investigate, check the :test-paths and :ns-patterns keys in tests.edn."))) (throw+ {:kaocha/early-exit 0})) + (when (:parallel config) + (output/warn (str "Parallelization enabled. This is a beta " + "feature. If you encounter errors, try " + "running with the feature disabled and " + "consider filing a PR."))) (when (find-ns 'matcher-combinators.core) (require 'kaocha.matcher-combinators)) From cc983c1131f01553d0dddd16c7e31dc87423631b Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 8 Mar 2022 15:38:24 -0600 Subject: [PATCH 12/21] Add commandline flag. --- src/kaocha/api.clj | 6 +++--- src/kaocha/config.clj | 5 +++-- src/kaocha/runner.clj | 5 +++-- src/kaocha/testable.clj | 37 +++++++++++++++---------------------- 4 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/kaocha/api.clj b/src/kaocha/api.clj index ad154b6e..9f3c00a7 100644 --- a/src/kaocha/api.clj +++ b/src/kaocha/api.clj @@ -124,10 +124,10 @@ (throw+ {:kaocha/early-exit 0})) (when (:parallel config) - (output/warn (str "Parallelization enabled. This is a beta " - "feature. If you encounter errors, try " + (output/warn (str "Parallelization enabled. This feature is in " + "beta If you encounter errors, try " "running with the feature disabled and " - "consider filing a PR."))) + "consider filing an issue."))) (when (find-ns 'matcher-combinators.core) (require 'kaocha.matcher-combinators)) diff --git a/src/kaocha/config.clj b/src/kaocha/config.clj index 05775a93..2756bdce 100644 --- a/src/kaocha/config.clj +++ b/src/kaocha/config.clj @@ -31,7 +31,7 @@ (update config k vary-meta assoc :replace true) (do (output/error "Test suite configuration value with key " k " should be a collection or symbol, but got '" v "' of type " (type v)) - (throw+ {:kaocha/early-exit 250})))) + (throw+ {:kaocha/early-exit 252})))) config)) (defn merge-config @@ -204,6 +204,7 @@ (some? (:color options)) (assoc :kaocha/color? (:color options)) (some? (:diff-style options)) (assoc :kaocha/diff-style (:diff-style options)) (:plugin options) (update :kaocha/plugins #(distinct (concat % (:plugin options)))) + (some? (:parallel options)) (assoc :parallel (:parallel options)) true (assoc :kaocha/cli-options options))) (defn apply-cli-args [config args] @@ -227,7 +228,7 @@ cli-opts (apply-cli-opts cli-opts) cli-args (apply-cli-args cli-args))) -(defn find-config-and-warn +(defn find-config-and-warn [config-file] (let [final-config-file (or config-file "tests.edn")] (when (not (.exists (io/file (or config-file "tests.edn")))) diff --git a/src/kaocha/runner.clj b/src/kaocha/runner.clj index 5c1a9fcc..184111ee 100644 --- a/src/kaocha/runner.clj +++ b/src/kaocha/runner.clj @@ -8,6 +8,7 @@ [clojure.spec.alpha :as spec] [clojure.string :as str] [clojure.tools.cli :as cli] + [clojure.pprint :as pp] [expound.alpha :as expound] [kaocha.api :as api] [kaocha.config :as config] @@ -33,6 +34,7 @@ [nil "--[no-]fail-fast" "Stop testing after the first failure."] [nil "--[no-]color" "Enable/disable ANSI color codes in output. Defaults to true."] [nil "--[no-]watch" "Watch filesystem for changes and re-run tests."] + [nil "--[no-]parallel" "Run tests in parallel. Warning: This feature is beta."] [nil "--reporter SYMBOL" "Change the test reporter, can be specified multiple times." :parse-fn (fn [s] (let [sym (symbol s)] @@ -41,8 +43,7 @@ (symbol "kaocha.report" s)))) :assoc-fn accumulate] [nil "--diff-style STYLE" "The style of diff to print on failing tests, either :none or :deep" - :parse-fn parse-kw - ] + :parse-fn parse-kw] [nil "--plugin KEYWORD" "Load the given plugin." :parse-fn (fn [s] (let [kw (parse-kw s)] diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 2e23fb7e..22916dd3 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -29,7 +29,6 @@ and `:line`." nil) - (def REQUIRE_LOCK (Object.)) (defn add-desc [testable description] @@ -50,12 +49,11 @@ (try-require (symbol (namespace type)))) (try-require (symbol (name type))))) - (defn- retry-assert-spec [type testable n] - (let [ result (try (assert-spec type testable) (catch Exception _e false))] + (let [result (try (assert-spec type testable) (catch Exception _e false))] (if (or result (<= n 1)) result - (retry-assert-spec type testable (dec n))) ;otherwise, retry - )) + (retry-assert-spec type testable (dec n))) ;otherwise, retry +)) (defn deref-recur [testables] (cond (future? testables) (deref testables) @@ -125,8 +123,8 @@ (assoc testable ::load-error t) (throw t))))) -(spec/fdef load - :args (spec/cat :testable :kaocha/testable) +(s/fdef load + :args (s/cat :testable :kaocha/testable) :ret :kaocha.test-plan/testable) (defmulti -run @@ -241,16 +239,16 @@ (plugin/run-hook :kaocha.hooks/post-test % test-plan))))) (defn try-run-testable [test test-plan n] - (let [ result (try (run-testable test test-plan) (catch Exception _e false))] + (let [result (try (run-testable test test-plan) (catch Exception _e false))] (if (or result (> n 1)) result ;success or last try, return - (try-run-testable test test-plan (dec n))) ;otherwise retry - )) + (try-run-testable test test-plan (dec n))) ;otherwise retry +)) (defn f [acc value] - (if (instance? BlockingQueue value) - (.drainTo value acc) - (.put acc value)) - acc) + (if (instance? BlockingQueue value) + (.drainTo value acc) + (.put acc value)) + acc) (defn f [acc value] (doto acc (.put value))) @@ -259,7 +257,6 @@ (.put r 5) - (reduce f [q 1 2 r]) (defn run-testables-serial @@ -267,9 +264,8 @@ [testables test-plan] (doall testables) #_(print "run-testables got a collection of size" (count testables) - " the first of which is " - (:kaocha.testable/type (first testables)) - ) + " the first of which is " + (:kaocha.testable/type (first testables))) (let [load-error? (some ::load-error testables)] (loop [result [] [test & testables] testables] @@ -283,7 +279,6 @@ (recur (conj result r) testables))) result)))) - (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] @@ -295,7 +290,7 @@ (binding [*config* (cond-> *config* (contains? types (:kaocha.testable/type %)) (dissoc :parallel) - true (update :levels (fn [x] (if (nil? x) 1 (inc x))))) ] + true (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] (run-testable % test-plan)))) testables)] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) @@ -313,14 +308,12 @@ result))) (deref-recur futures))) - (defn run-testables [testables test-plan] (if (:parallel *config*) (doall (run-testables-parallel testables test-plan)) (run-testables-serial testables test-plan))) - (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) (:kaocha.test-plan/tests testable) From e6382b8ba8133e5060d9c46c7b570e936ae4b5b1 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 6 Apr 2022 18:03:08 -0500 Subject: [PATCH 13/21] Clean up. --- src/kaocha/result.clj | 1 - src/kaocha/test_suite.clj | 10 +--------- src/kaocha/testable.clj | 14 -------------- src/kaocha/type/ns.clj | 2 -- test/unit/kaocha/type/ns_test.clj | 11 +++-------- 5 files changed, 4 insertions(+), 34 deletions(-) diff --git a/src/kaocha/result.clj b/src/kaocha/result.clj index b42bdd98..755bec2c 100644 --- a/src/kaocha/result.clj +++ b/src/kaocha/result.clj @@ -34,7 +34,6 @@ (defn ^:no-gen testable-totals "Return a map of summed up results for a testable, including descendants." [testable] - ;; (prn "TESTABLE: " testable) (if-let [testables (::tests testable)] (merge testable (totals testables)) (merge (sum) testable))) diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index eb3e66a8..56ffe354 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -2,20 +2,12 @@ (:require [clojure.test :as t] [kaocha.testable :as testable])) - - (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) - ;; _ (println "Done derefing") - ;; __ (println (class results)) - ;; __ (println (class (last results))) - ;; (doall (map #(if (future? %) (deref %) %) - ;; (testable/run-testables (:kaocha.test-plan/tests testable) test-plan))) testable (-> testable (dissoc :kaocha.test-plan/tests) (assoc :kaocha.result/tests results))] (t/do-report {:type :end-test-suite :kaocha/testable testable}) - (doto testable tap>) - )) + testable)) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 22916dd3..5778b249 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -244,20 +244,6 @@ (try-run-testable test test-plan (dec n))) ;otherwise retry )) -(defn f [acc value] - (if (instance? BlockingQueue value) - (.drainTo value acc) - (.put acc value)) - acc) - -(defn f [acc value] (doto acc (.put value))) - -(def q (ArrayBlockingQueue. 1024)) -(def r (ArrayBlockingQueue. 1024)) - -(.put r 5) - -(reduce f [q 1 2 r]) (defn run-testables-serial "Run a collection of testables, returning a result collection." diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index 97b6378a..470489c9 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -16,8 +16,6 @@ (defn run-tests [testable test-plan fixture-fn] ;; It's not guaranteed the the fixture-fn returns the result of calling the ;; tests function, so we need to put it in a box for reference. - ;; (println (keys test-plan)) - ;; (println (:kaocha.testable/meta test-plan)) (let [result (atom (:kaocha.test-plan/tests testable))] (fixture-fn #(swap! result testable/run-testables test-plan)) @result)) diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index 605239cb..dc1c1414 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -76,12 +76,8 @@ :kaocha/ns-patterns ["-test$"] :kaocha/source-paths ["src"] :kaocha/test-paths ["fixtures/f-tests"] - :kaocha.filter/skip-meta [:kaocha/skip]}) + :kaocha.filter/skip-meta [:kaocha/skip]})] - #_(testable/load {:kaocha.testable/type :kaocha.type/ns - :kaocha.testable/id :foo.bar-test - :kaocha.testable/desc "foo.bar-test" - :kaocha.ns/name 'foo.bar-test})] (is (match? {:kaocha.testable/type :kaocha.type/ns :kaocha.testable/id :foo.bar-test :kaocha.ns/name 'foo.bar-test @@ -103,6 +99,5 @@ (is (not (nil? (:result (binding [testable/*config* (assoc testable/*config* :parallel true)] (with-test-ctx {:fail-fast? true - :parallel true } - (testable/run testable testable))))))) - )) + :parallel true} + (testable/run testable testable))))))))) From 8e07ed6b093d221c5b4b4849811c01bc6b0ea2c5 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 6 Apr 2022 18:13:40 -0500 Subject: [PATCH 14/21] Add docs and CHANGELOG entry. --- CHANGELOG.md | 4 ++++ doc/04_running_kaocha_cli.md | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a304794..ee12dad8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Added +- You can now parallelize tests by enabling the `:parallel` key or + the `--parallel` flag. This is still a beta feature, but works on a variety + of code bases. + ## Fixed ## Changed diff --git a/doc/04_running_kaocha_cli.md b/doc/04_running_kaocha_cli.md index e4fb4351..f4205fe4 100644 --- a/doc/04_running_kaocha_cli.md +++ b/doc/04_running_kaocha_cli.md @@ -95,6 +95,31 @@ unhelpful output in a particular scenario, you can turn it off using the ![Terminal screenshot showing an expected value of "{:expected-key 1}" and an actual value. ":unexpected-key 1" is in green because it is an extra key not expected and "expected-key 1" is in red because it was expected but not present.](./deep-diff.png) +## Parallelization + +Kaocha allows you to run tests in parallel using the `:parallel` key or +`--parallel` flag. This is primarily useful for I/O heavy tests, but could also +be useful for CPU-bound tests. + +Before enabling parallelization, be sure to test it. Consider using a tool like +`bench` or `hyperfine`. While Kaocha's built-in profiling tools are great for +identifying specific slow tests, but don't repeatedly measure your entire test suite +to account for variation and noise. If you want to test it on CI, test it for CI +specifically. CI runners are often lower powered than even a middle-of-the-road laptop. + +`test.check` tests consist of repeatedly testing a property against random data. +In principle, these tests would be an excellent use case for parallelization. +Because this repeated testing happens within `test.check`, Kaocha sees it as a +single test. If you have many property-based tests that take a similar amount of +time, parallelization is a great fit. However, if you have one or two +property-based tests that take the bulk of the time, parallelization may not +make a significant difference because the work cannot be split up. + +If you want to disable parallelization that's enabled in your configuration, you can +pass `--no-parallel`. If you find yourself frequently reaching for this flag, +it's probably worth reconsidering your configuration---having to frequently +disable parallelization might be negating any time saved by parallelization. + ## Debug information `--version` prints version information, whereas `--test-help` will print the From fd5bfe9c62c76683b935d024f9d08aff2bb624b9 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 6 Apr 2022 18:54:44 -0500 Subject: [PATCH 15/21] Fix dash. --- doc/04_running_kaocha_cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/04_running_kaocha_cli.md b/doc/04_running_kaocha_cli.md index f4205fe4..6c9cf039 100644 --- a/doc/04_running_kaocha_cli.md +++ b/doc/04_running_kaocha_cli.md @@ -117,7 +117,7 @@ make a significant difference because the work cannot be split up. If you want to disable parallelization that's enabled in your configuration, you can pass `--no-parallel`. If you find yourself frequently reaching for this flag, -it's probably worth reconsidering your configuration---having to frequently +it's probably worth reconsidering your configuration—having to frequently disable parallelization might be negating any time saved by parallelization. ## Debug information From dc52eb1654dd10a900923b24ea741aeb0eb14cfa Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 6 Apr 2022 19:00:56 -0500 Subject: [PATCH 16/21] Tweak wording further. --- doc/04_running_kaocha_cli.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/04_running_kaocha_cli.md b/doc/04_running_kaocha_cli.md index 6c9cf039..153be341 100644 --- a/doc/04_running_kaocha_cli.md +++ b/doc/04_running_kaocha_cli.md @@ -101,18 +101,19 @@ Kaocha allows you to run tests in parallel using the `:parallel` key or `--parallel` flag. This is primarily useful for I/O heavy tests, but could also be useful for CPU-bound tests. -Before enabling parallelization, be sure to test it. Consider using a tool like +Before enabling parallelization, strongly consider timing it to ensure it +actually makes a difference. Consider using a tool like `bench` or `hyperfine`. While Kaocha's built-in profiling tools are great for -identifying specific slow tests, but don't repeatedly measure your entire test suite -to account for variation and noise. If you want to test it on CI, test it for CI -specifically. CI runners are often lower powered than even a middle-of-the-road laptop. +identifying specific tests that take a disproportionate amount of time, they don't repeatedly measure your entire test suite +to account for variation and noise. If you want to use parallelization to +speed up continuous integration, try to use the CI service itself or similar hardware. CI runners are often lower powered than even a middle-of-the-road laptop. `test.check` tests consist of repeatedly testing a property against random data. In principle, these tests would be an excellent use case for parallelization. -Because this repeated testing happens within `test.check`, Kaocha sees it as a -single test. If you have many property-based tests that take a similar amount of +However, because this repeated testing happens within `test.check`, Kaocha sees each `defspec` as a +single test. If you have many property-based tests that take a significant amount of time, parallelization is a great fit. However, if you have one or two -property-based tests that take the bulk of the time, parallelization may not +property-based tests that take up the bulk of the runtme time, parallelization may not make a significant difference because the work cannot be split up. If you want to disable parallelization that's enabled in your configuration, you can From 0136e656543b97dab269ef50bc3680c98c40a515 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Mon, 22 Aug 2022 17:48:54 -0500 Subject: [PATCH 17/21] Add option to filter out by test-suite. --- src/kaocha/testable.clj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 5778b249..40c5b5b2 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -271,11 +271,13 @@ (doall testables) (let [load-error? (some ::load-error testables) types (set (:parallel-children-exclude *config*)) + suites (:parallel-suites-exclude *config*) futures (map #(do (future (binding [*config* (cond-> *config* (contains? types (:kaocha.testable/type %)) (dissoc :parallel) + (and (hierarchy/suite? %) (contains? suites (:kaocha.testable/desc %))) (dissoc :parallel) true (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] (run-testable % test-plan)))) testables)] From 78f2312686633c75d6e2f44fd4ccecb863c3772f Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Mon, 22 Aug 2022 17:49:47 -0500 Subject: [PATCH 18/21] Add thread info. Add some information about the thread each test ran on. May want to move this to something not specific to parallelization, otherwise non-parallel tests won't have any thread info. --- src/kaocha/testable.clj | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 40c5b5b2..de95a04b 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -265,6 +265,12 @@ (recur (conj result r) testables))) result)))) +(defn current-thread-info [] + (let [thread (Thread/currentThread)] + {:name (.getName thread) + :id (.getId thread) + :group-name (.getName (.getThreadGroup thread))})) + (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] @@ -279,7 +285,7 @@ (contains? types (:kaocha.testable/type %)) (dissoc :parallel) (and (hierarchy/suite? %) (contains? suites (:kaocha.testable/desc %))) (dissoc :parallel) true (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] - (run-testable % test-plan)))) + (run-testable (assoc % ::thread (current-thread-info) ) test-plan)))) testables)] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] From 356d22c0a2cfed2390b21c2a96470c5192e3a5c6 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 21 Dec 2022 16:30:41 -0600 Subject: [PATCH 19/21] Very WIP plugin for profiling parallel execution. Internal refers to whether there were delays between running tests in the same thread. It doesn't measure waiting for I/O or many kinds of contention. External refers to how much of the overall time the thread spent. Why do I think this is useful? Well, if you've parallelized your test suite and you end up with a thread that runs from time 0 to time 100 and another that runs from time 1 to time 10, the burden is heavily on the first thread, suggsting the load is not well-balanced. Both may have good internal utilization, but the second has poor external globalization. Part of the reason this is WIP is that there may be better (ideally standard) terms for "internal" and "external". There are some calculation issues and hacks: * Some testables are missing the ::start and ::end for some reason. I don't think this should happen? * I don't think this version counts the number of threads accurately. --- .../plugin/alpha/parallel_profiling.clj | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/kaocha/plugin/alpha/parallel_profiling.clj diff --git a/src/kaocha/plugin/alpha/parallel_profiling.clj b/src/kaocha/plugin/alpha/parallel_profiling.clj new file mode 100644 index 00000000..3df104fd --- /dev/null +++ b/src/kaocha/plugin/alpha/parallel_profiling.clj @@ -0,0 +1,90 @@ +(ns kaocha.plugin.alpha.parallel-profiling + (:require [clojure.java.io :as io] + [clojure.string :as str] + [kaocha.plugin :as plugin :refer [defplugin]] + [kaocha.output :as output] + [kaocha.testable :as testable]) + (:import java.time.Instant + java.time.temporal.ChronoUnit)) + +(defn start [testable] + (assoc testable ::start (Instant/now))) + +(defn stop [testable] + (cond-> testable + true (assoc ::end (Instant/now)) + (::start testable) + (assoc ::duration (.until (::start testable) + (Instant/now) + ChronoUnit/NANOS)))) + +(defplugin kaocha.plugin.alpha/parallel-profiling + (config [config] + (output/warn "Warning: The kaocha.plugin.alpha/parallel-profiling plugin is in an alpha status, like the parallel feature in general.") + config + ) + (pre-run [test-plan] + (start test-plan)) + + (post-run [test-plan] + (stop test-plan)) + + (pre-test [testable _] + (start testable)) + + (post-test [testable _] + (stop testable)) + + (cli-options [opts] + (conj opts + [nil "--[no-]-parallel-profiling" "Show slowest tests of each type with timing information."] + #_[nil "--profiling-count NUM" "Show this many slow tests of each kind in profile results." + :parse-fn #(Integer/parseInt %)])) + + (config [{:kaocha/keys [cli-options] :as config}] + (assoc config + ::parallel-profiling? (:parallel-profiling cli-options (::parallel-profiling? config true)) + #_#_::count (:profiling-count cli-options (::count config 3)))) + + (post-summary [result] + (when (::parallel-profiling? result) + (let [tests (->> result + testable/test-seq + (remove ::testable/load-error) + (remove ::testable/skip)) + #_#_types (group-by :kaocha.testable/type tests) + threads (group-by #(get-in % [ :kaocha.testable/thread :name]) tests) + total-duration (::duration result) + #_#_limit (::count result) + ] + (->> (for [[thread tests] threads + :when (and thread + (some (complement nil?) (map ::start tests)) + (some (complement nil?) (map ::end tests))) ;temporary fix until I figure out why these keys are sometimes missing. + :let [start (reduce min (map ::start tests)) + end (reduce max (map ::end tests)) + span-ns (.until start + end + ChronoUnit/NANOS) + span (cond + (> span-ns 1e8) (format "%.2f s" (/ span-ns 1e9)) + (> span-ns 1e5) (format "%.2f ms" (/ span-ns 1e9)) + :else (str span-ns " ns")) + utilization (float (* 100 (/ (reduce + (map ::duration tests)) span-ns))) + utilization-external (float (* 100 (/ (reduce + (map ::duration tests)) total-duration))) + ]] + + (println (format "Thread %s ran from %s to %s (%s), utilizing %.2f%% (internal) and %.2f%% (external)" + thread start end span utilization utilization-external))) + (flatten) + (apply str) + print) + + + (println (format "\n%d threads ran in %f seconds." (count threads) (float (/ total-duration 1e9)))) + (flush) + + )) + result)) + +(.until (Instant/now) (Instant/now) ChronoUnit/NANOS) From fbfd541eff80a63a674ab9560259dde3ef327423 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 12 May 2023 14:48:10 -0400 Subject: [PATCH 20/21] Post-rebase fixups --- src/kaocha/testable.clj | 10 +++++----- test/unit/kaocha/config_test.clj | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index de95a04b..bab03499 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -123,8 +123,8 @@ (assoc testable ::load-error t) (throw t))))) -(s/fdef load - :args (s/cat :testable :kaocha/testable) +(spec/fdef load + :args (spec/cat :testable :kaocha/testable) :ret :kaocha.test-plan/testable) (defmulti -run @@ -267,8 +267,8 @@ (defn current-thread-info [] (let [thread (Thread/currentThread)] - {:name (.getName thread) - :id (.getId thread) + {:name (.getName thread) + :id (.getId thread) :group-name (.getName (.getThreadGroup thread))})) (defn run-testables-parallel @@ -277,7 +277,7 @@ (doall testables) (let [load-error? (some ::load-error testables) types (set (:parallel-children-exclude *config*)) - suites (:parallel-suites-exclude *config*) + suites (:parallel-suites-exclude *config*) futures (map #(do (future (binding [*config* diff --git a/test/unit/kaocha/config_test.clj b/test/unit/kaocha/config_test.clj index b666147c..478702b5 100644 --- a/test/unit/kaocha/config_test.clj +++ b/test/unit/kaocha/config_test.clj @@ -66,7 +66,7 @@ (deftest merge-ns-patterns-issue-124-test (testing "https://github.com/lambdaisland/kaocha/issues/124" - (is (= #:kaocha{:early-exit 250} + (is (= #:kaocha{:early-exit 252} (try+ (config/merge-config {:kaocha/ns-patterns "test"} {:kaocha/ns-patterns "test"}) (catch :kaocha/early-exit e @@ -158,11 +158,11 @@ :kaocha/plugins (m/embeds [:some.kaocha.plugin/qux :other.kaocha.plugin/bar])} (config/load-config-for-cli-and-validate (io/resource "kaocha/config/loaded-test-resource.edn") {})))) (testing "falls back to default when resource does not exist" - (is (match? - ;; Deliberately minimal case because we want to test this behavior - ;; (fallback to tests.edn) without tying too much to tests.edn - {:kaocha.hooks/pre-load ['kaocha.assertions/load-assertions] } - (config/load-config-for-cli-and-validate (io/resource "resource-that-does-not-exist.edn") {}))))) + (is (match? + ;; Deliberately minimal case because we want to test this behavior + ;; (fallback to tests.edn) without tying too much to tests.edn + {:kaocha.hooks/pre-load ['kaocha.assertions/load-assertions] } + (config/load-config-for-cli-and-validate (io/resource "resource-that-does-not-exist.edn") {}))))) (testing "loading a file with profiles" (testing "specifying a profile" (is (match? {:kaocha/reporter 'kaocha.report.progress/report} From 615ea82040d08f819a4df85f01b71ace0e5e90ac Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 12 May 2023 16:53:12 -0400 Subject: [PATCH 21/21] Polish up parallelization: clearer config, docs, vestigial code Clean up the changes around parallelization, make opting-in from test types explicit, allow config on testable and metadata level, simplify future handling. --- doc/03_configuration.md | 3 +- doc/11_parallelization.md | 85 +++++++++++++++++++++++++++ src/kaocha/api.clj | 22 +++---- src/kaocha/config.clj | 24 ++++---- src/kaocha/runner.clj | 2 +- src/kaocha/test_suite.clj | 4 +- src/kaocha/testable.clj | 96 ++++++++++++------------------- src/kaocha/type/clojure/test.clj | 1 + src/kaocha/type/ns.clj | 15 +++-- test/unit/kaocha/type/ns_test.clj | 10 ++-- test/unit/kaocha/watch_test.clj | 8 +-- 11 files changed, 166 insertions(+), 104 deletions(-) create mode 100644 doc/11_parallelization.md diff --git a/doc/03_configuration.md b/doc/03_configuration.md index fce20040..e69a9472 100644 --- a/doc/03_configuration.md +++ b/doc/03_configuration.md @@ -29,7 +29,8 @@ Here's an example test configuration with a single test suite: :kaocha.plugin.randomize/seed 950716166 :kaocha.plugin.randomize/randomize? true :kaocha.plugin.profiling/count 3 - :kaocha.plugin.profiling/profiling? true} + :kaocha.plugin.profiling/profiling? true + :kaocha/parallize? false} ``` Writing a full test configuration by hand is tedious, which is why in diff --git a/doc/11_parallelization.md b/doc/11_parallelization.md new file mode 100644 index 00000000..48e6ba2b --- /dev/null +++ b/doc/11_parallelization.md @@ -0,0 +1,85 @@ +# 11. Parallelization + +Parallelization is an optional Kaocha feature, where it distributes your test +workload across multiple threads, to make better use of multiple CPU cores. + +This is still a relatively new feature, one that has a chance of interfering in +various ways with plugins, custom hooks, or the particulars of test setups that +people have "in the wild". We very much welcome feedback and improvements. +Please be mindful and respectful of the maintainers though and report issues +clearly, preferably with a link to a git repository containing a minimal +reproduction of the issue. + +## Configuration and high-level behavior + +You can enable parallelization either via the `--parallelize` command line flag, +or by setting `:parallelize? true` in your `tests.edn`. This is assuming that +you are using the `#kaocha/v1` reader literal to provide normalization. The +canonical configuration key is `:kaocha/parallelize?`. + +Kaocha looks at your tests as a hierarchy, at the top level there are your test +suites (e.g. unit vs intergration, or clj vs cljs). These contain groups of +tests (their children), e.g. one for each namespace, and these in turn contain +multiple tests, e.g. one for each test var. + +Setting `:parallelize true?` at the top-level configuration, or using the +command line flag, will run any suites you have in parallel, as well making +parallelization the default for any type of testable that has children. So say +for instance you have a suite of type `clojure.test`, then multiple test +namespaces will be run in parallel, and individual test vars within those +namespaces will also be started in parallel. + +Test type implementations need to opt-in to parallelization. For instance, +Clojure is multi-threaded, but ClojureScript (running on a JavaScript runtime) +is not, so thre is little benefit in trying to parallelize ClojureScript tests. +So even with parallelization on, `kaocha.cljs` or `kaocha.cljs2` tests will +still run in series. + +## Fine-grained opting in or out + +Using the command line flag or setting `:parallelize? true` at the top-level of +tests.edn will cause any testable that is parallelizable to be run in parallel. +If you want more fine-grained control you can configure specific test suites to +be parallelized, or set metadata on namespaces to opt in or out of +parallelization. + +```clj +#kaocha/v1 +{:tests [{:id :unit, :parallelize? true}]) +``` + +This will cause all namespaces in the unit test suite to be run in parallel, but +since the default (top-level config) is not set, vars within those namespaces +will not run in parallel. But you can again opt-in at that specific level, +through metadata. + +```clj +(ns ^{:kaocha/parallelize? true} my-test + (:require [clojure.test :as t])) + +... +``` + +Conversely you can opt-out of parallelization on the test suite or test +namespace level by setting `:kaocha/parallelize? false`. + +## Caveats + +When you start running your tests in parallel you will likely notice one or two +things. The first is that your output looks all messed up. Before you might see +something like `[(....)(.......)][(.......)]`, whereas now it looks more like +`[[(..(..).....(..)...]....)]`. This will be even more pronounced if you are for +instance using the documentation reporter. Output from multiple tests gets +interleaved, causing a garbled mess. + +The default dots reporter is probably the most usable reporter right now. + +The second thing you might notice is that you are getting failures where before +you got none. This likely indicates that your tests themselves are not thread +safe. They may for instance be dealing with shared mutable state. + +You will have to examine your code carefully. Starting out with a more piecemeal +opting in might be helpful to narrow things down. + +It is also possible that you encounters failures caused by Kaocha itself. In +that case please report them on our issue tracker. diff --git a/src/kaocha/api.clj b/src/kaocha/api.clj index 9f3c00a7..84922f5e 100644 --- a/src/kaocha/api.clj +++ b/src/kaocha/api.clj @@ -123,11 +123,6 @@ " To investigate, check the :test-paths and :ns-patterns keys in tests.edn."))) (throw+ {:kaocha/early-exit 0})) - (when (:parallel config) - (output/warn (str "Parallelization enabled. This feature is in " - "beta If you encounter errors, try " - "running with the feature disabled and " - "consider filing an issue."))) (when (find-ns 'matcher-combinators.core) (require 'kaocha.matcher-combinators)) @@ -142,8 +137,7 @@ ;; don't know where in the process we've ;; been interrupted, output capturing may ;; still be in effect. - (System/setOut - orig-out) + (System/setOut orig-out) (System/setErr orig-err) (binding [history/*history* history] @@ -160,13 +154,13 @@ on-exit) (let [test-plan (plugin/run-hook :kaocha.hooks/pre-run test-plan)] (binding [testable/*test-plan* test-plan] - (let [test-plan-tests (:kaocha.test-plan/tests test-plan) - result-tests (testable/run-testables test-plan-tests test-plan) - result (plugin/run-hook :kaocha.hooks/post-run - (-> test-plan - (dissoc :kaocha.test-plan/tests) - (assoc :kaocha.result/tests result-tests)))] - (assert (= (count test-plan-tests) (count (:kaocha.result/tests result)))) + (let [result-tests (testable/run-testables-parent test-plan test-plan) + result (plugin/run-hook :kaocha.hooks/post-run + (-> test-plan + (dissoc :kaocha.test-plan/tests) + (assoc :kaocha.result/tests result-tests)))] + (assert (= (count (:kaocha.test-plan/tests test-plan)) + (count (:kaocha.result/tests result)))) (-> result result/testable-totals result/totals->clojure-test-summary diff --git a/src/kaocha/config.clj b/src/kaocha/config.clj index 2756bdce..5e9c84e6 100644 --- a/src/kaocha/config.clj +++ b/src/kaocha/config.clj @@ -55,7 +55,8 @@ (rename-key :skip :kaocha.filter/skip) (rename-key :focus :kaocha.filter/focus) (rename-key :skip-meta :kaocha.filter/skip-meta) - (rename-key :focus-meta :kaocha.filter/focus-meta))] + (rename-key :focus-meta :kaocha.filter/focus-meta) + (rename-key :parallelize? :kaocha/parallelize?))] (as-> m $ (merge-config (first (:kaocha/tests (default-config))) $) (merge {:kaocha.testable/desc (str (name (:kaocha.testable/id $)) @@ -82,7 +83,8 @@ randomize? capture-output? watch? - bindings]} config + bindings + parallelize?]} config tests (some->> tests (mapv normalize-test-suite))] (cond-> {} tests (assoc :kaocha/tests (vary-meta tests assoc :replace true)) @@ -95,6 +97,7 @@ (some? watch?) (assoc :kaocha/watch? watch?) (some? randomize?) (assoc :kaocha.plugin.randomize/randomize? randomize?) (some? capture-output?) (assoc :kaocha.plugin.capture-output/capture-output? capture-output?) + (some? parallelize?) (assoc :kaocha/parallelize? parallelize?) :-> (merge (dissoc config :tests :plugins :reporter :color? :fail-fast? :watch? :randomize?))))) (defmethod aero/reader 'kaocha [_opts _tag value] @@ -195,17 +198,16 @@ config (read-config nil opts)))) - (defn apply-cli-opts [config options] (cond-> config - (some? (:fail-fast options)) (assoc :kaocha/fail-fast? (:fail-fast options)) - (:reporter options) (assoc :kaocha/reporter (:reporter options)) - (:watch options) (assoc :kaocha/watch? (:watch options)) - (some? (:color options)) (assoc :kaocha/color? (:color options)) - (some? (:diff-style options)) (assoc :kaocha/diff-style (:diff-style options)) - (:plugin options) (update :kaocha/plugins #(distinct (concat % (:plugin options)))) - (some? (:parallel options)) (assoc :parallel (:parallel options)) - true (assoc :kaocha/cli-options options))) + (some? (:fail-fast options)) (assoc :kaocha/fail-fast? (:fail-fast options)) + (:reporter options) (assoc :kaocha/reporter (:reporter options)) + (:watch options) (assoc :kaocha/watch? (:watch options)) + (some? (:color options)) (assoc :kaocha/color? (:color options)) + (some? (:diff-style options)) (assoc :kaocha/diff-style (:diff-style options)) + (:plugin options) (update :kaocha/plugins #(distinct (concat % (:plugin options)))) + (some? (:parallelize options)) (assoc :kaocha/parallelize? (:parallelize options)) + true (assoc :kaocha/cli-options options))) (defn apply-cli-args [config args] (if (seq args) diff --git a/src/kaocha/runner.clj b/src/kaocha/runner.clj index 184111ee..cdcc9941 100644 --- a/src/kaocha/runner.clj +++ b/src/kaocha/runner.clj @@ -34,7 +34,7 @@ [nil "--[no-]fail-fast" "Stop testing after the first failure."] [nil "--[no-]color" "Enable/disable ANSI color codes in output. Defaults to true."] [nil "--[no-]watch" "Watch filesystem for changes and re-run tests."] - [nil "--[no-]parallel" "Run tests in parallel. Warning: This feature is beta."] + [nil "--[no-]parallelize" "Run tests in parallel."] [nil "--reporter SYMBOL" "Change the test reporter, can be specified multiple times." :parse-fn (fn [s] (let [sym (symbol s)] diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index 56ffe354..fc90e991 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -4,10 +4,10 @@ (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) - (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) + (let [results (testable/run-testables-parent testable test-plan) testable (-> testable (dissoc :kaocha.test-plan/tests) (assoc :kaocha.result/tests results))] (t/do-report {:type :end-test-suite :kaocha/testable testable}) - testable)) + testable)) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index bab03499..002575b3 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -49,22 +49,6 @@ (try-require (symbol (namespace type)))) (try-require (symbol (name type))))) -(defn- retry-assert-spec [type testable n] - (let [result (try (assert-spec type testable) (catch Exception _e false))] - (if (or result (<= n 1)) result - (retry-assert-spec type testable (dec n))) ;otherwise, retry -)) - -(defn deref-recur [testables] - (cond (future? testables) (deref testables) - (vector? testables) (doall (mapv deref-recur testables)) - (seq? testables) (deref-recur (into [] (doall testables))) - (contains? testables :kaocha.test-plan/tests) - (update testables :kaocha.test-plan/tests deref-recur) - (contains? testables :kaocha.result/tests) - (update testables :kaocha.result/tests deref-recur) - :else testables)) - (defn- load-type+validate "Try to load a testable type, and validate it both to be a valid generic testable, and a valid instance given the type. @@ -162,9 +146,9 @@ result)))) (spec/fdef run - :args (spec/cat :testable :kaocha.test-plan/testable - :test-plan :kaocha/test-plan) - :ret :kaocha.result/testable) + :args (spec/cat :testable :kaocha.test-plan/testable + :test-plan :kaocha/test-plan) + :ret :kaocha.result/testable) (defn load-testables "Load a collection of testables, returning a test-plan collection" @@ -240,18 +224,15 @@ (defn try-run-testable [test test-plan n] (let [result (try (run-testable test test-plan) (catch Exception _e false))] - (if (or result (> n 1)) result ;success or last try, return - (try-run-testable test test-plan (dec n))) ;otherwise retry -)) - + (if (or result (> n 1)) + ;; success or last try, return + result + ;; otherwise retry + (try-run-testable test test-plan (dec n))))) (defn run-testables-serial "Run a collection of testables, returning a result collection." [testables test-plan] - (doall testables) - #_(print "run-testables got a collection of size" (count testables) - " the first of which is " - (:kaocha.testable/type (first testables))) (let [load-error? (some ::load-error testables)] (loop [result [] [test & testables] testables] @@ -261,7 +242,7 @@ (assoc ::skip true)) r (run-testable test test-plan)] (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - (reduce into result [[r] testables]) + (into (conj result r) testables) (recur (conj result r) testables))) result)))) @@ -272,41 +253,36 @@ :group-name (.getName (.getThreadGroup thread))})) (defn run-testables-parallel - "Run a collection of testables, returning a result collection." + "Run a collection of testables in parallel, returning a result collection." [testables test-plan] - (doall testables) (let [load-error? (some ::load-error testables) - types (set (:parallel-children-exclude *config*)) - suites (:parallel-suites-exclude *config*) - futures (map #(do + futures (doall + (map (fn [t] (future - (binding [*config* - (cond-> *config* - (contains? types (:kaocha.testable/type %)) (dissoc :parallel) - (and (hierarchy/suite? %) (contains? suites (:kaocha.testable/desc %))) (dissoc :parallel) - true (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] - (run-testable (assoc % ::thread (current-thread-info) ) test-plan)))) - testables)] - (comment (loop [result [] ;(ArrayBlockingQueue. 1024) - [test & testables] testables] - (if test - (let [test (cond-> test - (and load-error? (not (::load-error test))) - (assoc ::skip true)) - r (run-testable test test-plan)] - (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - ;(reduce put-return result [[r] testables]) - (reduce into result [[r] testables]) - ;(recur (doto result (.put r)) testables) - (recur (conj result r) testables))) - result))) - (deref-recur futures))) + (run-testable (assoc t ::thread-info (current-thread-info)) test-plan))) + testables))] + (doall (map deref futures)))) (defn run-testables + "Original run-testables, left for backwards compatibility, and still usable for + test types that don't want to opt-in to parallelization. Generally + implementations should move to [[run-testables-parent]]." [testables test-plan] - (if (:parallel *config*) - (doall (run-testables-parallel testables test-plan)) - (run-testables-serial testables test-plan))) + (run-testables-serial testables test-plan)) + +(defn run-testables-parent + "Test type implementations should call this in their [[-run]] method, rather + than [[run-testables]], so we can inspect the parent and parent metadata to + decide if the children should get parallelized." + [parent test-plan] + (let [testables (:kaocha.test-plan/tests parent)] + (if (or (true? (:kaocha/parallelize? (::meta parent))) ; explicit opt-in via metadata + (and (:kaocha/parallelize? test-plan) ; enable parallelization in top-level config + (or (::parallelizable? parent) ; test type has opted in, children are considered parallelizable + (:kaocha/parallelize? parent)) ; or we're at the top level, suites are parallelizable. Can also be used as an explicit override/opt-in + (not (false? (:kaocha/parallelize? (::meta parent)))))) ; explicit opt-out via metadata + (run-testables-parallel testables test-plan) + (run-testables-serial testables test-plan)))) (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) @@ -319,12 +295,12 @@ (defn test-seq-with-skipped [testable] - "Create a seq of all tests, including any skipped tests. + "Create a seq of all tests, including any skipped tests. Typically you want to look at `test-seq` instead." (cond->> (mapcat test-seq (or (:kaocha/tests testable) - (:kaocha.test-plan/tests testable) - (:kaocha.result/tests testable))) + (:kaocha.test-plan/tests testable) + (:kaocha.result/tests testable))) ;; When calling test-seq on the top level test-plan/result, don't include ;; the outer map. When running on an actual testable, do include it. (:kaocha.testable/id testable) diff --git a/src/kaocha/type/clojure/test.clj b/src/kaocha/type/clojure/test.clj index e6c7712c..66d29d80 100644 --- a/src/kaocha/type/clojure/test.clj +++ b/src/kaocha/type/clojure/test.clj @@ -11,6 +11,7 @@ (defmethod testable/-load :kaocha.type/clojure.test [testable] (-> testable + (assoc :kaocha.testable/parallelizable? true) (load/load-test-namespaces type.ns/->testable) (testable/add-desc "clojure.test"))) diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index 470489c9..92f08c96 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -8,16 +8,19 @@ [kaocha.type :as type])) (defn ->testable [ns-name] - {:kaocha.testable/type :kaocha.type/ns - :kaocha.testable/id (keyword (str ns-name)) - :kaocha.testable/desc (str ns-name) - :kaocha.ns/name ns-name}) + {:kaocha.testable/type :kaocha.type/ns + :kaocha.testable/id (keyword (str ns-name)) + :kaocha.testable/desc (str ns-name) + :kaocha.testable/parallelizable? true + :kaocha.ns/name ns-name}) (defn run-tests [testable test-plan fixture-fn] ;; It's not guaranteed the the fixture-fn returns the result of calling the ;; tests function, so we need to put it in a box for reference. - (let [result (atom (:kaocha.test-plan/tests testable))] - (fixture-fn #(swap! result testable/run-testables test-plan)) + (let [result (promise)] + (fixture-fn + (fn [] + (deliver result (testable/run-testables-parent testable test-plan)))) @result)) (defmethod testable/-load :kaocha.type/ns [testable] diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index dc1c1414..fbd02cfc 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -68,7 +68,7 @@ (with-test-ctx {:fail-fast? true} (testable/run testable testable))))))) -(deftest run-test-parallel +(deftest run-test-parallel (classpath/add-classpath "fixtures/f-tests") (let [testable (testable/load {:kaocha.testable/type :kaocha.type/clojure.test @@ -97,7 +97,7 @@ (with-test-ctx {:fail-fast? true} (testable/run testable testable))))) (is (not (nil? (:result - (binding [testable/*config* (assoc testable/*config* :parallel true)] - (with-test-ctx {:fail-fast? true - :parallel true} - (testable/run testable testable))))))))) + (binding [testable/*config* (assoc testable/*config* :parallel true)] + (with-test-ctx {:fail-fast? true + :parallel true} + (testable/run testable testable))))))))) diff --git a/test/unit/kaocha/watch_test.clj b/test/unit/kaocha/watch_test.clj index e5678cf9..0c8bb049 100644 --- a/test/unit/kaocha/watch_test.clj +++ b/test/unit/kaocha/watch_test.clj @@ -133,7 +133,7 @@ (is (str/includes? @out-str - (str/replace + (str/replace (str/replace "[(F)]\n\nFAIL in foo.bar-test/xxx-test (bar_test.clj:1)\nExpected:\n :xxx\nActual:\n -:xxx +:yyy\n1 tests, 1 assertions, 1 failures.\n\nbin/kaocha --config-file PATH --focus 'foo.bar-test/xxx-test'\n\n[watch] Reloading #{foo.bar-test}\n[watch] Re-running failed tests #{:foo.bar-test/xxx-test}\n[(F)]\n\nFAIL in foo.bar-test/xxx-test (bar_test.clj:1)\nExpected:\n :xxx\nActual:\n -:xxx +:zzz" "foo" @@ -155,8 +155,8 @@ (is (= #{"one" "two"} (set (w/merge-ignore-files (str test-dir))))))) (deftest watch-set-dynamic-vars-test - ; sanity test for #133. Should succeed when this file - ; is checked via ./bin/kaocha with --watch mode + ;; sanity test for #133. Should succeed when this file + ;; is checked via ./bin/kaocha with --watch mode (is (do (set! *warn-on-reflection* false) true)) (let [{:keys [config-file test-dir] :as m} (integration/test-dir-setup {}) @@ -203,7 +203,7 @@ (let [orig-config (config/load-config-for-cli-and-validate "test/unit/kaocha/config/loaded-test-profile.edn" {:profile :test}) [reloaded-config _] (w/reload-config orig-config nil)] (is (= orig-config reloaded-config))))) - + ;;TODO move to cucumber (deftest ^{:min-java-version "1.11"} watch-load-error-test (let [{:keys [config-file test-dir] :as m} (integration/test-dir-setup {})