diff --git a/.gitignore b/.gitignore index a96d1d7..0beb0f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /luacov.* -/site \ No newline at end of file +/site +/lua_install \ No newline at end of file diff --git a/.luacheckrc b/.luacheckrc index 9c4b496..d0f2bc1 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -12,8 +12,18 @@ stds.roblox = { } } +stds.testez = { + read_globals = { + "it", "describe", "beforeAll", "beforeEach", "afterAll", "afterEach", + }, +} + std = "lua51+roblox" ignore = { "212", -- Unused argument, which triggers on unused 'self' too +} + +files["tests/lifecycleHooks.lua"] = { + std = "+testez", } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 98c8ec7..dbba00d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # TestEZ Changelog ## Current master -* No changes +* Added beforeAll, beforeEach, afterEach, afterAll lifecycle hooks for testing + * The setup and teardown behavior of these hooks attempt to reach feature parity with [jest](https://jestjs.io/docs/en/setup-teardown). + ## 0.1.0 (2019-11-01) -* Initial release. \ No newline at end of file +* Initial release. diff --git a/lib/LifecycleHooks.lua b/lib/LifecycleHooks.lua new file mode 100644 index 0000000..3ef05fe --- /dev/null +++ b/lib/LifecycleHooks.lua @@ -0,0 +1,169 @@ +local TestEnum = require(script.Parent.TestEnum) + +local LifecycleHooks = {} +LifecycleHooks.__index = LifecycleHooks + +function LifecycleHooks.new() + local self = { + _stack = {}, + } + setmetatable(self, LifecycleHooks) + return self +end + +--[[ + Returns an array of `beforeEach` hooks in FIFO order +]] +function LifecycleHooks:getBeforeEachHooks() + local key = TestEnum.NodeType.BeforeEach + local hooks = {} + + for _, level in ipairs(self._stack) do + for _, hook in ipairs(level[key]) do + table.insert(hooks, hook) + end + end + + return hooks +end + +--[[ + Returns an array of `afterEach` hooks in FILO order +]] +function LifecycleHooks:getAfterEachHooks() + local key = TestEnum.NodeType.AfterEach + local hooks = {} + for _, level in ipairs(self._stack) do + for _, hook in ipairs(level[key]) do + table.insert(hooks, 1, hook) + end + end + + return hooks +end + +--[[ + Pushes uncalled beforeAll and afterAll hooks back up the stack +]] +function LifecycleHooks:popHooks() + local popped = self._stack[#self._stack] + table.remove(self._stack, #self._stack) + + local function pushHooksUp(type) + + local back = self:_getBackOfStack() + + if not back then + return + end + + back[type] = popped[type] + end + + pushHooksUp(TestEnum.NodeType.BeforeAll) + pushHooksUp(TestEnum.NodeType.AfterAll) +end + +function LifecycleHooks:pushHooksFrom(planNode) + assert(planNode ~= nil) + + table.insert(self._stack, { + [TestEnum.NodeType.BeforeAll] = self:_getBeforeAllHooksUncalledAtCurrentLevel(planNode.children), + [TestEnum.NodeType.AfterAll] = self:_getAfterAllHooksUncalledAtCurrentLevel(planNode.children), + [TestEnum.NodeType.BeforeEach] = self:_getHooksOfType(planNode.children, TestEnum.NodeType.BeforeEach), + [TestEnum.NodeType.AfterEach] = self:_getHooksOfType(planNode.children, TestEnum.NodeType.AfterEach), + }) +end + +function LifecycleHooks:getPendingBeforeAllHooks() + return self:_getAndClearPendingHooks(TestEnum.NodeType.BeforeAll) +end + +function LifecycleHooks:getAfterAllHooks() + if #self._stack > 0 then + return self:_getAndClearPendingHooks(TestEnum.NodeType.AfterAll) + else + return {} + end +end + +--[[ + Return any hooks that have not yet been returned for this key and clear those hooks +]] +function LifecycleHooks:_getAndClearPendingHooks(key) + assert(key ~= nil) + + if #self._stack > 0 then + + local back = self._stack[#self._stack] + + local hooks = back[key] + + back[key] = {} + + return hooks + else + return {} + end +end + +--[[ + Transfers uncalled beforeAll and afterAll hooks down the stack +]] +function LifecycleHooks:_getBeforeAllHooksUncalledAtCurrentLevel(childNodes) + local hookType = TestEnum.NodeType.BeforeAll + local hooks = self:_getHooksOfTypeFromBackOfStack(hookType) + + for _, hook in pairs(self:_getHooksOfType(childNodes, hookType)) do + table.insert(hooks, hook) + end + + return hooks +end + +function LifecycleHooks:_getAfterAllHooksUncalledAtCurrentLevel(childNodes) + local hookType = TestEnum.NodeType.AfterAll + local hooks = self:_getHooksOfTypeFromBackOfStack(hookType) + + for _, hook in pairs(self:_getHooksOfType(childNodes, hookType)) do + table.insert(hooks, 1, hook) + end + + return hooks +end + +function LifecycleHooks:_getHooksOfTypeFromBackOfStack(hookType) + assert(hookType, "Expected hookType to be an argument") + + local currentBack = self:_getBackOfStack() + + local hooks = {} + + if currentBack then + for _, hook in pairs(currentBack[hookType]) do + table.insert(hooks, hook) + end + + currentBack[hookType] = {} + end + + return hooks +end + +function LifecycleHooks:_getBackOfStack() + return self._stack[#self._stack] or nil +end + +function LifecycleHooks:_getHooksOfType(nodes, type) + local hooks = {} + + for _, node in pairs(nodes) do + if node.type == type then + table.insert(hooks, node.callback) + end + end + + return hooks +end + +return LifecycleHooks diff --git a/lib/TestEnum.lua b/lib/TestEnum.lua index 68fd276..30addab 100644 --- a/lib/TestEnum.lua +++ b/lib/TestEnum.lua @@ -13,7 +13,11 @@ TestEnum.TestStatus = { TestEnum.NodeType = { Try = "Try", Describe = "Describe", - It = "It" + It = "It", + BeforeAll = "BeforeAll", + AfterAll = "AfterAll", + BeforeEach = "BeforeEach", + AfterEach = "AfterEach" } TestEnum.NodeModifier = { diff --git a/lib/TestPlanner.lua b/lib/TestPlanner.lua index ada12ae..c0316ba 100644 --- a/lib/TestPlanner.lua +++ b/lib/TestPlanner.lua @@ -105,6 +105,27 @@ function TestPlanner.createEnvironment(builder, extraEnvironment) builder:popNode() end + -- Incrementing counter used to ensure that beforeAll, afterAll, beforeEach, afterEach have unique phrases + local lifecyclePhaseId = 0 + + local lifecycleHooks = { + [TestEnum.NodeType.BeforeAll] = "beforeAll", + [TestEnum.NodeType.AfterAll] = "afterAll", + [TestEnum.NodeType.BeforeEach] = "beforeEach", + [TestEnum.NodeType.AfterEach] = "afterEach" + } + + for nodeType, name in pairs(lifecycleHooks) do + env[name] = function(callback) + local node = builder:pushNode(name .. "_" .. tostring(lifecyclePhaseId), nodeType) + lifecyclePhaseId = lifecyclePhaseId + 1 + + node.callback = callback + + builder:popNode() + end + end + function env.itFOCUS(phrase, callback) local node = builder:pushNode(phrase, TestEnum.NodeType.It, TestEnum.NodeModifier.Focus) diff --git a/lib/TestResults.lua b/lib/TestResults.lua index ab8924c..c39c829 100644 --- a/lib/TestResults.lua +++ b/lib/TestResults.lua @@ -82,7 +82,7 @@ function TestResults:visualize(root, level) child.planNode.phrase ) - if #child.messages > 0 then + if child.messages and #child.messages > 0 then str = str .. "\n " .. (" "):rep(3 * level) .. table.concat(child.messages, "\n " .. (" "):rep(3 * level)) end @@ -90,7 +90,7 @@ function TestResults:visualize(root, level) else local str = ("%s%s"):format( (" "):rep(3 * level), - child.planNode.phrase + child.planNode.phrase or "" ) if child.status then diff --git a/lib/TestRunner.lua b/lib/TestRunner.lua index 8f7b806..b49ffec 100644 --- a/lib/TestRunner.lua +++ b/lib/TestRunner.lua @@ -10,6 +10,7 @@ local Expectation = require(script.Parent.Expectation) local TestEnum = require(script.Parent.TestEnum) local TestSession = require(script.Parent.TestSession) local Stack = require(script.Parent.Stack) +local LifecycleHooks = require(script.Parent.LifecycleHooks) local RUNNING_GLOBAL = "__TESTEZ_RUNNING_TEST__" @@ -28,6 +29,7 @@ end function TestRunner.runPlan(plan) local session = TestSession.new(plan) local tryStack = Stack.new() + local lifecycleHooks = LifecycleHooks.new() local exclusiveNodes = plan:findNodes(function(node) return node.modifier == TestEnum.NodeModifier.Focus @@ -35,7 +37,7 @@ function TestRunner.runPlan(plan) session.hasFocusNodes = #exclusiveNodes > 0 - TestRunner.runPlanNode(session, plan, tryStack) + TestRunner.runPlanNode(session, plan, tryStack, lifecycleHooks) return session:finalize() end @@ -44,7 +46,89 @@ end Run the given test plan node and its descendants, using the given test session to store all of the results. ]] -function TestRunner.runPlanNode(session, planNode, tryStack, noXpcall) +function TestRunner.runPlanNode(session, planNode, tryStack, lifecycleHooks, noXpcall) + -- We prefer xpcall, but yielding doesn't work from xpcall. + -- As a workaround, you can mark nodes as "not xpcallable" + local call = noXpcall and pcall or xpcall + + local function runCallback(callback, always, messagePrefix) + local success = true + local errorMessage + -- Any code can check RUNNING_GLOBAL to fork behavior based on + -- whether a test is running. We use this to avoid accessing + -- protected APIs; it's a workaround that will go away someday. + _G[RUNNING_GLOBAL] = true + + messagePrefix = messagePrefix or "" + + local testEnvironment = getfenv(callback) + + for key, value in pairs(TestRunner.environment) do + testEnvironment[key] = value + end + + testEnvironment.fail = function(message) + if message == nil then + message = "fail() was called." + end + + success = false + errorMessage = messagePrefix .. message .. "\n" .. debug.traceback() + end + + local nodeSuccess, nodeResult = call(callback, function(message) + return messagePrefix .. message .. "\n" .. debug.traceback() + end) + + -- If a node threw an error, we prefer to use that message over + -- one created by fail() if it was set. + if not nodeSuccess then + success = false + errorMessage = nodeResult + end + + _G[RUNNING_GLOBAL] = nil + + return success, errorMessage + end + + local function runNode(childPlanNode) + -- Errors can be set either via `error` propagating upwards or + -- by a test calling fail([message]). + + for _, hook in pairs(lifecycleHooks:getPendingBeforeAllHooks()) do + local success, errorMessage = runCallback(hook, false, "beforeAll hook: ") + if not success then + return false, errorMessage + end + end + + for _, hook in pairs(lifecycleHooks:getBeforeEachHooks()) do + local success, errorMessage = runCallback(hook, false, "beforeEach hook: ") + if not success then + return false, errorMessage + end + end + + do + local success, errorMessage = runCallback(childPlanNode.callback) + if not success then + return false, errorMessage + end + end + + for _, hook in pairs(lifecycleHooks:getAfterEachHooks()) do + local success, errorMessage = runCallback(hook, true, "afterEach hook: ") + if not success then + return false, errorMessage + end + end + + return true, nil + end + + lifecycleHooks:pushHooksFrom(planNode) + for _, childPlanNode in ipairs(planNode.children) do local childResultNode = session:pushNode(childPlanNode) @@ -53,52 +137,13 @@ function TestRunner.runPlanNode(session, planNode, tryStack, noXpcall) childResultNode.status = TestEnum.TestStatus.Skipped else if tryStack:size() > 0 and tryStack:getBack().isOk == false then + childResultNode.status = TestEnum.TestStatus.Failure table.insert(childResultNode.errors, string.format("%q failed without trying, because test case %q failed", childPlanNode.phrase, tryStack:getBack().failedNode.phrase)) else - -- Errors can be set either via `error` propagating upwards or - -- by a test calling fail([message]). - local success = true - local errorMessage - - local testEnvironment = getfenv(childPlanNode.callback) - - for key, value in pairs(TestRunner.environment) do - testEnvironment[key] = value - end - - testEnvironment.fail = function(message) - if message == nil then - message = "fail() was called." - end - - success = false - errorMessage = message .. "\n" .. debug.traceback() - end - - -- We prefer xpcall, but yielding doesn't work from xpcall. - -- As a workaround, you can mark nodes as "not xpcallable" - local call = noXpcall and pcall or xpcall - - -- Any code can check RUNNING_GLOBAL to fork behavior based on - -- whether a test is running. We use this to avoid accessing - -- protected APIs; it's a workaround that will go away someday. - _G[RUNNING_GLOBAL] = true - - local nodeSuccess, nodeResult = call(childPlanNode.callback, function(message) - return message .. "\n" .. debug.traceback() - end) - - _G[RUNNING_GLOBAL] = nil - - -- If a node threw an error, we prefer to use that message over - -- one created by fail() if it was set. - if not nodeSuccess then - success = false - errorMessage = nodeResult - end + local success, errorMessage = runNode(childPlanNode) if success then childResultNode.status = TestEnum.TestStatus.Success @@ -110,7 +155,7 @@ function TestRunner.runPlanNode(session, planNode, tryStack, noXpcall) end elseif childPlanNode.type == TestEnum.NodeType.Describe or childPlanNode.type == TestEnum.NodeType.Try then if childPlanNode.type == TestEnum.NodeType.Try then tryStack:push({isOk = true, failedNode = nil}) end - TestRunner.runPlanNode(session, childPlanNode, tryStack, childPlanNode.HACK_NO_XPCALL) + TestRunner.runPlanNode(session, childPlanNode, tryStack, lifecycleHooks, childPlanNode.HACK_NO_XPCALL) if childPlanNode.type == TestEnum.NodeType.Try then tryStack:pop() end local status = TestEnum.TestStatus.Success @@ -147,6 +192,14 @@ function TestRunner.runPlanNode(session, planNode, tryStack, noXpcall) session:popNode() end + + for _, hook in pairs(lifecycleHooks:getAfterAllHooks()) do + runCallback(hook, true, "afterAll hook: ") + -- errors in an afterAll hook are currently not caught + -- or attributed to a set of child nodes + end + + lifecycleHooks:popHooks() end return TestRunner \ No newline at end of file diff --git a/tests/lifecycleHooks.lua b/tests/lifecycleHooks.lua new file mode 100644 index 0000000..10acd63 --- /dev/null +++ b/tests/lifecycleHooks.lua @@ -0,0 +1,243 @@ +local TestEZ = require(script.Parent.Parent.TestEZ) + +local function expectShallowEquals(array1, array2) + local function shallowEquals() + for index, value in ipairs(array1) do + if array2[index] ~= value then + return false + end + end + + for index, value in ipairs(array2) do + if array1[index] ~= value then + return false + end + end + + return true + end + + if not shallowEquals() then + error(string.format( + "Expected: {\n\t%s\n}.\nGot: {\n\t%s\n}", + table.concat(array2, "\n\t"), + table.concat(array1, "\n\t") + )) + end +end + +local function expectNoFailures(results) + assert(results.failureCount == 0, "Some lifecycleHook test failed!") +end + +local function runTestPlan(testPlan) + local lifecycleOrder = {} + local function insertLifecycleEvent(lifecycleString) + table.insert(lifecycleOrder, lifecycleString) + end + + local plan = TestEZ.TestPlanner.createPlan({ + { + method = function() + testPlan(insertLifecycleEvent) + end, + path = {'lifecycleHooksTest'} + } + }) + + local results = TestEZ.TestRunner.runPlan(plan) + return results, lifecycleOrder +end + +return { + ["should run lifecycle methods in single-level"] = function() + local results, lifecycleOrder = runTestPlan(function(insertLifecycleEvent) + beforeAll(function() + insertLifecycleEvent("1 - beforeAll") + end) + + afterAll(function() + insertLifecycleEvent("1 - afterAll") + end) + + beforeEach(function() + insertLifecycleEvent("1 - beforeEach") + end) + + afterEach(function() + insertLifecycleEvent("1 - afterEach") + end) + + it("runs root", function() + insertLifecycleEvent("1 - test") + end) + end) + + expectShallowEquals(lifecycleOrder, { + "1 - beforeAll", + "1 - beforeEach", + "1 - test", + "1 - afterEach", + "1 - afterAll", + }) + + expectNoFailures(results) + end, + ["should run lifecycle methods in order in nested trees"] = function() + -- follows spec from jest https://jestjs.io/docs/en/setup-teardown#scoping + local results, lifecycleOrder = runTestPlan(function(insertLifecycleEvent) + beforeAll(function() + insertLifecycleEvent("1 - beforeAll") + end) + + afterAll(function() + insertLifecycleEvent("1 - afterAll") + end) + + beforeEach(function() + insertLifecycleEvent("1 - beforeEach") + end) + + afterEach(function() + insertLifecycleEvent("1 - afterEach") + end) + + it("runs root", function() + insertLifecycleEvent("1 - test") + end) + + describe("nestedDescribe", function() + beforeAll(function() + insertLifecycleEvent("2 - beforeAll") + end) + + afterAll(function() + insertLifecycleEvent("2 - afterAll") + end) + + beforeEach(function() + insertLifecycleEvent("2 - beforeEach") + end) + + afterEach(function() + insertLifecycleEvent("2 - afterEach") + end) + + it("runs", function() + insertLifecycleEvent("2 - test") + end) + + describe("no tests", function() + beforeAll(function() + insertLifecycleEvent("3 - beforeAll") + end) + end) + end) + end) + + expectShallowEquals(lifecycleOrder, { + "1 - beforeAll", + "1 - beforeEach", + "1 - test", + "1 - afterEach", + "2 - beforeAll", + "1 - beforeEach", + "2 - beforeEach", + "2 - test", + "2 - afterEach", + "1 - afterEach", + "2 - afterAll", + "1 - afterAll", + }) + expectNoFailures(results) + end, + ["beforeAll should only run once per describe block"] = function() + local results, lifecycleOrder = runTestPlan(function(insertLifecycleEvent) + beforeAll(function() + insertLifecycleEvent("1 - beforeAll") + end) + + it("runs 1", function() + insertLifecycleEvent("1 - test") + end) + + describe("nestedDescribe", function() + beforeAll(function() + insertLifecycleEvent("2 - beforeAll") + end) + + it("runs 2", function() + insertLifecycleEvent("2 - test") + end) + + it("runs 2 again", function() + insertLifecycleEvent("2 - test again") + end) + end) + end) + + expectShallowEquals(lifecycleOrder, { + "1 - beforeAll", + "1 - test", + "2 - beforeAll", + "2 - test", + "2 - test again", + }) + expectNoFailures(results) + end, + ["lifecycle failures should fail test node"] = function() + local function failLifecycleCase(hookType) + local itWasRun = false + local results = runTestPlan(function(insertLifecycleEvent) + + if hookType == "beforeAll" then + beforeAll(function() + error("this is an error") + end) + end + + if hookType == "beforeEach" then + beforeEach(function() + error("this is an error") + end) + end + + if hookType == "afterEach" then + afterEach(function() + error("this is an error") + end) + end + + if hookType == "afterAll" then + afterAll(function() + error("this is an error") + end) + end + + it("runs root", function() + itWasRun = true + end) + end) + + assert(results.failureCount == 1, string.format("Expected %s failure to fail test run", hookType)) + + if hookType:find("before") then + -- if before* hooks fail, our test node should not run + assert(itWasRun == false, "it node was ran despite failure on run: " .. hookType) + end + end + + failLifecycleCase("beforeAll") + failLifecycleCase("beforeEach") + failLifecycleCase("afterEach") + -- `afterAll` failure case is intentionally missing. + -- Currently it is not easy to attach an afterAll failure to + -- a particular set of childNodes without some refactoring. + -- Additionally, when jest afterAll hooks fail, it fails the test suite + -- and not any particular node which is a different flavor of failure + -- that TestEZ does not offer right now + -- Consult the following: + -- https://github.com/facebook/jest/issues/3266 + -- https://github.com/facebook/jest/pull/5884 + end, +} \ No newline at end of file