diff --git a/DESCRIPTION b/DESCRIPTION
index a85039c..365518c 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -37,4 +37,4 @@ Config/Needs/website: tidyverse/tidytemplate
 Config/testthat/edition: 3
 Encoding: UTF-8
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.1
diff --git a/R/eval.R b/R/eval.R
index 8d2b84f..31a8d7c 100644
--- a/R/eval.R
+++ b/R/eval.R
@@ -238,14 +238,7 @@ evaluate_call <- function(call,
     timing_fn <- function(x) {x; NULL};
   }
 
-  if (length(funs <- .env$inject_funs)) {
-    funs_names <- names(funs)
-    funs_new <- !vapply(funs_names, exists, logical(1), envir, inherits = FALSE)
-    funs_names <- funs_names[funs_new]
-    funs <- funs[funs_new]
-    on.exit(rm(list = funs_names, envir = envir), add = TRUE)
-    for (i in seq_along(funs_names)) assign(funs_names[i], funs[[i]], envir)
-  }
+  local_inject_funs(envir)
 
   user_handlers <- output_handler$calling_handlers
 
@@ -304,34 +297,6 @@ eval_with_user_handlers <- function(expr, envir, enclos, calling_handlers) {
   eval(call)
 }
 
-#' Inject functions into the environment of `evaluate()`
-#'
-#' Create functions in the environment specified in the `envir` argument of
-#' [evaluate()]. This can be helpful if you want to substitute certain
-#' functions when evaluating the code. To make sure it does not wipe out
-#' existing functions in the environment, only functions that do not exist in
-#' the environment are injected.
-#' @param ... Named arguments of functions. If empty, previously injected
-#'   functions will be emptied.
-#' @note For expert use only. Do not use it unless you clearly understand it.
-#' @keywords internal
-#' @examples library(evaluate)
-#' # normally you cannot capture the output of system
-#' evaluate("system('R --version')")
-#'
-#' # replace the system() function
-#' inject_funs(system = function(...) cat(base::system(..., intern = TRUE), sep = '\n'))
-#'
-#' evaluate("system('R --version')")
-#'
-#' inject_funs()  # empty previously injected functions
-#' @export
-inject_funs <- function(...) {
-  funs <- list(...)
-  funs <- funs[names(funs) != '']
-  .env$inject_funs <- Filter(is.function, funs)
-}
-
 format_warning <- function(x) {
   if (inherits(x, "rlang_warning")) {
     format(x)
diff --git a/R/inject-funs.R b/R/inject-funs.R
new file mode 100644
index 0000000..cbbabd9
--- /dev/null
+++ b/R/inject-funs.R
@@ -0,0 +1,54 @@
+#' Inject functions into the environment of `evaluate()`
+#'
+#' Create functions in the environment specified in the `envir` argument of
+#' [evaluate()]. This can be helpful if you want to substitute certain
+#' functions when evaluating the code. To make sure it does not wipe out
+#' existing functions in the environment, only functions that do not exist in
+#' the environment are injected.
+#' @param ... Named arguments of functions. If empty, previously injected
+#'   functions will be emptied.
+#' @note For expert use only. Do not use it unless you clearly understand it.
+#' @keywords internal
+#' @return Invisibly returns previous values.
+#' @examples library(evaluate)
+#' # normally you cannot capture the output of system
+#' evaluate("system('R --version')")
+#'
+#' # replace the system() function
+#' old <- inject_funs(system = function(...) {
+#'   cat(base::system(..., intern = TRUE), sep = '\n')
+#' })
+#'
+#' evaluate("system('R --version')")
+#'
+#' # restore previously injected functions
+#' inject_funs(old) 
+#' @export
+inject_funs <- function(...) {
+  funs <- list(...)
+  funs <- funs[names(funs) != '']
+  old <- .env$inject_funs
+  .env$inject_funs <- Filter(is.function, funs)
+
+  invisible(old)
+}
+
+local_inject_funs <- function(envir, frame = parent.frame()) {
+  funs <- .env$inject_funs
+  if (length(funs) == 0) {
+    return()
+  }
+
+  funs_names <- names(funs)
+  funs_new <- !vapply(funs_names, exists, logical(1), envir, inherits = FALSE)
+  funs_names <- funs_names[funs_new]
+  funs <- funs[funs_new]
+  
+  defer(rm(list = funs_names, envir = envir), frame = frame)
+  
+  for (i in seq_along(funs_names)) {
+    assign(funs_names[i], funs[[i]], envir)
+  }
+  
+  invisible()
+}
diff --git a/R/utils.R b/R/utils.R
new file mode 100644
index 0000000..c46dda2
--- /dev/null
+++ b/R/utils.R
@@ -0,0 +1,4 @@
+defer <- function(expr, frame = parent.frame(), after = FALSE) {
+  thunk <- as.call(list(function() expr))
+  do.call(on.exit, list(thunk, TRUE, after), envir = frame)
+}
diff --git a/man/inject_funs.Rd b/man/inject_funs.Rd
index ec6b1ab..5f1c2ee 100644
--- a/man/inject_funs.Rd
+++ b/man/inject_funs.Rd
@@ -1,5 +1,5 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/eval.R
+% Please edit documentation in R/inject-funs.R
 \name{inject_funs}
 \alias{inject_funs}
 \title{Inject functions into the environment of \code{evaluate()}}
@@ -10,6 +10,9 @@ inject_funs(...)
 \item{...}{Named arguments of functions. If empty, previously injected
 functions will be emptied.}
 }
+\value{
+Invisibly returns previous values.
+}
 \description{
 Create functions in the environment specified in the \code{envir} argument of
 \code{\link[=evaluate]{evaluate()}}. This can be helpful if you want to substitute certain
@@ -26,10 +29,13 @@ library(evaluate)
 evaluate("system('R --version')")
 
 # replace the system() function
-inject_funs(system = function(...) cat(base::system(..., intern = TRUE), sep = '\n'))
+old <- inject_funs(system = function(...) {
+  cat(base::system(..., intern = TRUE), sep = '\n')
+})
 
 evaluate("system('R --version')")
 
-inject_funs()  # empty previously injected functions
+# restore previously injected functions
+inject_funs(old) 
 }
 \keyword{internal}
diff --git a/tests/testthat/test-inject-funs.R b/tests/testthat/test-inject-funs.R
new file mode 100644
index 0000000..a182507
--- /dev/null
+++ b/tests/testthat/test-inject-funs.R
@@ -0,0 +1,7 @@
+test_that("can inject functons into evaluation context", {
+  old <- inject_funs(f = function() 1)
+  defer(inject_funs(old))
+
+  ev <- evaluate("f()")
+  expect_equal(ev[[2]], "[1] 1\n")
+})