diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 80db876..0018c40 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -35,7 +35,9 @@ jobs:
     - name: Pre-commit checks
       run: pre-commit run --all-files
     - name: Test with pytest
+      # Create the ssh key file once for all testing
       run: |
+        ssh-keygen -t ecdsa -m PEM -N '' -f /tmp/buildrunner-test-id_rsa
         pytest -v -m "not serial" --numprocesses=auto --junitxml=test-reports/non-serial-test-results.xml
         pytest -v -m "serial" --junitxml=test-reports/serial-test-results.xml
         python scripts/combine_xml.py test-reports/serial-test-results.xml test-reports/non-serial-test-results.xml > test-reports/test-result.xml
diff --git a/tests/test_buildrunner_files.py b/tests/test_buildrunner_files.py
index 818ed73..0957312 100644
--- a/tests/test_buildrunner_files.py
+++ b/tests/test_buildrunner_files.py
@@ -26,29 +26,32 @@
 @pytest.fixture(autouse=True, scope="session")
 def setup_buildrunner_test_ssh_key():
     key_file_path = Path(TEST_SSH_KEY_FILE)
-    key_file_path.unlink(missing_ok=True)
-    subprocess.run(
-        [
-            "ssh-keygen",
-            "-t",
-            "ecdsa",
-            "-m",
-            "PEM",
-            "-N",
-            "",
-            "-f",
-            TEST_SSH_KEY_FILE,
-        ],
-        check=True,
-    )
+    cleanup_key_file = False
+    pub_key_file_path = Path(f"{TEST_SSH_KEY_FILE}.pub")
+    if not key_file_path.exists():
+        subprocess.run(
+            [
+                "ssh-keygen",
+                "-t",
+                "ecdsa",
+                "-m",
+                "PEM",
+                "-N",
+                "",
+                "-f",
+                TEST_SSH_KEY_FILE,
+            ],
+            check=True,
+        )
+        cleanup_key_file = True
     # Set the public key in an environment variable to use in the test buildrunner files
-    os.environ["BUILDRUNNER_TEST_SSH_PUB_KEY"] = (
-        Path(f"{TEST_SSH_KEY_FILE}.pub").read_text().strip()
-    )
+    os.environ["BUILDRUNNER_TEST_SSH_PUB_KEY"] = pub_key_file_path.read_text().strip()
     yield
     # Cleanup
     del os.environ["BUILDRUNNER_TEST_SSH_PUB_KEY"]
-    key_file_path.unlink()
+    if cleanup_key_file:
+        key_file_path.unlink()
+        pub_key_file_path.unlink()
 
 
 def _get_test_args(file_name: str) -> Optional[List[str]]: