Refactor deployed environment CI checks

.github/workflows/test.yml

@@ -162,7 +162,7 @@ jobs:
           pattern: coverage-data-*
           merge-multiple: true
 
-      - name: Combine coverage & fail if it's <100%
+      - name: Combine coverage & fail if it goes down
         run: |
           uv tool install 'coverage[toml]'
 
@@ -172,9 +172,11 @@ jobs:
           # Report and write to summary.
           coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
 
-          # Report again and fail if under 92%.
-          # (threshold is based on 0.1.0rc1 CI statement coverage)
-          coverage report --fail-under=92
+          # Report again and fail if under 91%.
+          # Highest historical coverage: 92%
+          # Subsequent proportional coverage reductions:
+          # - de-duplicated the deployment checking code
+          coverage report --fail-under=91
 
       - name: Upload HTML report if check failed
         uses: actions/upload-artifact@v4

tests/support.py

@@ -5,18 +5,22 @@
 import subprocess
 import sys
 import tomllib
+import unittest
 
 from dataclasses import dataclass, fields
 from pathlib import Path
-from typing import Any, cast, Mapping
+from typing import Any, Callable, cast, Mapping, Sequence, TypeVar
 from unittest.mock import create_autospec
 
 import pytest
 
-from venvstacks._util import run_python_command
+from venvstacks._util import get_env_python, run_python_command
+
 from venvstacks.stacks import (
     BuildEnvironment,
     EnvNameDeploy,
+    ExportedEnvironmentPaths,
+    ExportMetadata,
     LayerBaseName,
     PackageIndexConfig,
 )
@@ -216,3 +220,87 @@ def get_sys_path(env_python: Path) -> list[str]:
 def run_module(env_python: Path, module_name: str) -> subprocess.CompletedProcess[str]:
     command = [str(env_python), "-Im", module_name]
     return capture_python_output(command)
+
+
+###########################################################
+# Checking deployed environments for the expected details
+###########################################################
+
+
+class DeploymentTestCase(unittest.TestCase):
+    """Native unittest test case with additional deployment validation checks"""
+    EXPECTED_APP_OUTPUT = ""
+
+    def assertSysPathEntry(self, expected: str, env_sys_path: Sequence[str]) -> None:
+        self.assertTrue(
+            any(expected in path_entry for path_entry in env_sys_path),
+            f"No entry containing {expected!r} found in {env_sys_path}",
+        )
+
+    T = TypeVar("T", bound=Mapping[str, Any])
+
+    def check_deployed_environments(
+        self,
+        layered_metadata: dict[str, Sequence[T]],
+        get_exported_python: Callable[[T], tuple[str, Path, list[str]]],
+    ) -> None:
+        for rt_env in layered_metadata["runtimes"]:
+            env_name, _, env_sys_path = get_exported_python(rt_env)
+            self.assertTrue(env_sys_path)  # Environment should have sys.path entries
+            # Runtime environment layer should be completely self-contained
+            self.assertTrue(
+                all(env_name in path_entry for path_entry in env_sys_path),
+                f"Path outside {env_name} in {env_sys_path}",
+            )
+        for fw_env in layered_metadata["frameworks"]:
+            env_name, _, env_sys_path = get_exported_python(fw_env)
+            self.assertTrue(env_sys_path)  # Environment should have sys.path entries
+            # Framework and runtime should both appear in sys.path
+            runtime_name = fw_env["runtime_name"]
+            short_runtime_name = ".".join(runtime_name.split(".")[:2])
+            self.assertSysPathEntry(env_name, env_sys_path)
+            self.assertSysPathEntry(short_runtime_name, env_sys_path)
+        for app_env in layered_metadata["applications"]:
+            env_name, env_python, env_sys_path = get_exported_python(app_env)
+            self.assertTrue(env_sys_path)  # Environment should have sys.path entries
+            # Application, frameworks and runtime should all appear in sys.path
+            runtime_name = app_env["runtime_name"]
+            short_runtime_name = ".".join(runtime_name.split(".")[:2])
+            self.assertSysPathEntry(env_name, env_sys_path)
+            self.assertTrue(
+                any(env_name in path_entry for path_entry in env_sys_path),
+                f"No entry containing {env_name} found in {env_sys_path}",
+            )
+            for fw_env_name in app_env["required_layers"]:
+                self.assertSysPathEntry(fw_env_name, env_sys_path)
+            self.assertSysPathEntry(short_runtime_name, env_sys_path)
+            # Launch module should be executable
+            launch_module = app_env["app_launch_module"]
+            launch_result = run_module(env_python, launch_module)
+            # Tolerate extra trailing whitespace on stdout
+            self.assertEqual(launch_result.stdout.rstrip(), self.EXPECTED_APP_OUTPUT)
+            # Nothing at all should be emitted on stderr
+            self.assertEqual(launch_result.stderr, "")
+
+    def check_environment_exports(self, export_paths: ExportedEnvironmentPaths) -> None:
+        metadata_path, snippet_paths, env_paths = export_paths
+        exported_manifests = ManifestData(metadata_path, snippet_paths)
+        env_name_to_path: dict[str, Path] = {}
+        for env_metadata, env_path in zip(exported_manifests.snippet_data, env_paths):
+            # TODO: Check more details regarding expected metadata contents
+            self.assertTrue(env_path.exists())
+            env_name = EnvNameDeploy(env_metadata["install_target"])
+            self.assertEqual(env_path.name, env_name)
+            env_name_to_path[env_name] = env_path
+        layered_metadata = exported_manifests.combined_data["layers"]
+
+        def get_exported_python(
+            env: ExportMetadata,
+        ) -> tuple[EnvNameDeploy, Path, list[str]]:
+            env_name = env["install_target"]
+            env_path = env_name_to_path[env_name]
+            env_python = get_env_python(env_path)
+            env_sys_path = get_sys_path(env_python)
+            return env_name, env_python, env_sys_path
+
+        self.check_deployed_environments(layered_metadata, get_exported_python)

tests/test_minimal_project.py

@@ -6,7 +6,7 @@
 
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Callable, cast, Mapping, Sequence, TypeVar
+from typing import Any, cast
 
 # Use unittest for consistency with test_sample_project (which needs the better diff support)
 import unittest
@@ -15,13 +15,13 @@
 import pytest  # To mark slow test cases
 
 from support import (
+    ApplicationEnvSummary,
+    DeploymentTestCase,
     EnvSummary,
     LayeredEnvSummary,
-    ApplicationEnvSummary,
     ManifestData,
     make_mock_index_config,
     get_sys_path,
-    run_module,
 )
 
 from venvstacks.stacks import (
@@ -31,8 +31,6 @@
     BuildEnvironment,
     EnvNameDeploy,
     StackSpec,
-    ExportedEnvironmentPaths,
-    ExportMetadata,
     PackageIndexConfig,
     PublishedArchivePaths,
     get_build_platform,
@@ -331,7 +329,7 @@ def test_custom_output_directory_absolute(self) -> None:
         self.assertFalse(expected_output_path.exists())
 
 
-class TestMinimalBuild(unittest.TestCase):
+class TestMinimalBuild(DeploymentTestCase):
     # Test cases that actually create the build environment folders
 
     working_path: Path
@@ -424,56 +422,6 @@ def check_publication_result(
         expected_archive_paths.sort()
         self.assertEqual(sorted(archive_paths), expected_archive_paths)
 
-    # TODO: Refactor to share the environment checking code with test_sample_project
-    def assertSysPathEntry(self, expected: str, env_sys_path: Sequence[str]) -> None:
-        self.assertTrue(
-            any(expected in path_entry for path_entry in env_sys_path),
-            f"No entry containing {expected!r} found in {env_sys_path}",
-        )
-
-    T = TypeVar("T", bound=Mapping[str, Any])
-
-    def check_deployed_environments(
-        self,
-        layered_metadata: dict[str, Sequence[T]],
-        get_exported_python: Callable[[T], tuple[str, Path, list[str]]],
-    ) -> None:
-        for rt_env in layered_metadata["runtimes"]:
-            env_name, _, env_sys_path = get_exported_python(rt_env)
-            self.assertTrue(env_sys_path)  # Environment should have sys.path entries
-            # Runtime environment layer should be completely self-contained
-            self.assertTrue(
-                all(env_name in path_entry for path_entry in env_sys_path),
-                f"Path outside {env_name} in {env_sys_path}",
-            )
-        for fw_env in layered_metadata["frameworks"]:
-            env_name, _, env_sys_path = get_exported_python(fw_env)
-            self.assertTrue(env_sys_path)  # Environment should have sys.path entries
-            # Framework and runtime should both appear in sys.path
-            runtime_name = fw_env["runtime_name"]
-            short_runtime_name = ".".join(runtime_name.split(".")[:2])
-            self.assertSysPathEntry(env_name, env_sys_path)
-            self.assertSysPathEntry(short_runtime_name, env_sys_path)
-        for app_env in layered_metadata["applications"]:
-            env_name, env_python, env_sys_path = get_exported_python(app_env)
-            self.assertTrue(env_sys_path)  # Environment should have sys.path entries
-            # Application, frameworks and runtime should all appear in sys.path
-            runtime_name = app_env["runtime_name"]
-            short_runtime_name = ".".join(runtime_name.split(".")[:2])
-            self.assertSysPathEntry(env_name, env_sys_path)
-            self.assertTrue(
-                any(env_name in path_entry for path_entry in env_sys_path),
-                f"No entry containing {env_name} found in {env_sys_path}",
-            )
-            for fw_env_name in app_env["required_layers"]:
-                self.assertSysPathEntry(fw_env_name, env_sys_path)
-            self.assertSysPathEntry(short_runtime_name, env_sys_path)
-            # Launch module should be executable
-            launch_module = app_env["app_launch_module"]
-            launch_result = run_module(env_python, launch_module)
-            self.assertEqual(launch_result.stdout, "")
-            self.assertEqual(launch_result.stderr, "")
-
     @staticmethod
     def _run_postinstall(base_python_path: Path, env_path: Path) -> None:
         postinstall_script = env_path / "postinstall.py"
@@ -485,7 +433,10 @@ def check_archive_deployment(self, published_paths: PublishedArchivePaths) -> No
         published_manifests = ManifestData(metadata_path, snippet_paths)
         # TODO: read the base Python path for each environment from the metadata
         #       https://github.com/lmstudio-ai/venvstacks/issues/19
-        with tempfile.TemporaryDirectory() as deployment_dir:
+        # TODO: figure out a more robust way of handling Windows potentially still
+        #       having the Python executables in the environment open when the
+        #       parent process tries to clean up the deployment directory.
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as deployment_dir:
             # Extract archives
             deployment_path = Path(deployment_dir)
             env_name_to_path: dict[EnvNameDeploy, Path] = {}
@@ -532,29 +483,6 @@ def get_exported_python(
 
             self.check_deployed_environments(layered_metadata, get_exported_python)
 
-    def check_environment_exports(self, export_paths: ExportedEnvironmentPaths) -> None:
-        metadata_path, snippet_paths, env_paths = export_paths
-        exported_manifests = ManifestData(metadata_path, snippet_paths)
-        env_name_to_path: dict[str, Path] = {}
-        for env_metadata, env_path in zip(exported_manifests.snippet_data, env_paths):
-            # TODO: Check more details regarding expected metadata contents
-            self.assertTrue(env_path.exists())
-            env_name = EnvNameDeploy(env_metadata["install_target"])
-            self.assertEqual(env_path.name, env_name)
-            env_name_to_path[env_name] = env_path
-        layered_metadata = exported_manifests.combined_data["layers"]
-
-        def get_exported_python(
-            env: ExportMetadata,
-        ) -> tuple[EnvNameDeploy, Path, list[str]]:
-            env_name = env["install_target"]
-            env_path = env_name_to_path[env_name]
-            env_python = get_env_python(env_path)
-            env_sys_path = get_sys_path(env_python)
-            return env_name, env_python, env_sys_path
-
-        self.check_deployed_environments(layered_metadata, get_exported_python)
-
     @pytest.mark.slow
     def test_locking_and_publishing(self) -> None:
         # This is organised as subtests in a monolothic test sequence to reduce CI overhead