feat(fw/consume): write the test class & function docstrings to `_inf…

…o["description"]` for use in hive reports (ethereum#579)

* feat(fw): write test class and function docstrings to the fixture _info

* feat(consume): propagate the test fixture description to the hive test report

* feat(consume): prepend the test id to the test description

* refactor(docs|fw): move utility functions to ethereum_test_tools.utility

* feat(fill): add test function github permalink to _info['url']

* feat(consume): add test source url to hive test report description

* fix(fw): add workaround to fix fw pytest_plugin tests

* fix(fw): fix pytest plugin fw tests on macos

* feat(consume): format id as code

* Revert "feat(consume): format id as code"

This reverts commit cffb871.

* fix(consume): use the consume pytest test id as hive test id

* fix(fw): add improvements to versioning.py from review

* docs: update changelog

docs/CHANGELOG.md

@@ -35,6 +35,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Add a "slow" pytest marker, in order to be able to limit the filled tests until release ([#562](https://github.com/ethereum/execution-spec-tests/pull/562)).
 - ✨ Add a CLI tool that generates blockchain tests as Python from a transaction hash ([#470](https://github.com/ethereum/execution-spec-tests/pull/470), [#576](https://github.com/ethereum/execution-spec-tests/pull/576)).
 - ✨ Add more Transaction and Block exceptions from existing ethereum/tests repo ([#572](https://github.com/ethereum/execution-spec-tests/pull/572)).
+- ✨ Add "description" and "url" fields containing test case documentation and a source code permalink to fixtures during `fill` and use them in `consume`-generated Hive test reports ([#579](https://github.com/ethereum/execution-spec-tests/pull/579)).
 
 ### 🔧 EVM Tools
 

docs/gen_test_case_reference.py

@@ -16,9 +16,12 @@
 
 import mkdocs_gen_files
 import pytest
-from git import Repo
 
 from ethereum_test_forks import get_development_forks, get_forks
+from ethereum_test_tools.utility.versioning import (
+    generate_github_url,
+    get_current_commit_hash_or_tag,
+)
 
 logger = logging.getLogger("mkdocs")
 
@@ -206,44 +209,6 @@ def run_collect_only(test_path: Path = source_directory) -> Tuple[str, str]:
     return f'fill {" ".join(collect_only_args)}', collect_only_output
 
 
-def generate_github_url(file_path, branch_or_commit_or_tag="main"):
-    """
-    Generate a link to a source file in Github.
-    """
-    base_url = "https://github.com"
-    username = "ethereum"
-    repository = "execution-spec-tests"
-    if re.match(
-        r"^v[0-9]{1,2}\.[0-9]{1,3}\.[0-9]{1,3}(a[0-9]+|b[0-9]+|rc[0-9]+)?$",
-        branch_or_commit_or_tag,
-    ):
-        return f"{base_url}/{username}/{repository}/tree/{branch_or_commit_or_tag}/{file_path}"
-    else:
-        return f"{base_url}/{username}/{repository}/blob/{branch_or_commit_or_tag}/{file_path}"
-
-
-def get_current_commit_hash_or_tag(repo_path="."):
-    """
-    Get the latest commit hash or tag from the clone where doc is being built.
-    """
-    repo = Repo(repo_path)
-    try:
-        # Get the tag that points to the current commit
-        current_tag = next((tag for tag in repo.tags if tag.commit == repo.head.commit))
-        return current_tag.name
-    except StopIteration:
-        # If there are no tags that point to the current commit, return the commit hash
-        return repo.head.commit.hexsha
-
-
-def get_current_commit_hash(repo_path="."):
-    """
-    Get the latest commit hash from the clone where doc is being built.
-    """
-    repo = Repo(repo_path)
-    return repo.head.commit.hexsha
-
-
 COMMIT_HASH_OR_TAG = get_current_commit_hash_or_tag()
 
 

setup.cfg

@@ -28,6 +28,7 @@ install_requires =
     hive.py@git+https://github.com/danceratopz/hive.py@chore/setup.cfg/move-mypy-deps-to-lint-extras
     setuptools
     types-setuptools
+    gitpython>=3.1.31,<4
     PyJWT>=2.3.0,<3
     tenacity>8.2.0,<9
     bidict>=0.23,<1
@@ -87,7 +88,6 @@ lint =
 
 docs =
     cairosvg>=2.7.0,<3  # required for social plugin (material)
-    gitpython>=3.1.31,<4
     mike>=1.1.2,<2
     mkdocs>=1.4.3,<2
     mkdocs-gen-files>=0.5.0,<1

src/ethereum_test_tools/spec/base/base_test.py

@@ -100,6 +100,8 @@ def json_dict_with_info(self, hash_only: bool = False) -> Dict[str, Any]:
     def fill_info(
         self,
         t8n: TransitionTool,
+        fixture_description: str,
+        fixture_source_url: str,
         ref_spec: ReferenceSpec | None,
     ):
         """
@@ -108,6 +110,8 @@ def fill_info(
         if "comment" not in self.info:
             self.info["comment"] = "`execution-spec-tests` generated test"
         self.info["filling-transition-tool"] = t8n.version()
+        self.info["description"] = fixture_description
+        self.info["url"] = fixture_source_url
         if ref_spec is not None:
             ref_spec.write_info(self.info)
 

src/ethereum_test_tools/utility/__init__.py

@@ -0,0 +1,3 @@
+"""
+Sub-package for utility functions and classes.
+"""

src/ethereum_test_tools/utility/versioning.py

@@ -0,0 +1,44 @@
+"""
+Utility module with helper functions for versioning.
+"""
+
+import re
+
+from git import InvalidGitRepositoryError, Repo  # type: ignore
+
+
+def get_current_commit_hash_or_tag(repo_path="."):
+    """
+    Get the latest commit hash or tag from the clone where doc is being built.
+    """
+    try:
+        repo = Repo(repo_path)
+        # Try to get the current tag that points to the current commit
+        current_tag = next((tag for tag in repo.tags if tag.commit == repo.head.commit), None)
+        # Return the commit hash if no such tag exits
+        return current_tag.name if current_tag else repo.head.commit.hexsha
+    except InvalidGitRepositoryError:
+        # This hack is necessary for our framework tests. We use the pytester/tempdir fixtures
+        # to execute pytest within a pytest session (for top-level tests of our pytest plugins).
+        # The pytester fixture executes these tests in a temporary directory, which is not a git
+        # repository; this is a workaround to stop these tests failing.
+        #
+        # Tried monkeypatching the pytest plugin tests, but it didn't play well with pytester.
+        return "Not a git repository; this should only be seen in framework tests."
+
+
+def generate_github_url(file_path, branch_or_commit_or_tag="main", line_number=""):
+    """
+    Generate a permalink to a source file in Github.
+    """
+    base_url = "https://github.com"
+    username = "ethereum"
+    repository = "execution-spec-tests"
+    if line_number:
+        line_number = f"#L{line_number}"
+    release_tag_regex = r"^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(a[0-9]+|b[0-9]+|rc[0-9]+)?$"
+    tree_or_blob = "tree" if re.match(release_tag_regex, branch_or_commit_or_tag) else "blob"
+    return (
+        f"{base_url}/{username}/{repository}/{tree_or_blob}/"
+        f"{branch_or_commit_or_tag}/{file_path}{line_number}"
+    )

src/pytest_plugins/consume/simulator_common.py

@@ -36,3 +36,18 @@ def fixture(fixture_source: JsonSource, test_case: TestCase) -> Fixture:
         fixtures = BlockchainFixtures.from_file(Path(fixture_source) / test_case.json_path)
         fixture = fixtures[test_case.id]
     return fixture
+
+
+@pytest.fixture(scope="function")
+def fixture_description(fixture: Fixture, test_case: TestCase) -> str:
+    """
+    Return the description of the current test case.
+    """
+    description = f"Test id: {test_case.id}"
+    if "url" in fixture.info:
+        description += f"\n\nTest source: {fixture.info['url']}"
+    if "description" not in fixture.info:
+        description += "\n\nNo description field provided in the fixture's 'info' section."
+    else:
+        description += f"\n\n{fixture.info['description']}"
+    return description

src/pytest_plugins/pytest_hive/pytest_hive.py

@@ -107,11 +107,17 @@ def hive_test(request, test_suite: HiveTestSuite):
     """
     Propagate the pytest test case and its result to the hive server.
     """
-    test_parameter_string = request.node.nodeid.split("[")[-1].rstrip("]")  # test fixture name
+    try:
+        fixture_description = request.getfixturevalue("fixture_description")
+    except pytest.FixtureLookupError:
+        pytest.exit(
+            "Error: The 'fixture_description' fixture has not been defined by the simulator "
+            "or pytest plugin using this plugin!"
+        )
+    test_parameter_string = request.node.nodeid  # consume pytest test id
     test: HiveTest = test_suite.start_test(
-        # TODO: pass test case documentation when available
         name=test_parameter_string,
-        description="TODO: This should come from the '_info' field.",
+        description=fixture_description,
     )
     yield test
     try:

src/pytest_plugins/test_filler/test_filler.py

@@ -22,6 +22,10 @@
     get_forks_with_solc_support,
 )
 from ethereum_test_tools import SPEC_TYPES, BaseTest, FixtureCollector, TestInfo, Yul
+from ethereum_test_tools.utility.versioning import (
+    generate_github_url,
+    get_current_commit_hash_or_tag,
+)
 from evm_transition_tool import FixtureFormats, TransitionTool
 from pytest_plugins.spec_version_checker.spec_version_checker import EIPSpecTestItem
 
@@ -564,6 +568,38 @@ def node_to_test_info(node) -> TestInfo:
     )
 
 
+@pytest.fixture(scope="function")
+def fixture_source_url(request):
+    """
+    Returns the URL to the fixture source.
+    """
+    function_line_number = request.function.__code__.co_firstlineno
+    module_relative_path = os.path.relpath(request.module.__file__)
+    hash_or_tag = get_current_commit_hash_or_tag()
+    github_url = generate_github_url(
+        module_relative_path, branch_or_commit_or_tag=hash_or_tag, line_number=function_line_number
+    )
+    return github_url
+
+
+@pytest.fixture(scope="function")
+def fixture_description(request):
+    """Fixture to extract and combine docstrings from the test class and the test function."""
+    description_unavailable = (
+        "No description available - add a docstring to the python test class or function."
+    )
+    test_class_doc = f"Test class documentation:\n{request.cls.__doc__}" if request.cls else ""
+    test_function_doc = (
+        f"Test function documentation:\n{request.function.__doc__}"
+        if request.function.__doc__
+        else ""
+    )
+    if not test_class_doc and not test_function_doc:
+        return description_unavailable
+    combined_docstring = f"{test_class_doc}\n\n{test_function_doc}".strip()
+    return combined_docstring
+
+
 def base_test_parametrizer(cls: Type[BaseTest]):
     """
     Generates a pytest.fixture for a given BaseTest subclass.
@@ -584,6 +620,8 @@ def base_test_parametrizer_func(
         eips,
         dump_dir_parameter_level,
         fixture_collector,
+        fixture_description,
+        fixture_source_url,
     ):
         """
         Fixture used to instantiate an auto-fillable BaseTest object from within
@@ -608,7 +646,12 @@ def __init__(self, *args, **kwargs):
                     fixture_format=fixture_format,
                     eips=eips,
                 )
-                fixture.fill_info(t8n, reference_spec)
+                fixture.fill_info(
+                    t8n,
+                    fixture_description,
+                    fixture_source_url=fixture_source_url,
+                    ref_spec=reference_spec,
+                )
 
                 fixture_path = fixture_collector.add_fixture(
                     node_to_test_info(request.node),

whitelist.txt

@@ -128,6 +128,7 @@ extcodehash
 extcodesize
 F00
 filesystem
+firstlineno
 fn
 fname
 forkchoice
@@ -250,6 +251,7 @@ parseable
 pathlib
 pdb
 perf
+permalink
 petersburg
 pformat
 png
@@ -420,6 +422,7 @@ makepyfile
 makereport
 metafunc
 modifyitems
+monkeypatching
 nodeid
 noop
 oog
@@ -453,6 +456,7 @@ substring
 substrings
 tf
 teardown
+tempdir
 testdir
 teststatus
 tmpdir