Improve coverage reports (#361)

* [CI,nox] show the 5 longest tests in pytest output
* ignore coverage for impractical use cases
* test custom admonitions
* test code annotations
* test content tabs
* test keys role
* test syntax highlighting
* test task lists
* test rst-example directive
* test mermaid graphs
* improve json domain coverage
* test sitemap generation
* test inline icons
* switch to using codecov

.github/workflows/build.yml

@@ -223,7 +223,13 @@ jobs:
           name: coverage-report
           path: htmlcov/
       - name: Post coverage summary
-        run: cat .coverage_.md >> $GITHUB_STEP_SUMMARY
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{secrets.CODECOV_TOKEN}}
+        with:
+          files: ./coverage.xml
+          fail_ci_if_error: true # optional (default = false)
+          verbose: true # optional (default = false)
 
   compare-wheels:
     runs-on: ubuntu-latest
@@ -258,7 +264,7 @@ jobs:
     # Only publish package on push to tag or default branch.
     if: ${{ github.event_name == 'push' && github.repository == 'jbms/sphinx-immaterial' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') }}
     runs-on: ubuntu-latest
-    needs: [build, test]
+    needs: [build, coverage-report]
     steps:
       - uses: actions/download-artifact@v4
         with:

README.rst

@@ -1,7 +1,7 @@
 Sphinx-Immaterial Theme
 =======================
 
-|MIT License| |PyPI Package| |CI status|
+|MIT License| |PyPI Package| |CI status| |codecov-badge|
 
 This theme is an adaptation of the popular `mkdocs-material
 <https://github.com/squidfunk/mkdocs-material/>`__ theme for the `Sphinx
@@ -118,3 +118,7 @@ few differences, primarily due to differences between Sphinx and MkDocs:
 
 .. |CI status| image:: https://github.com/jbms/sphinx-immaterial/actions/workflows/build.yml/badge.svg
    :target: https://github.com/jbms/sphinx-immaterial/actions/workflows/build.yml
+
+.. |codecov-badge| image:: https://codecov.io/gh/jbms/sphinx-immaterial/graph/badge.svg?token=IGK0B3WN42
+   :alt: codecov
+   :target: https://codecov.io/gh/jbms/sphinx-immaterial

codecov.yml

@@ -0,0 +1,11 @@
+coverage:
+  status:
+    patch:
+      default:
+        informational: true
+    project:
+      default:
+        target: auto
+        # adjust accordingly based on how flaky your tests are
+        # this allows a 2% drop from the previous base commit coverage
+        threshold: 2%

docs/apidoc/json/domain.rst

@@ -77,7 +77,7 @@ specifying the JSON schema definition files.
       :caption: Setting default roles and highlight language in :file:`conf.py`
 
       rst_prolog = """
-      .. role json(code)
+      .. role:: json(code)
          :language: json
          :class: highlight
       """

noxfile.py

@@ -215,7 +215,7 @@ def tests(session: nox.Session, sphinx: str):
         # sphinxcontrib deps that dropped support for sphinx v4.x
         session.install("-r", "tests/requirements-sphinx4.txt")
     session.install("-r", "tests/requirements.txt")
-    session.run("coverage", "run", "-m", "pytest", "-vv", "-s")
+    session.run("coverage", "run", "-m", "pytest", "-vv", "-s", "--durations=5")
     # session.notify("docs") <- only calls docs(html), not dirhtml or latex builders
 
 
@@ -230,5 +230,5 @@ def coverage(session: nox.Session):
         f"<details><summary>Coverage is {total}%</summary>\n\n{md}\n\n</details>",
         encoding="utf-8",
     )
-    session.run("coverage", "html")
+    session.run("coverage", "xml")
     session.log("Coverage is %d%%", total)

pyproject.toml

@@ -41,6 +41,23 @@ omit = [
 
 [tool.coverage.report]
 skip_empty = true
+# Regexes for lines to exclude from consideration
+exclude_lines = [
+    # Have to re-enable the standard pragma
+    "pragma: no cover",
+    # Don't complain about missing debug-only code:
+    "def __repr__",
+    # the point of unit tests is to test parts of main()
+    "def main",
+    # ignore any branch that makes the module executable
+    'if __name__ == "__main__"',
+    # ignore missing implementations
+    "raise NotImplementedError",
+    # ignore the type checking specific code (only executed by mypy)
+    "if typing.TYPE_CHECKING",
+    # ignore import errors for conditional deps (test run with a strict set of deps)
+    "except ImportError",
+]
 
 [tool.coverage.html]
 show_contexts = true

sphinx_immaterial/content_tabs.py

@@ -172,7 +172,7 @@ def visit_tab_set(self: HTMLTranslator, node: content_tab_set):
 
 
 def depart_tab_set(self: HTMLTranslator, node: content_tab_set):
-    pass
+    pass  # pragma: no cover
 
 
 def setup(app: Sphinx):

sphinx_immaterial/inlinesyntaxhighlight.py

@@ -73,7 +73,8 @@ def visit_literal(self, node: docutils.nodes.literal) -> None:
         if "code" not in node["classes"] or not lang:
             return orig_visit_literal(self, node)
 
-        def warner(msg):
+        # This isn't used by pygments; it may be for a different highlighter
+        def warner(msg):  # pragma: no cover
             self.builder.warn(msg, (self.builder.current_docname, node.line))
 
         highlight_args = dict(node.get("highlight_args", {}), nowrap=True)

tests/admonition_test.py

@@ -0,0 +1,127 @@
+"""Tests related to theme's patched admonitions."""
+
+import pytest
+from sphinx.testing.util import SphinxTestApp
+from sphinx.errors import ExtensionError
+
+conf = [
+    {
+        "name": "legacy",
+        "color": (236, 64, 11),
+        "icon": "fontawesome/solid/recycle",
+        "classes": ["custom-class"],
+    },
+    {
+        "name": "attention",
+        "classes": ["important"],
+    },
+    {
+        "name": "versionremoved",
+        "classes": ["warning"],
+        "override": True,
+    },
+    {
+        "name": "deprecated",
+        "color": (0, 0, 0),
+        "override": True,
+    },
+    {
+        # This is impractical, but it covers a condition in production code
+        "name": "versionchanged",
+        "override": True,
+    },
+]
+
+
+def test_admonitions(immaterial_make_app):
+    app: SphinxTestApp = immaterial_make_app(
+        extra_conf=f"sphinx_immaterial_custom_admonitions={repr(conf)}",
+        files={
+            "index.rst": """
+The Test
+========
+
+.. deprecated:: 0.0.1 scheduled for removal
+
+.. versionremoved:: 0.1.0 Code was removed!
+    :collapsible: open
+    :class: custom-class
+
+    Some rationale.
+
+.. legacy::
+    :collapsible: open
+    :class: custom-class
+
+    Some content.
+
+.. legacy:: A
+    :title: Custom title
+
+    Some content.
+
+""",
+        },
+    )
+
+    app.build()
+    assert not app._warning.getvalue()  # type: ignore[attr-defined]
+
+
+def test_admonition_name_error(immaterial_make_app):
+    # both exceptions must be raised here, so signify this using nested with statements
+    with pytest.raises(ExtensionError):
+        with pytest.raises(ValueError):
+            immaterial_make_app(
+                extra_conf="sphinx_immaterial_custom_admonitions=[{"
+                '"name":"legacy!","classes":["note"]}]',
+                files={
+                    "index.rst": "",
+                },
+            )
+
+
+def test_admonitions_warnings(immaterial_make_app):
+    app: SphinxTestApp = immaterial_make_app(
+        files={
+            "index.rst": """
+The Test
+========
+
+.. note::
+    :collapsible:
+    :no-title:
+
+    Some content.
+
+.. deprecated:: 0.1.0
+    :collapsible:
+
+""",
+        },
+    )
+
+    app.build()
+    warnings = app._warning.getvalue()  # type: ignore[attr-defined]
+    assert warnings
+    assert "title is needed for collapsible admonitions" in warnings
+    assert "Expected 2 arguments before content in deprecated directive" in warnings
+
+
+def test_todo_admonition(immaterial_make_app):
+    app: SphinxTestApp = immaterial_make_app(
+        extra_conf="extensions.append('sphinx.ext.todo')",
+        files={
+            "index.rst": """
+The Test
+========
+
+.. todo::
+    Some content.
+
+""",
+        },
+    )
+
+    app.build()
+    assert not app._warning.getvalue()  # type: ignore[attr-defined]

tests/code_annotations_test.py

@@ -0,0 +1,50 @@
+"""Tests related to theme's code block annotations."""
+
+from sphinx.testing.util import SphinxTestApp
+
+index_rst = """
+The Test
+========
+
+.. code-block:: yaml
+    :caption: A caption for good measure
+
+    # (1)!
+
+.. code-annotations::
+    1. An annotation
+
+.. code-annotations::
+    #. A regular list not used in annotations.
+
+.. code-block:: python
+
+    # A normal code block without annotations
+"""
+
+
+def test_code_annotation(immaterial_make_app):
+    app: SphinxTestApp = immaterial_make_app(
+        files={"index.rst": index_rst},
+    )
+
+    app.build()
+    assert not app._warning.getvalue()  # type: ignore[attr-defined]
+
+def test_code_annotation_error(immaterial_make_app):
+    app: SphinxTestApp = immaterial_make_app(
+        files={
+            "index.rst": """
+The Test
+========
+
+.. code-annotations::
+    This is not an enumerated list!
+
+"""},
+    )
+
+    app.build()
+    warnings = app._warning.getvalue()  # type: ignore[attr-defined]
+    assert warnings
+    assert "The code-annotations directive only accepts an enumerated list" in warnings

tests/content_tabs_test.py

@@ -0,0 +1,70 @@
+"""Tests related to theme's content tabs."""
+
+from docutils.nodes import Node
+import pytest
+from sphinx.testing.util import SphinxTestApp
+from sphinx_immaterial.content_tabs import is_md_tab_type
+
+index_rst = """
+The Test
+========
+
+.. md-tab-set::
+    :name: tab_set_ref
+
+    .. md-tab-item:: A
+
+        Tab A content
+
+    .. md-tab-item:: B
+
+        Tab B content
+"""
+
+
+def test_content_tabs(immaterial_make_app):
+    app: SphinxTestApp = immaterial_make_app(
+        files={"index.rst": index_rst},
+    )
+
+    app.build()
+    assert not app._warning.getvalue()  # type: ignore[attr-defined]
+
+def test_tab_ext_error():
+    assert not is_md_tab_type(Node(), "")
+
+def test_tab_set_child_error(immaterial_make_app):
+    app: SphinxTestApp = immaterial_make_app(
+        files={
+            "index.rst": """
+The Test
+========
+
+.. md-tab-set::
+
+    This is not a ``md-tab-item``!
+"""},
+    )
+
+    with pytest.raises(ValueError):
+        app.build()
+        warnings = app._warning.getvalue()  # type: ignore[attr-defined]
+        assert warnings
+        assert "All children of a 'md-tab-set' should be 'md-tab-item'" in warnings
+
+def test_tab_item_parent_error(immaterial_make_app):
+    app: SphinxTestApp = immaterial_make_app(
+        files={
+            "index.rst": """
+The Test
+========
+
+.. md-tab-item:: Orphan
+
+    This is ``md-tab-item`` has no parent ``md-tab-set``!
+"""},
+    )
+    app.build()
+    warnings = app._warning.getvalue()  # type: ignore[attr-defined]
+    assert warnings
+    assert "The parent of a 'md-tab-item' should be a 'md-tab-set'" in warnings