Added more tests for notebook checker

pyproject.toml

@@ -36,16 +36,11 @@ path = "src/databricks/labs/pylint/__about__.py"
 dependencies = [
     "coverage[toml]>=6.5",
     "pytest",
-    "pylint",
     "pytest-xdist",
     "pytest-cov>=4.0.0,<5.0.0",
-    "pytest-mock>=3.0.0,<4.0.0",
     "pytest-timeout",
     "ruff>=0.0.243",
     "isort>=2.5.0",
-    "mypy",
-    "types-PyYAML",
-    "types-requests",
 ]
 
 python="3.10"
@@ -55,17 +50,14 @@ path = ".venv"
 
 [tool.hatch.envs.default.scripts]
 test        = "pytest -n 2 --cov src --cov-report=xml --timeout 30 tests --durations 20"
-coverage    = "pytest -n 2 --cov src tests/unit --timeout 30 --cov-report=html --durations 20"
-integration = "pytest -n 10 --cov src tests/integration --durations 20"
+coverage    = "pytest -n 2 --cov src tests --timeout 30 --cov-report=html --durations 20"
 fmt         = ["isort .",
                "ruff format",
                "ruff check . --fix",
-               #"mypy .",
                "pylint --output-format=colorized -j 0 src"]
 verify      = ["black --check .",
                "isort . --check-only",
                "ruff .",
-               "mypy .",
                "pylint --output-format=colorized -j 0 src"]
 
 

src/databricks/labs/pylint/notebooks.py

@@ -45,6 +45,7 @@ def process_module(self, node: astroid.Module):
         - https://github.com/PyCQA/redbaron (LGPLv3)
         """
         cells = 1
+        too_many_cells_raised = False
         with node.stream() as stream:
             for lineno, line in enumerate(stream):
                 lineno += 1
@@ -53,8 +54,9 @@ def process_module(self, node: astroid.Module):
                     return
                 if line == b"# COMMAND ----------\n":
                     cells += 1
-                if cells > self.linter.config.max_cells:
-                    self.add_message("notebooks-too-many-cells", line=lineno)
+                if cells > self.linter.config.max_cells and not too_many_cells_raised:
+                    self.add_message("notebooks-too-many-cells", line=lineno + 1)
+                    too_many_cells_raised = True
                     continue
                 if line.startswith(b"# MAGIC %run"):
                     self.add_message("notebooks-percent-run", line=lineno)

tests/samples/m/many_cells.py

@@ -0,0 +1,305 @@
+# Databricks notebook source
+# pylint: disable=missing-module-docstring
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+
+print("Hello, World!")
+
+# COMMAND ----------
+# [notebooks-too-many-cells]
+
+print("Hello, World!")

tests/samples/m/many_cells.txt

@@ -0,0 +1 @@
+notebooks-too-many-cells:303:0:None:None::Notebooks should not have more than 75 cells:UNDEFINED

tests/test_airflow.py

@@ -31,7 +31,7 @@ def test_missing_data_security_mode_in_job_clusters(lint_with):
 )"""
     )
     assert (
-        "[missing-data-security-mode] banana cluster missing 'data_security_mode' "
+        "[missing-data-security-mode] banana cluster missing `data_security_mode` "
         "required for Unity Catalog compatibility"
     ) in messages
 
@@ -59,7 +59,7 @@ def test_missing_data_security_mode_in_task_clusters(lint_with):
 )"""
     )
     assert (
-        "[missing-data-security-mode] banana cluster missing 'data_security_mode' "
+        "[missing-data-security-mode] banana cluster missing `data_security_mode` "
         "required for Unity Catalog compatibility"
     ) in messages
 
@@ -78,6 +78,6 @@ def test_missing_data_security_mode_in_submit_run_clusters(lint_with):
     )
     assert "[unsupported-runtime] ephemeral cluster has unsupported runtime: 10.1.x-scala2.12" in messages
     assert (
-        "[missing-data-security-mode] ephemeral cluster missing 'data_security_mode' "
+        "[missing-data-security-mode] ephemeral cluster missing `data_security_mode` "
         "required for Unity Catalog compatibility"
     ) in messages