feat(general): initial support for python 3.13 (#6962)

* initial support for python 3.13

* revert pipenv

* update workflow

* fix mypy

* make parallel run fail instead of returning none

* fix mypy

* fix parallel run failure state

* revert mypy pinning

* fix mypy

* fix mypy

* revert

* fix mypy

* try to keep traceback

* update README.md

* remove asserts

.github/workflows/pr-test.yml

@@ -68,7 +68,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -88,9 +88,9 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install pipenv
         run: |
-          if [ '${{ matrix.python }}' == '3.12' ]; then
+          if [ "${{ matrix.python }}" = "3.12" ] || [ "${{ matrix.python }}" = "3.13" ]; then
             # needed for numpy
-            python -m pip install --no-cache-dir --upgrade pipenv==2024.0.3
+            python -m pip install --no-cache-dir --upgrade pipenv==2024.4.0
           else
             python -m pip install --no-cache-dir --upgrade pipenv
           fi
@@ -100,7 +100,9 @@ jobs:
           pipenv --rm || true
           pipenv --python ${{ matrix.python }}
 
-          if [ '${{ matrix.python }}' == '3.12' ]; then
+          if [ "${{ matrix.python }}" = "3.12" ] || [ "${{ matrix.python }}" = "3.13" ]; then
+            echo "patching >3.12 issues"
+            pipenv run pip install setuptools
             # needed for numpy
             pipenv install --skip-lock --dev -v
           else
@@ -119,7 +121,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python: ["3.10", "3.11", "3.12"]
+        python: ["3.10", "3.11", "3.12", "3.13"]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -229,7 +231,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python: ["3.12"]
+        python: ["3.12", "3.13"]
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -325,7 +327,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python: ["3.12"]
+        python: ["3.12", "3.13"]
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:

Pipfile

@@ -12,7 +12,7 @@ pytest-xdist = "*"
 pytest-asyncio = "*"
 pytest-cov = "*"
 pytest-mock = "*"
-coverage ="==5.5"
+coverage ="==7.6.1"
 coverage-badge = "*"
 bandit = "*"
 urllib3-mock = "*"
@@ -86,6 +86,5 @@ license-expression = ">=30.1.0,<31.0.0"
 rustworkx = ">=0.13.0,<0.14.0"
 pydantic = ">=2.0.0,<3.0.0"
 
-
 [requires]
-python_version = "3.8"
+python_version = ">=3.8.0,<3.14"

README.md

@@ -483,4 +483,4 @@ To skip this API call use the flag `--skip-download`.
 Start with our [Documentation](https://www.checkov.io/1.Welcome/Quick%20Start.html) for quick tutorials and examples.
 
 ## Python Version Support
-We follow the official support cycle of Python, and we use automated tests for supported versions of Python. This means we currently support Python 3.9 - 3.12, inclusive. Note that Python 3.8 reached EOL on October 2024 and Python 3.9 will reach EOL in October 2025. We are investigating support for 3.13. If you run into any issues with any non-EOL Python version, please open an Issue.
+We follow the official support cycle of Python, and we use automated tests for supported versions of Python. This means we currently support Python 3.9 - 3.13, inclusive. Note that Python 3.8 reached EOL on October 2024 and Python 3.9 will reach EOL in October 2025. If you run into any issues with any non-EOL Python version, please open an Issue.

checkov/arm/checks/resource/AKSMaxPodsMinimum.py

@@ -17,7 +17,7 @@ def __init__(self) -> None:
     def scan_resource_conf(self, conf: dict[str, Any]) -> CheckResult:
         max_pods: Optional[int] = 30
 
-        properties = conf.get("properties")
+        properties = conf.get("properties", {})
         if properties and isinstance(properties, dict):
             max_pods = properties.get("maxPods")
 

checkov/bicep/graph_builder/local_graph.py

@@ -34,6 +34,7 @@
 BicepElementsAlias: TypeAlias = Literal["globals", "parameters", "variables", "resources", "modules", "outputs"]
 
 
+# mypy: disable-error-code="misc"
 class BicepElements(str, Enum):
     GLOBALS: Literal["globals"] = "globals"
     PARAMETERS: Literal["parameters"] = "parameters"

checkov/common/parallelizer/parallel_runner.py

@@ -17,6 +17,12 @@
 _T = TypeVar("_T")
 
 
+class ParallelRunException(Exception):
+    def __init__(self, internal_exception: Exception) -> None:
+        self.internal_exception = internal_exception
+        super().__init__(internal_exception)
+
+
 class ParallelRunner:
     def __init__(
         self, workers_number: int | None = None,
@@ -72,14 +78,15 @@ def func_wrapper(original_func: Callable[[Any], _T], items_group: List[Any], con
                         result = original_func(*item)
                     else:
                         result = original_func(item)
-                except Exception:
+
+                    connection.send(result)
+                except Exception as e:
                     logging.error(
                         f"Failed to invoke function {func.__code__.co_filename.replace('.py', '')}.{func.__name__} with {item}",
                         exc_info=True,
                     )
-                    result = None
+                    connection.send(ParallelRunException(e))
 
-                connection.send(result)
             connection.close()
 
         logging.debug(
@@ -97,7 +104,12 @@ def func_wrapper(original_func: Callable[[Any], _T], items_group: List[Any], con
         for _, parent_conn, group_len in processes:
             for _ in range(group_len):
                 try:
-                    yield parent_conn.recv()
+                    v = parent_conn.recv()
+
+                    if isinstance(v, ParallelRunException):
+                        raise v.internal_exception.with_traceback(v.internal_exception.__traceback__)
+
+                    yield v
                 except EOFError:
                     pass
 

checkov/common/parsers/json/decoder.py

@@ -3,10 +3,10 @@
 import logging
 from collections.abc import Sequence
 from json import JSONDecoder
-from json.decoder import WHITESPACE, WHITESPACE_STR, BACKSLASH, STRINGCHUNK, JSONArray  # type:ignore[attr-defined]  # they are not explicitly exported
+from json.decoder import WHITESPACE, WHITESPACE_STR, BACKSLASH, STRINGCHUNK, JSONArray  # type:ignore  # they are not explicitly exported
 from typing import Any, Callable, Pattern, Match
 
-from json.scanner import NUMBER_RE  # type:ignore[import-not-found]  # is not explicitly exported
+from json.scanner import NUMBER_RE  # type:ignore  # is not explicitly exported
 
 from checkov.common.parsers.node import StrNode, DictNode, ListNode
 from checkov.common.parsers.json.errors import NullError, DuplicateError, DecodeError

checkov/secrets/scan_git_history.py

@@ -4,7 +4,7 @@
 import logging
 import os
 import platform
-from typing import TYPE_CHECKING, Optional, List, Tuple
+from typing import TYPE_CHECKING, Optional, List, Tuple, Union
 
 from detect_secrets.core import scan
 
@@ -148,7 +148,7 @@ def _get_commits_diff(self, last_commit_sha: Optional[str] = None) -> List[Commi
                     )
                 )
                 for file_diff in git_diff:
-                    file_name = file_diff.a_path if file_diff.a_path else file_diff.b_path
+                    file_name: str = file_diff.a_path if file_diff.a_path else file_diff.b_path  # type:ignore
                     if file_name.endswith(FILES_TO_IGNORE_IN_GIT_HISTORY):
                         continue
                     file_path = os.path.join(self.root_folder, file_name)
@@ -157,8 +157,8 @@ def _get_commits_diff(self, last_commit_sha: Optional[str] = None) -> List[Commi
                         logging.debug(f"File was renamed from {file_diff.rename_from} to {file_diff.rename_to}")
                         curr_diff.rename_file(
                             file_path=file_path,
-                            prev_filename=file_diff.rename_from,
-                            new_filename=file_diff.rename_to
+                            prev_filename=file_diff.rename_from or "",
+                            new_filename=file_diff.rename_to or ""
                         )
                         continue
 
@@ -240,7 +240,7 @@ def _get_first_commit(self) -> Commit:
         )
 
         for file_diff in git_diff:
-            file_name = file_diff.b_path
+            file_name: str = file_diff.b_path  # type:ignore
             if file_name.endswith(FILES_TO_IGNORE_IN_GIT_HISTORY):
                 continue
             file_path = os.path.join(self.root_folder, file_name)
@@ -250,9 +250,13 @@ def _get_first_commit(self) -> Commit:
         return first_commit_diff
 
     @staticmethod
-    def get_decoded_diff(diff: bytes) -> str:
+    def get_decoded_diff(diff: Union[str, bytes, None]) -> str:
         if diff is None:
             return ''
+
+        if isinstance(diff, str):
+            return diff
+
         try:
             decoded_diff = diff.decode('utf-8')
         except UnicodeDecodeError as ue:

setup.py

@@ -56,8 +56,8 @@ def run(self) -> None:
     ],
     extras_require={
         "dev": [
-            "pytest==5.3.1",
-            "coverage==5.5",
+            "pytest<8.0.0",
+            "coverage==7.6.1",
             "coverage-badge",
             "GitPython==3.1.41",
             "bandit",
@@ -107,7 +107,7 @@ def run(self) -> None:
         "spdx-tools>=0.8.0,<0.9.0",
         "license-expression<31.0.0,>=30.1.0",
         "rustworkx>=0.13.0,<0.14.0",
-        "pydantic<3.0.0,>=2.0.0"
+        "pydantic<3.0.0,>=2.0.0",
     ],
     dependency_links=[],  # keep it empty, needed for pipenv-setup
     license="Apache License 2.0",

tests/common/checks/test_base_check.py

@@ -1,5 +1,6 @@
 import os
 import unittest
+from typing import List
 
 from unittest import mock
 from parameterized import parameterized
@@ -63,6 +64,10 @@ def scan_entity_conf(self, conf, entity_type):
             return CheckResult.FAILED
 
 
+def _clean_doc(st: str) -> List[str]:
+    return [line.strip() for line in st.splitlines() if not line.isspace()]
+
+
 # noinspection DuplicatedCode
 class TestBaseCheck(unittest.TestCase):
 
@@ -74,11 +79,11 @@ def test_entity_type_is_not_required_in_signature(self):
         # noinspection PyArgumentList
         scan_result = check.scan_entity_conf({}, "Some name")
         self.assertEqual(CheckResult.PASSED, scan_result)
-        self.assertEqual(check.scan_entity_conf.__doc__, """
+        self.assertEqual(_clean_doc(check.scan_entity_conf.__doc__), _clean_doc("""
         My documentation
         :param conf:
         :return:
-        """)
+        """))
 
     def test_invalid_signature_is_detected(self):
         with self.assertRaises(TypeError) as context:

tests/sca_package_2/test_output_reports.py

@@ -1,14 +1,11 @@
 import json
-import sys
 import xml
 import xml.dom.minidom
 import os
 from operator import itemgetter
 from pathlib import Path
 from typing import List
-from unittest import mock
 
-import pytest
 from pytest_mock import MockerFixture
 
 from checkov.common.bridgecrew.check_type import CheckType
@@ -31,7 +28,8 @@ def _get_deterministic_items_in_cyclonedx(pretty_xml_as_list: List[str]) -> List
         if not any(word in line for word in black_list_words):
             if i == 0 or not any(tool_name in pretty_xml_as_list[i - 1] for tool_name in
                                  ("<name>checkov</name>", "<name>cyclonedx-python-lib</name>")):
-                filtered_list.append(line)
+                filtered_list.append(
+                    line.replace('&quot;', '\"'))  # fixes differences in xml prettyprint between python 3.12 and 3.13
     return filtered_list
 
 
@@ -153,7 +151,7 @@ def test_get_cyclonedx_report(sca_package_2_report, tmp_path: Path):
     actual_pretty_xml_as_list = _get_deterministic_items_in_cyclonedx(pretty_xml_as_string.split("\n"))
     expected_pretty_xml_as_list = _get_deterministic_items_in_cyclonedx(expected_pretty_xml.split("\n"))
 
-    assert actual_pretty_xml_as_list == expected_pretty_xml_as_list
+    assert '\n'.join(actual_pretty_xml_as_list) == '\n'.join(expected_pretty_xml_as_list)
 
 
 def test_get_cyclonedx_report_with_licenses_with_comma(sca_package_report_2_with_comma_in_licenses, tmp_path: Path):