From 388c5cf75e7470b21c29710336fe711390363acc Mon Sep 17 00:00:00 2001 From: Tomas Pereira de Vasconcelos Date: Sat, 28 Sep 2024 03:56:25 +0100 Subject: [PATCH 1/6] Escape `ignored_directories` --- checkov/common/runners/base_runner.py | 10 +++++++++- checkov/secrets/utils.py | 7 +++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/checkov/common/runners/base_runner.py b/checkov/common/runners/base_runner.py index f43d6caa02a..56add517644 100644 --- a/checkov/common/runners/base_runner.py +++ b/checkov/common/runners/base_runner.py @@ -43,7 +43,15 @@ def strtobool(val: str) -> int: raise ValueError("invalid boolean value %r for environment variable CKV_IGNORE_HIDDEN_DIRECTORIES" % (val,)) -IGNORED_DIRECTORIES_ENV = os.getenv("CKV_IGNORED_DIRECTORIES", "node_modules,.terraform,.serverless") +def re_dir(path: str) -> str: + """Compile a regex pattern that matches paths containing the given directory at any level.""" + return rf"(^|.*/){re.escape(path)}($|/.*)" + + +IGNORED_DIRECTORIES_ENV = os.getenv( + "CKV_IGNORED_DIRECTORIES", + ",".join(re_dir(p) for p in ["node_modules", ".terraform", ".serverless"]) +) IGNORE_HIDDEN_DIRECTORY_ENV = strtobool(os.getenv("CKV_IGNORE_HIDDEN_DIRECTORIES", "True")) ignored_directories = IGNORED_DIRECTORIES_ENV.split(",") diff --git a/checkov/secrets/utils.py b/checkov/secrets/utils.py index 16e5a9cbffa..82ef0155b65 100644 --- a/checkov/secrets/utils.py +++ b/checkov/secrets/utils.py @@ -4,10 +4,13 @@ import re from collections.abc import Iterable -from checkov.common.runners.base_runner import ignored_directories, safe_remove +from checkov.common.runners.base_runner import ignored_directories, safe_remove, re_dir from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR -EXCLUDED_PATHS = [*ignored_directories, DEFAULT_EXTERNAL_MODULES_DIR, ".idea", ".git", "venv"] +EXCLUDED_PATHS = [ + *ignored_directories, + *(re_dir(p) for p in [DEFAULT_EXTERNAL_MODULES_DIR, ".idea", ".git", "venv"]) +] def filter_excluded_paths( From 590b7cd994196521d635b4ff19c8c0ca7762638a Mon Sep 17 00:00:00 2001 From: Tomas Pereira de Vasconcelos Date: Sat, 28 Sep 2024 17:30:37 +0100 Subject: [PATCH 2/6] Implement windows compatability in `re_dir` --- checkov/common/runners/base_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/checkov/common/runners/base_runner.py b/checkov/common/runners/base_runner.py index 56add517644..2bc05166aff 100644 --- a/checkov/common/runners/base_runner.py +++ b/checkov/common/runners/base_runner.py @@ -45,7 +45,8 @@ def strtobool(val: str) -> int: def re_dir(path: str) -> str: """Compile a regex pattern that matches paths containing the given directory at any level.""" - return rf"(^|.*/){re.escape(path)}($|/.*)" + sep = re.escape(os.sep) # windows compatibility + return rf"(^|.*{sep}){re.escape(path)}($|{sep}.*)" IGNORED_DIRECTORIES_ENV = os.getenv( From 4bbb2bcdb895001cec7aa6efe7e447a9924c0907 Mon Sep 17 00:00:00 2001 From: Tomas Pereira de Vasconcelos Date: Sat, 28 Sep 2024 17:32:40 +0100 Subject: [PATCH 3/6] refactor `EXCLUDED_PATHS` --- checkov/secrets/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/checkov/secrets/utils.py b/checkov/secrets/utils.py index 82ef0155b65..1b8fed3b408 100644 --- a/checkov/secrets/utils.py +++ b/checkov/secrets/utils.py @@ -9,7 +9,10 @@ EXCLUDED_PATHS = [ *ignored_directories, - *(re_dir(p) for p in [DEFAULT_EXTERNAL_MODULES_DIR, ".idea", ".git", "venv"]) + re_dir(DEFAULT_EXTERNAL_MODULES_DIR), + re_dir(".idea"), + re_dir(".git"), + re_dir("venv"), ] From 1f9cda93b249df1dcf7d36b9c3bcd92b4dd5e6c9 Mon Sep 17 00:00:00 2001 From: Tomas Pereira de Vasconcelos Date: Sat, 28 Sep 2024 17:39:51 +0100 Subject: [PATCH 4/6] Add test for `re_dir` --- tests/common/runners/test_base_runner.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/common/runners/test_base_runner.py b/tests/common/runners/test_base_runner.py index 63226a7a96b..e50bf629ed6 100644 --- a/tests/common/runners/test_base_runner.py +++ b/tests/common/runners/test_base_runner.py @@ -3,12 +3,19 @@ from typing import Optional, List from checkov.common.output.report import Report -from checkov.common.runners.base_runner import filter_ignored_paths, BaseRunner +from checkov.common.runners.base_runner import filter_ignored_paths, BaseRunner, re_dir from checkov.runner_filter import RunnerFilter class TestBaseRunner(unittest.TestCase): + def test_re_dir(self): + sep = '\\' if os.name == 'nt' else '/' + # add regex prefix and suffix to the (unmodified) directory name + self.assertEqual(re_dir('dir'), fr'(^|.*{sep})dir($|{sep}.*)') + # escape the directory name (but leave the os separator unaltered) + self.assertEqual(re_dir('.dir1/.dir2'), fr'(^|.*{sep})\.dir1/\.dir2($|{sep}.*)') + def test_filter_ignored_directories_regex_legacy(self): d_names = ['bin', 'integration_tests', 'tests', 'docs', '.github', 'checkov', 'venv', '.git', 'kubernetes', '.idea'] expected = ['bin', 'docs', 'checkov', 'venv', 'kubernetes'] From d7b0f8feb57a9115c5b080ad5c35b47925f86284 Mon Sep 17 00:00:00 2001 From: Tomas Pereira de Vasconcelos Date: Sat, 28 Sep 2024 17:59:42 +0100 Subject: [PATCH 5/6] Add test: `TestBaseRunner::tests_re_dir_test_pattern` --- tests/common/runners/test_base_runner.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/common/runners/test_base_runner.py b/tests/common/runners/test_base_runner.py index e50bf629ed6..4baef5c9095 100644 --- a/tests/common/runners/test_base_runner.py +++ b/tests/common/runners/test_base_runner.py @@ -1,4 +1,5 @@ import os +import re import unittest from typing import Optional, List @@ -16,6 +17,31 @@ def test_re_dir(self): # escape the directory name (but leave the os separator unaltered) self.assertEqual(re_dir('.dir1/.dir2'), fr'(^|.*{sep})\.dir1/\.dir2($|{sep}.*)') + def tests_re_dir_test_pattern(self): + dir_name_to_ignore = ".hidden" + dir_name_to_ignore_re = re.compile(re_dir(dir_name_to_ignore)) + paths_to_ignore = [ + ".hidden", + "/.hidden", + "/path/.hidden", + "path/.hidden", + ".hidden/path", + "path/.hidden/path", + "path/.hidden/path/", + ] + paths_to_keep = [ + ".hidden1", + "not.hidden", + "nothidden", + "not.hidden/path", + "path/not.hidden", + "also/nothidden", + "hidden/not", + "also/hidden/not", + ] + self.assertTrue(all(dir_name_to_ignore_re.match(p) for p in paths_to_ignore)) + self.assertFalse(any(dir_name_to_ignore_re.match(p) for p in paths_to_keep)) + def test_filter_ignored_directories_regex_legacy(self): d_names = ['bin', 'integration_tests', 'tests', 'docs', '.github', 'checkov', 'venv', '.git', 'kubernetes', '.idea'] expected = ['bin', 'docs', 'checkov', 'venv', 'kubernetes'] From 4c1d359e6214f273933dc6dd3a202cfd53cd7835 Mon Sep 17 00:00:00 2001 From: Tomas Pereira de Vasconcelos Date: Sat, 28 Sep 2024 18:00:38 +0100 Subject: [PATCH 6/6] typo --- tests/common/runners/test_base_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/runners/test_base_runner.py b/tests/common/runners/test_base_runner.py index 4baef5c9095..82f4b110504 100644 --- a/tests/common/runners/test_base_runner.py +++ b/tests/common/runners/test_base_runner.py @@ -17,7 +17,7 @@ def test_re_dir(self): # escape the directory name (but leave the os separator unaltered) self.assertEqual(re_dir('.dir1/.dir2'), fr'(^|.*{sep})\.dir1/\.dir2($|{sep}.*)') - def tests_re_dir_test_pattern(self): + def tests_re_dir_pattern(self): dir_name_to_ignore = ".hidden" dir_name_to_ignore_re = re.compile(re_dir(dir_name_to_ignore)) paths_to_ignore = [