From cce8cf2b24abf71e65ced9d813c49d48c442e832 Mon Sep 17 00:00:00 2001 From: jyejare Date: Tue, 5 Dec 2023 16:53:56 +0530 Subject: [PATCH 1/4] Test Markers as Test Fields --- betelgeuse/__init__.py | 2 +- betelgeuse/collector.py | 31 +++++++++++++++++++++++++++++++ betelgeuse/parser.py | 27 +++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/betelgeuse/__init__.py b/betelgeuse/__init__.py index a74ebd7..30121e7 100644 --- a/betelgeuse/__init__.py +++ b/betelgeuse/__init__.py @@ -423,7 +423,7 @@ def get_requirement_field_values(config, requirement): def update_testcase_fields(config, testcase): """Apply testcase fields default values and transformations.""" - if testcase.docstring and not type(testcase.docstring) == str: + if testcase.docstring and not isinstance(testcase.docstring, str): testcase.docstring = testcase.docstring.decode('utf8') # Check if any field needs a default value diff --git a/betelgeuse/collector.py b/betelgeuse/collector.py index 95edce6..00511f9 100644 --- a/betelgeuse/collector.py +++ b/betelgeuse/collector.py @@ -6,6 +6,7 @@ import os from betelgeuse.parser import parse_docstring +from betelgeuse.parser import parse_markers from betelgeuse.source_generator import gen_source @@ -88,11 +89,20 @@ def __init__(self, function_def, parent_class=None, testmodule=None): for decorator in self.parent_class_def.decorator_list ] self._parse_docstring() + self._parse_markers() self.junit_id = self._generate_junit_id() if 'id' not in self.fields: self.fields['id'] = self.junit_id + def _parse_markers(self): + """Parse module, class and function markers.""" + markers = [self.module_def.marker_list, + self.class_decorators, + self.decorators] + if markers: + self.fields.update({'markers': parse_markers(markers)}) + def _parse_docstring(self): """Parse package, module, class and function docstrings.""" if self.docstring is None: @@ -137,12 +147,33 @@ def is_test_module(filename): return False +def _module_markers(module_def): + """Extract markers applied to testcases from the test module level. + + The markers list would be collected from the pytestmark global variable. + """ + markers = [] + for node in module_def.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == 'pytestmark': + if isinstance(node.value, ast.List): + for item in node.value.elts: + if isinstance(item, ast.Attribute): + markers.append(item.attr) + elif isinstance(node.value, ast.Attribute): + markers.append(node.value.attr) + return markers or None + + def _get_tests(path): """Collect tests for the test module located at ``path``.""" tests = [] with open(path) as handler: root = ast.parse(handler.read()) root.path = path # TODO improve how to pass the path to TestFunction + # Updating test module with module level markers + root.__dict__['marker_list'] = _module_markers(root) for node in ast.iter_child_nodes(root): if isinstance(node, ast.ClassDef): [ diff --git a/betelgeuse/parser.py b/betelgeuse/parser.py index e4ea5d7..9917e29 100644 --- a/betelgeuse/parser.py +++ b/betelgeuse/parser.py @@ -187,3 +187,30 @@ def parse_docstring(docstring=None): field_value = output fields_dict[field_name] = field_value return fields_dict + + +def parse_markers(all_markers=None): + """Parse the markers.""" + ignore_list = ['parametrize', 'skipif', 'usefixtures', 'skip_if_not_set'] + resolved_markers = [] + + def _process_marker(marker): + # Fetching exact marker name + marker_name = marker.split('mark.')[-1] if 'mark' in marker else marker + + # ignoring the marker if in ignore list + if not any(ignore_word in marker_name for ignore_word in ignore_list): + resolved_markers.append(marker_name) + + for sec_marker in all_markers: + # If the marker is none + if not sec_marker: + continue + elif isinstance(sec_marker, list): + for marker in sec_marker: + _process_marker(marker) + else: + _process_marker(sec_marker) + + resolved_markers = ', '.join(resolved_markers) + return resolved_markers From 6bdf0219fb39f9052134b83e72ab55ebc52a93da Mon Sep 17 00:00:00 2001 From: jyejare Date: Mon, 11 Dec 2023 13:59:08 +0530 Subject: [PATCH 2/4] Markers parsing unit test --- tests/data/test_sample.py | 14 ++++++++++++++ tests/test_collector.py | 4 ++-- tests/test_parser.py | 24 ++++++++++++++++++++++++ tests/test_source_generator.py | 11 +++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/data/test_sample.py b/tests/data/test_sample.py index 4233131..2793f50 100644 --- a/tests/data/test_sample.py +++ b/tests/data/test_sample.py @@ -1,8 +1,11 @@ # encoding=utf-8 """Sample test module.""" import unittest +import pytest +pytestmark = [pytest.mark.run_in_one_thread, pytest.mark.tier1] + CONSTANT = 'contant-value' @@ -72,3 +75,14 @@ def test_method(self): def test_without_docstring(self): # noqa: D102 pass + + +@pytest.mark.on_prem_provisioning +class TestclasswithMarkers: + """Class to verify tests markers are collected from class.""" + + @pytest.mark.skipif(2 == 3, reason='2 is not 3') + @pytest.mark.osp + def test_markers_sample(self): + """Test for markers at test level.""" + assert True diff --git a/tests/test_collector.py b/tests/test_collector.py index bfec3da..5c8f4c1 100644 --- a/tests/test_collector.py +++ b/tests/test_collector.py @@ -11,7 +11,7 @@ def test_collect_tests(path): """Check if ``collect_tests`` 'tests/data'collect tests.""" tests = collector.collect_tests(path) assert 'tests/data/test_sample.py' in tests - assert len(tests['tests/data/test_sample.py']) == 4 + assert len(tests['tests/data/test_sample.py']) == 5 # Check if we are not doing a specific python module collection if path.endswith('.py'): @@ -30,7 +30,7 @@ def test_collect_ignore_path(ignore_path): tests = collector.collect_tests('tests/data', [ignore_path]) assert 'tests/data/ignore_dir/test_ignore_dir.py' not in tests assert 'tests/data/test_sample.py' in tests - assert len(tests['tests/data/test_sample.py']) == 4 + assert len(tests['tests/data/test_sample.py']) == 5 @pytest.mark.parametrize('filename', ('test_module.py', 'module_test.py')) diff --git a/tests/test_parser.py b/tests/test_parser.py index b05ac49..f6f7829 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -83,3 +83,27 @@ def test_parse_rst_special_characters(): u'

String with special character like é

\n' u'\n' ) + + +def test_parse_markers(): + """ + Test if the markers list is parsed. + + List should be comma separated list of markers from all levels after + removing 'pytest.mark' text and ignore some markers. + """ + _mod_markers = ['pytest.mark.e2e', 'pytest.mark.destructive'] + _class_markers = [ + 'pytest.mark.on_prem_provisioning', + "pytest.mark.usefixtures('cleandir')" + ] + _test_markers = [ + "pytest.mark.parametrize('something', ['a', 'b'])", + 'pytest.mark.skipif(not settings.robottelo.REPOS_HOSTING_URL)', + 'pytest.mark.tier1' + ] + _all_markers = [_mod_markers, _class_markers, _test_markers] + + expected = 'e2e, destructive, on_prem_provisioning, tier1' + + assert parser.parse_markers(_all_markers) == expected diff --git a/tests/test_source_generator.py b/tests/test_source_generator.py index dab73dc..b369853 100644 --- a/tests/test_source_generator.py +++ b/tests/test_source_generator.py @@ -37,3 +37,14 @@ def test_source_generator(): '(lambda v: (v if v else None))' ')', ] + + +def test_source_markers(): + """Verifies if the test collection collects test markers.""" + tests = collector.collect_tests('tests/data/test_sample.py') + marked_test = [ + test for test in tests['tests/data/test_sample.py'] + if test.name == 'test_markers_sample' + ].pop() + assert marked_test.fields['markers'] == ('run_in_one_thread, tier1, ' + 'on_prem_provisioning, osp') From d5e901916a12d3a46886829809f340fdbf7e39fb Mon Sep 17 00:00:00 2001 From: jyejare Date: Wed, 13 Dec 2023 19:18:56 +0530 Subject: [PATCH 3/4] Configurable Markers ignore list --- betelgeuse/__init__.py | 2 +- betelgeuse/collector.py | 25 +++++++++++++++---------- betelgeuse/parser.py | 23 +++++++++++++++-------- tests/test_parser.py | 11 +++++++---- tests/test_source_generator.py | 23 ++++++++++++++++++++++- 5 files changed, 60 insertions(+), 24 deletions(-) diff --git a/betelgeuse/__init__.py b/betelgeuse/__init__.py index 30121e7..dac5ecc 100644 --- a/betelgeuse/__init__.py +++ b/betelgeuse/__init__.py @@ -656,7 +656,7 @@ def test_case( testcases.append(properties) source_testcases = itertools.chain(*collector.collect_tests( - source_code_path, collect_ignore_path).values()) + source_code_path, collect_ignore_path, config=config).values()) for testcase in source_testcases: testcases.append( create_xml_testcase(config, testcase, automation_script_format)) diff --git a/betelgeuse/collector.py b/betelgeuse/collector.py index 00511f9..48b0e8e 100644 --- a/betelgeuse/collector.py +++ b/betelgeuse/collector.py @@ -22,7 +22,10 @@ def __init__(self, title, fields=None): class TestFunction(object): """Wrapper for ``ast.FunctionDef`` which parse docstring information.""" - def __init__(self, function_def, parent_class=None, testmodule=None): + def __init__( + self, function_def, parent_class=None, testmodule=None, + config=None + ): """``ast.FunctionDef`` instance used to extract information.""" #: The unparsed testcase docstring self.docstring = ast.get_docstring(function_def) @@ -89,19 +92,19 @@ def __init__(self, function_def, parent_class=None, testmodule=None): for decorator in self.parent_class_def.decorator_list ] self._parse_docstring() - self._parse_markers() + self._parse_markers(config) self.junit_id = self._generate_junit_id() if 'id' not in self.fields: self.fields['id'] = self.junit_id - def _parse_markers(self): + def _parse_markers(self, config=None): """Parse module, class and function markers.""" markers = [self.module_def.marker_list, self.class_decorators, self.decorators] if markers: - self.fields.update({'markers': parse_markers(markers)}) + self.fields.update({'markers': parse_markers(markers, config)}) def _parse_docstring(self): """Parse package, module, class and function docstrings.""" @@ -166,7 +169,7 @@ def _module_markers(module_def): return markers or None -def _get_tests(path): +def _get_tests(path, config=None): """Collect tests for the test module located at ``path``.""" tests = [] with open(path) as handler: @@ -177,20 +180,22 @@ def _get_tests(path): for node in ast.iter_child_nodes(root): if isinstance(node, ast.ClassDef): [ - tests.append(TestFunction(subnode, node, root)) + tests.append(TestFunction(subnode, node, root, config)) for subnode in ast.iter_child_nodes(node) if isinstance(subnode, ast.FunctionDef) and subnode.name.startswith('test_') ] elif (isinstance(node, ast.FunctionDef) and node.name.startswith('test_')): - tests.append(TestFunction(node, testmodule=root)) + tests.append(TestFunction( + node, testmodule=root, config=config)) return tests -def collect_tests(path, ignore_paths=None): +def collect_tests(path, ignore_paths=None, config=None): """Walk ``path`` and collect test methods and functions found. + :param config: The config object of `config.BetelgeuseConfig` :param path: Either a file or directory path to look for test methods and functions. :return: A dict mapping a test module path and its test cases. @@ -201,7 +206,7 @@ def collect_tests(path, ignore_paths=None): tests = collections.OrderedDict() if os.path.isfile(path) and path not in ignore_paths: if is_test_module(os.path.basename(path)): - tests[path] = _get_tests(path) + tests[path] = _get_tests(path, config) return tests for dirpath, _, filenames in os.walk(path): if dirpath in ignore_paths: @@ -211,5 +216,5 @@ def collect_tests(path, ignore_paths=None): if path in ignore_paths: continue if is_test_module(filename): - tests[path] = _get_tests(path) + tests[path] = _get_tests(path, config) return tests diff --git a/betelgeuse/parser.py b/betelgeuse/parser.py index 9917e29..03e1058 100644 --- a/betelgeuse/parser.py +++ b/betelgeuse/parser.py @@ -189,18 +189,26 @@ def parse_docstring(docstring=None): return fields_dict -def parse_markers(all_markers=None): - """Parse the markers.""" - ignore_list = ['parametrize', 'skipif', 'usefixtures', 'skip_if_not_set'] +def parse_markers(all_markers=None, config=None): + """Parse the markers from module, class and test level for a test. + + This removes the mark prepended words and also pops out the + ignorable marker from the list received from the config object. + + :returns string: Comma separated list of markers from all levels for a test + """ resolved_markers = [] + ignore_list = getattr(config, 'MARKERS_IGNORE_LIST', None) def _process_marker(marker): # Fetching exact marker name - marker_name = marker.split('mark.')[-1] if 'mark' in marker else marker + marker_name = marker.split('mark.')[-1] # ignoring the marker if in ignore list - if not any(ignore_word in marker_name for ignore_word in ignore_list): - resolved_markers.append(marker_name) + if ignore_list and any( + ignore_word in marker_name for ignore_word in ignore_list): + return + resolved_markers.append(marker_name) for sec_marker in all_markers: # If the marker is none @@ -212,5 +220,4 @@ def _process_marker(marker): else: _process_marker(sec_marker) - resolved_markers = ', '.join(resolved_markers) - return resolved_markers + return ', '.join(resolved_markers) diff --git a/tests/test_parser.py b/tests/test_parser.py index f6f7829..b73ce95 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,6 +1,7 @@ # coding=utf-8 """Tests for :mod:`betelgeuse.parser`.""" import pytest +import mock from betelgeuse import parser @@ -92,7 +93,7 @@ def test_parse_markers(): List should be comma separated list of markers from all levels after removing 'pytest.mark' text and ignore some markers. """ - _mod_markers = ['pytest.mark.e2e', 'pytest.mark.destructive'] + _mod_markers = 'pytest.mark.destructive' _class_markers = [ 'pytest.mark.on_prem_provisioning', "pytest.mark.usefixtures('cleandir')" @@ -104,6 +105,8 @@ def test_parse_markers(): ] _all_markers = [_mod_markers, _class_markers, _test_markers] - expected = 'e2e, destructive, on_prem_provisioning, tier1' - - assert parser.parse_markers(_all_markers) == expected + expected = 'destructive, on_prem_provisioning, tier1' + config = mock.MagicMock() + config.MARKERS_IGNORE_LIST = [ + 'parametrize', 'skipif', 'usefixtures', 'skip_if_not_set'] + assert parser.parse_markers(_all_markers, config=config) == expected diff --git a/tests/test_source_generator.py b/tests/test_source_generator.py index b369853..b8f804a 100644 --- a/tests/test_source_generator.py +++ b/tests/test_source_generator.py @@ -1,5 +1,6 @@ """Tests for :mod:`betelgeuse.source_generator`.""" from betelgeuse import collector +import mock def test_source_generator(): @@ -41,10 +42,30 @@ def test_source_generator(): def test_source_markers(): """Verifies if the test collection collects test markers.""" - tests = collector.collect_tests('tests/data/test_sample.py') + config = mock.Mock() + config.MARKERS_IGNORE_LIST = [ + 'parametrize', 'skipif', 'usefixtures', 'skip_if_not_set'] + tests = collector.collect_tests('tests/data/test_sample.py', config=config) marked_test = [ test for test in tests['tests/data/test_sample.py'] if test.name == 'test_markers_sample' ].pop() assert marked_test.fields['markers'] == ('run_in_one_thread, tier1, ' 'on_prem_provisioning, osp') + + +def test_source_singular_module_marker(): + """Verifies the single module level marker is retrieved.""" + mod_string = 'import pytest\n\npytestmark = pytest.mark.tier2' \ + '\n\ndef test_sing():\n\tpass' + with open('/tmp/test_singular.py', 'w') as tfile: + tfile.writelines(mod_string) + + config = mock.Mock() + config.MARKERS_IGNORE_LIST = ['tier3'] + tests = collector.collect_tests('/tmp/test_singular.py', config=config) + marked_test = [ + test for test in tests['/tmp/test_singular.py'] + if test.name == 'test_sing' + ].pop() + assert marked_test.fields['markers'] == 'tier2' From c47bee78d818a5232f7b895e53e5d289c6fc36c6 Mon Sep 17 00:00:00 2001 From: Jitendra Yejare Date: Tue, 19 Dec 2023 17:06:14 +0530 Subject: [PATCH 4/4] regexed ignore list in marker Co-authored-by: Roman Plevka --- betelgeuse/parser.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/betelgeuse/parser.py b/betelgeuse/parser.py index 03e1058..840aff0 100644 --- a/betelgeuse/parser.py +++ b/betelgeuse/parser.py @@ -1,4 +1,5 @@ """Parsers for test docstrings.""" +import re from collections import namedtuple from io import StringIO from xml.dom import minidom @@ -200,14 +201,21 @@ def parse_markers(all_markers=None, config=None): resolved_markers = [] ignore_list = getattr(config, 'MARKERS_IGNORE_LIST', None) - def _process_marker(marker): - # Fetching exact marker name - marker_name = marker.split('mark.')[-1] + def _process_marker(_marker): + + marker_name = re.findall( + r'(?:pytest\.mark\.)?([^(\s()]+)(?=\s*\(|\s*$)', _marker) + if marker_name: + marker_name = marker_name[0] # ignoring the marker if in ignore list if ignore_list and any( - ignore_word in marker_name for ignore_word in ignore_list): + re.fullmatch( + ignore_word, marker_name + ) for ignore_word in ignore_list + ): return + resolved_markers.append(marker_name) for sec_marker in all_markers: