From 623e3db28bc1e372a84242733b7bad29c7f8f7b7 Mon Sep 17 00:00:00 2001 From: Jakob Ruckel Date: Thu, 10 Mar 2022 09:10:34 +0100 Subject: [PATCH 1/7] Analyzer: Write per-commit stats to CSV --- src/cfgnet/analyze/analyzer.py | 51 ++++++--- src/cfgnet/conflicts/conflict.py | 4 +- src/cfgnet/utility/statistics.py | 146 ++++++++++++++++++++++++++ tests/cfgnet/analyze/test_analyzer.py | 13 ++- 4 files changed, 195 insertions(+), 19 deletions(-) create mode 100644 src/cfgnet/utility/statistics.py diff --git a/src/cfgnet/analyze/analyzer.py b/src/cfgnet/analyze/analyzer.py index 4bb8ddd..f053edb 100644 --- a/src/cfgnet/analyze/analyzer.py +++ b/src/cfgnet/analyze/analyzer.py @@ -22,12 +22,14 @@ from cfgnet.vcs.git_history import GitHistory from cfgnet.network.network import Network, NetworkConfiguration from cfgnet.analyze.csv_writer import CSVWriter +from cfgnet.utility.statistics import CommitStatistics class Analyzer: def __init__(self, cfg: NetworkConfiguration): self.cfg: NetworkConfiguration = cfg self.conflicts_cvs_path: Optional[str] = None + self.stats_csv_path: Optional[str] = None self.time_last_progress_print: float = 0 self._setup_dirs() @@ -45,9 +47,13 @@ def _setup_dirs(self) -> None: self.conflicts_csv_path = os.path.join( analysis_dir, f"conflicts_{self.cfg.project_name()}.csv" ) + self.commit_stats_file = os.path.join( + analysis_dir, f"commit_stats_{self.cfg.project_name()}.csv" + ) - if os.path.exists(self.conflicts_csv_path): - os.remove(self.conflicts_csv_path) + for path in [self.conflicts_csv_path, self.commit_stats_file]: + if os.path.exists(path): + os.remove(path) def _print_progress(self, num_commit: int, final: bool = False) -> None: """Print the progress of th analysis.""" @@ -76,19 +82,34 @@ def analyze_commit_history(self) -> None: try: ref_network = Network.init_network(cfg=self.cfg) - while history.has_next_commit(): - commit = history.next_commit() - - detected_conflicts, ref_network = ref_network.validate( - commit.hexsha - ) - - conflicts.update(detected_conflicts) - - self._print_progress(num_commit=history.commit_index + 1) - - if commit.hexsha == commit_hash_pre_analysis: - break + with open( + self.commit_stats_file, "w+", encoding="utf-8" + ) as stats_csv: + CommitStatistics.setup_writer(stats_csv) + stats_prev = CommitStatistics() + while history.has_next_commit(): + commit = history.next_commit() + commit_number = history.commit_index + + detected_conflicts, ref_network = ref_network.validate( + commit.hexsha + ) + + conflicts.update(detected_conflicts) + + stats = CommitStatistics.calc_stats( + commit=commit, + commit_number=commit_number, + network=ref_network, + conflicts=conflicts, + prev=stats_prev, + ) + CommitStatistics.write_row(stats) + + self._print_progress(num_commit=commit_number + 1) + + if commit.hexsha == commit_hash_pre_analysis: + break except Exception as error: logging.error( diff --git a/src/cfgnet/conflicts/conflict.py b/src/cfgnet/conflicts/conflict.py index f4b6239..5b4ce39 100644 --- a/src/cfgnet/conflicts/conflict.py +++ b/src/cfgnet/conflicts/conflict.py @@ -19,7 +19,7 @@ import hashlib import logging -from typing import Any, List, Optional, Set +from typing import Any, Iterable, Optional, Set from cfgnet.linker.link import Link from cfgnet.network.nodes import ArtifactNode, OptionNode, ValueNode @@ -64,7 +64,7 @@ def is_involved(self, node: Any) -> bool: """ @staticmethod - def count_total(conflicts: List[Conflict]) -> int: + def count_total(conflicts: Iterable[Conflict]) -> int: """Total conflict count across a list.""" return sum([conflict.count() for conflict in conflicts]) diff --git a/src/cfgnet/utility/statistics.py b/src/cfgnet/utility/statistics.py new file mode 100644 index 0000000..1260bb1 --- /dev/null +++ b/src/cfgnet/utility/statistics.py @@ -0,0 +1,146 @@ +# This file is part of the CfgNet module. +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program. If not, see . + +import csv +from collections import OrderedDict +from typing import Optional, Iterable + +from cfgnet.network.network import Network +from cfgnet.network.nodes import OptionNode, ArtifactNode, ValueNode +from cfgnet.conflicts.conflict import Conflict +from cfgnet.vcs.git import Commit + + +class CommitStatistics: + writer: Optional[csv.DictWriter] = None + + def __init__(self, commit=None): + self.commit_hash = None + self.commit_number = 0 + self.total_artifact_nodes = 0 + self.total_option_nodes = 0 + self.num_value_nodes_added = 0 + self.num_value_nodes_removed = 0 + self.num_value_nodes_changed = 0 + self.total_links = 0 + self.total_links_changed = 0 + self.num_configuration_files_changed = 0 + self.num_detected_conflicts = 0 + self.num_fixed_conflicts = 0 + + self._value_node_ids = set() + self._value_node_parent_ids = set() + self._links = set() + self._conflicts = set() + + if commit: + self.commit_hash = commit.hexsha + + @staticmethod + # pylint: disable=protected-access + def calc_stats( + commit: Commit, + commit_number: int, + network: Network, + conflicts: Iterable[Conflict], + prev: "CommitStatistics", + ) -> "CommitStatistics": + stats = CommitStatistics(commit=commit) + + stats._value_node_ids = { + node.id for node in network.get_nodes(ValueNode) + } + stats._value_node_parent_ids = { + node.parent.id for node in network.get_nodes(ValueNode) + } + + stats.commit_number = commit_number + artifact_nodes = network.get_nodes(ArtifactNode) + stats.total_artifact_nodes = len(artifact_nodes) + stats.total_option_nodes = len(network.get_nodes(OptionNode)) + stats.num_value_nodes_added = len( + prev._value_node_ids.difference(stats._value_node_parent_ids) + ) + stats.num_value_nodes_removed = len( + stats._value_node_ids.difference(prev._value_node_parent_ids) + ) + + stats.num_value_nodes_changed = 0 + value_parents_in_common = stats._value_node_parent_ids.intersection( + prev._value_node_parent_ids + ) + for node in value_parents_in_common: + + def same_parent(node_id, parent=node): + return node_id.startswith(parent) + + value_prev = list(filter(same_parent, prev._value_node_ids))[0] + value_new = list(filter(same_parent, stats._value_node_ids))[0] + if value_prev != value_new: + stats.num_value_nodes_changed += 1 + + stats._links = set(network.links) + stats.total_links = len(stats._links) + stats.total_links_changed = len( + prev._links.symmetric_difference(stats._links) + ) + + config_files = {node.rel_file_path for node in artifact_nodes} + files_changed = set(commit.stats.files.keys()) + config_files_changed = config_files.intersection(files_changed) + stats.num_configuration_files_changed = len(config_files_changed) + + stats._conflicts = set(conflicts) + new_conflicts = stats._conflicts.difference(prev._conflicts) + fixed_conflicts = prev._conflicts.difference(stats._conflicts) + stats.num_detected_conflicts = Conflict.count_total(new_conflicts) + stats.num_fixed_conflicts = Conflict.count_total(fixed_conflicts) + + return stats + + @staticmethod + def setup_writer(commit_stats_file) -> None: + CommitStatistics.writer = csv.DictWriter( + commit_stats_file, fieldnames=CommitStatistics.fieldnames() + ) + CommitStatistics.writer.writeheader() + + @staticmethod + def write_row(stats): + if CommitStatistics.writer is not None: + CommitStatistics.writer.writerow(stats.data_dict()) + + @staticmethod + def fieldnames(): + return list(CommitStatistics().data_dict().keys()) + + def data_dict(self): + data = OrderedDict( + { + "commit_number": self.commit_number, + "commit_hash": self.commit_hash, + "total_option_nodes": self.total_option_nodes, + "num_value_nodes_changed": self.num_value_nodes_changed, + "total_artifact_nodes": self.total_artifact_nodes, + "total_links": self.total_links, + "total_links_changed": self.total_links_changed, + "num_configuration_files_changed": self.num_configuration_files_changed, + "num_detected_conflicts": self.num_detected_conflicts, + "num_fixed_conflicts": self.num_fixed_conflicts, + "num_value_nodes_added": self.num_value_nodes_added, + "num_value_nodes_removed": self.num_value_nodes_removed, + } + ) + return data diff --git a/tests/cfgnet/analyze/test_analyzer.py b/tests/cfgnet/analyze/test_analyzer.py index 4655d12..e835787 100644 --- a/tests/cfgnet/analyze/test_analyzer.py +++ b/tests/cfgnet/analyze/test_analyzer.py @@ -55,11 +55,20 @@ def test_analyze(get_config): conflicts_csv_path = os.path.join( analysis_dir, f"conflicts_{project_name}.csv" ) + stats_csv_path = os.path.join( + analysis_dir, f"commit_stats_{project_name}.csv" + ) assert os.path.exists(conflicts_csv_path) - with open(conflicts_csv_path, "r", encoding="utf-8") as csv_stats_file: - reader = csv.DictReader(csv_stats_file) + with open(conflicts_csv_path, "r", encoding="utf-8") as csv_conflicts_file: + reader = csv.DictReader(csv_conflicts_file) rows = list(reader) assert len(rows) == 2 + + with open(stats_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + assert len(rows) == 1 From dca33c0503470699b94025fc8f46986bb20410d1 Mon Sep 17 00:00:00 2001 From: simisimon Date: Tue, 15 Mar 2022 19:04:59 +0100 Subject: [PATCH 2/7] Add test for commit stats --- tests/cfgnet/analyze/test_commit_stats.py | 122 ++++++++++++++++++ .../commit_stats/0001-Add-package.json.patch | 32 +++++ .../commit_stats/0002-Add-Dockerfile.patch | 24 ++++ ...0003-Change-one-option-in-Dockerfile.patch | 23 ++++ ...0004-Delete-one-option-in-Dockerfile.patch | 22 ++++ .../0005-Add-one-option-in-Dockerfile.patch | 22 ++++ .../0006-Change-three-options.patch | 44 +++++++ 7 files changed, 289 insertions(+) create mode 100644 tests/cfgnet/analyze/test_commit_stats.py create mode 100644 tests/test_repos/commit_stats/0001-Add-package.json.patch create mode 100644 tests/test_repos/commit_stats/0002-Add-Dockerfile.patch create mode 100644 tests/test_repos/commit_stats/0003-Change-one-option-in-Dockerfile.patch create mode 100644 tests/test_repos/commit_stats/0004-Delete-one-option-in-Dockerfile.patch create mode 100644 tests/test_repos/commit_stats/0005-Add-one-option-in-Dockerfile.patch create mode 100644 tests/test_repos/commit_stats/0006-Change-three-options.patch diff --git a/tests/cfgnet/analyze/test_commit_stats.py b/tests/cfgnet/analyze/test_commit_stats.py new file mode 100644 index 0000000..3bb3211 --- /dev/null +++ b/tests/cfgnet/analyze/test_commit_stats.py @@ -0,0 +1,122 @@ +# This file is part of the CfgNet module. +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program. If not, see . + +import os +import csv +import pytest + +from cfgnet.network.network_configuration import NetworkConfiguration +from cfgnet.analyze.analyzer import Analyzer +from tests.utility.temporary_repository import TemporaryRepository + + +TOTAL_OPTIONS = ["8", "17", "17", "16", "17", "17"] +NUM_VALUE_NODES_CHANGED = ["0", "0", "1", "0", "0", "3"] +NUM_VALUE_NODES_ADDED = ["8", "9", "0", "0", "1", "0"] +NUM_VALUE_NODES_REMOVED = ["0", "0", "0", "1", "0", "0"] +NUM_CONFIG_FILES_CHANGED = ["1", "1", "1", "1", "1", "2"] + + +@pytest.fixture(name="get_repo") +def get_repo_(): + repo = TemporaryRepository("tests/test_repos/commit_stats") + return repo + + +@pytest.fixture(name="get_csv_path") +def get_config_(get_repo): + network_configuration = NetworkConfiguration( + project_root_abs=os.path.abspath(get_repo.root), + enable_static_blacklist=False, + enable_dynamic_blacklist=False, + enable_internal_links=False, + ) + + analyzer = Analyzer(network_configuration) + analyzer.analyze_commit_history() + + project_name = network_configuration.project_name() + data_dir = os.path.join( + network_configuration.project_root_abs, + network_configuration.cfgnet_path_rel, + ) + analysis_dir = os.path.join(data_dir, "analysis") + stats_csv_path = os.path.join( + analysis_dir, f"commit_stats_{project_name}.csv" + ) + + return stats_csv_path + + +def test_number_of_commits(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + assert len(rows) == 6 + + +def test_total_option_number(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert rows[i]["total_option_nodes"] == TOTAL_OPTIONS[i] + + +def test_num_config_file_changed(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert ( + rows[i]["num_configuration_files_changed"] + == NUM_CONFIG_FILES_CHANGED[i] + ) + + +def test_num_value_nodes_added(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert rows[i]["num_value_nodes_added"] == NUM_VALUE_NODES_ADDED[i] + + +def test_num_value_nodes_removed(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert ( + rows[i]["num_value_nodes_removed"] + == NUM_VALUE_NODES_REMOVED[i] + ) + + +def test_num_value_nodes_changed(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert ( + rows[i]["num_value_nodes_changed"] + == NUM_VALUE_NODES_CHANGED[i] + ) diff --git a/tests/test_repos/commit_stats/0001-Add-package.json.patch b/tests/test_repos/commit_stats/0001-Add-package.json.patch new file mode 100644 index 0000000..dccfb3c --- /dev/null +++ b/tests/test_repos/commit_stats/0001-Add-package.json.patch @@ -0,0 +1,32 @@ +From de20da557315fcb57bd33711080943f8ea663823 Mon Sep 17 00:00:00 2001 +From: simisimon +Date: Tue, 15 Mar 2022 16:57:55 +0100 +Subject: [PATCH 1/6] Add package.json + +--- + package.json | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + create mode 100644 package.json + +diff --git a/package.json b/package.json +new file mode 100644 +index 0000000..82ca76b +--- /dev/null ++++ b/package.json +@@ -0,0 +1,12 @@ ++{ ++ "name": "node-js-sample", ++ "version": "0.2.0", ++ "description": "Example Project", ++ "main": "index.js", ++ "scripts": { ++ "start": "node index.js" ++ }, ++ "dependencies": { ++ "express": "^4.13.3" ++ } ++} +\ No newline at end of file +-- +2.25.1 + diff --git a/tests/test_repos/commit_stats/0002-Add-Dockerfile.patch b/tests/test_repos/commit_stats/0002-Add-Dockerfile.patch new file mode 100644 index 0000000..993f187 --- /dev/null +++ b/tests/test_repos/commit_stats/0002-Add-Dockerfile.patch @@ -0,0 +1,24 @@ +From 61a1f60756dc1f94484d61f2433284ec662b6f81 Mon Sep 17 00:00:00 2001 +From: simisimon +Date: Tue, 15 Mar 2022 16:58:35 +0100 +Subject: [PATCH 2/6] Add Dockerfile + +--- + Dockerfile | 5 +++++ + 1 file changed, 5 insertions(+) + create mode 100644 Dockerfile + +diff --git a/Dockerfile b/Dockerfile +new file mode 100644 +index 0000000..4aa00ce +--- /dev/null ++++ b/Dockerfile +@@ -0,0 +1,5 @@ ++FROM java:8 as builder ++ ++EXPOSE 1234 ++ ++ADD --chown=1 /foo.jar bar.jar +-- +2.25.1 + diff --git a/tests/test_repos/commit_stats/0003-Change-one-option-in-Dockerfile.patch b/tests/test_repos/commit_stats/0003-Change-one-option-in-Dockerfile.patch new file mode 100644 index 0000000..269900f --- /dev/null +++ b/tests/test_repos/commit_stats/0003-Change-one-option-in-Dockerfile.patch @@ -0,0 +1,23 @@ +From e93b9f8fd22a8a1b312f11a594b75f340933fcb8 Mon Sep 17 00:00:00 2001 +From: simisimon +Date: Tue, 15 Mar 2022 16:59:02 +0100 +Subject: [PATCH 3/6] Change one option in Dockerfile + +--- + Dockerfile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Dockerfile b/Dockerfile +index 4aa00ce..ddf7643 100644 +--- a/Dockerfile ++++ b/Dockerfile +@@ -1,5 +1,5 @@ + FROM java:8 as builder + +-EXPOSE 1234 ++EXPOSE 8000 + + ADD --chown=1 /foo.jar bar.jar +-- +2.25.1 + diff --git a/tests/test_repos/commit_stats/0004-Delete-one-option-in-Dockerfile.patch b/tests/test_repos/commit_stats/0004-Delete-one-option-in-Dockerfile.patch new file mode 100644 index 0000000..40b0e2d --- /dev/null +++ b/tests/test_repos/commit_stats/0004-Delete-one-option-in-Dockerfile.patch @@ -0,0 +1,22 @@ +From fc6d5186a621c2032341e3e20ea36f1d0834d3b8 Mon Sep 17 00:00:00 2001 +From: simisimon +Date: Tue, 15 Mar 2022 16:59:16 +0100 +Subject: [PATCH 4/6] Delete one option in Dockerfile + +--- + Dockerfile | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/Dockerfile b/Dockerfile +index ddf7643..c9cf346 100644 +--- a/Dockerfile ++++ b/Dockerfile +@@ -1,5 +1,3 @@ + FROM java:8 as builder + +-EXPOSE 8000 +- + ADD --chown=1 /foo.jar bar.jar +-- +2.25.1 + diff --git a/tests/test_repos/commit_stats/0005-Add-one-option-in-Dockerfile.patch b/tests/test_repos/commit_stats/0005-Add-one-option-in-Dockerfile.patch new file mode 100644 index 0000000..353d306 --- /dev/null +++ b/tests/test_repos/commit_stats/0005-Add-one-option-in-Dockerfile.patch @@ -0,0 +1,22 @@ +From 5b99ba743cf4db9a7fd6d66ec0ab77dd1198831a Mon Sep 17 00:00:00 2001 +From: simisimon +Date: Tue, 15 Mar 2022 16:59:28 +0100 +Subject: [PATCH 5/6] Add one option in Dockerfile + +--- + Dockerfile | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/Dockerfile b/Dockerfile +index c9cf346..ddf7643 100644 +--- a/Dockerfile ++++ b/Dockerfile +@@ -1,3 +1,5 @@ + FROM java:8 as builder + ++EXPOSE 8000 ++ + ADD --chown=1 /foo.jar bar.jar +-- +2.25.1 + diff --git a/tests/test_repos/commit_stats/0006-Change-three-options.patch b/tests/test_repos/commit_stats/0006-Change-three-options.patch new file mode 100644 index 0000000..12c895d --- /dev/null +++ b/tests/test_repos/commit_stats/0006-Change-three-options.patch @@ -0,0 +1,44 @@ +From 9c119704b16a5d1865d64736ec57cef5e9e5bbf3 Mon Sep 17 00:00:00 2001 +From: simisimon +Date: Tue, 15 Mar 2022 16:59:58 +0100 +Subject: [PATCH 6/6] Change three options + +--- + Dockerfile | 2 +- + package.json | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/Dockerfile b/Dockerfile +index ddf7643..edab55f 100644 +--- a/Dockerfile ++++ b/Dockerfile +@@ -1,5 +1,5 @@ + FROM java:8 as builder + +-EXPOSE 8000 ++EXPOSE 7777 + + ADD --chown=1 /foo.jar bar.jar +diff --git a/package.json b/package.json +index 82ca76b..1fd5ffa 100644 +--- a/package.json ++++ b/package.json +@@ -1,12 +1,12 @@ + { + "name": "node-js-sample", +- "version": "0.2.0", ++ "version": "0.5.0", + "description": "Example Project", + "main": "index.js", + "scripts": { + "start": "node index.js" + }, + "dependencies": { +- "express": "^4.13.3" ++ "express": "5.1.3" + } + } +\ No newline at end of file +-- +2.25.1 + From 9460281dcaca9d3b32e516e71ac8cacdc49dd3df Mon Sep 17 00:00:00 2001 From: simisimon Date: Fri, 18 Mar 2022 16:13:47 +0100 Subject: [PATCH 3/7] Update commit stats tests --- tests/cfgnet/analyze/test_analyzer.py | 2 +- tests/cfgnet/analyze/test_commit_stats.py | 65 +++++++++++++++++-- .../commit_stats/0007-Add-pom.xml.patch | 24 +++++++ ...oy-one-link-between-Docker-and-Maven.patch | 23 +++++++ 4 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 tests/test_repos/commit_stats/0007-Add-pom.xml.patch create mode 100644 tests/test_repos/commit_stats/0008-Destroy-one-link-between-Docker-and-Maven.patch diff --git a/tests/cfgnet/analyze/test_analyzer.py b/tests/cfgnet/analyze/test_analyzer.py index e835787..5c45182 100644 --- a/tests/cfgnet/analyze/test_analyzer.py +++ b/tests/cfgnet/analyze/test_analyzer.py @@ -71,4 +71,4 @@ def test_analyze(get_config): reader = csv.DictReader(csv_stats_file) rows = list(reader) - assert len(rows) == 1 + assert len(rows) == 2 diff --git a/tests/cfgnet/analyze/test_commit_stats.py b/tests/cfgnet/analyze/test_commit_stats.py index 3bb3211..89804f1 100644 --- a/tests/cfgnet/analyze/test_commit_stats.py +++ b/tests/cfgnet/analyze/test_commit_stats.py @@ -22,11 +22,16 @@ from tests.utility.temporary_repository import TemporaryRepository -TOTAL_OPTIONS = ["8", "17", "17", "16", "17", "17"] -NUM_VALUE_NODES_CHANGED = ["0", "0", "1", "0", "0", "3"] -NUM_VALUE_NODES_ADDED = ["8", "9", "0", "0", "1", "0"] -NUM_VALUE_NODES_REMOVED = ["0", "0", "0", "1", "0", "0"] -NUM_CONFIG_FILES_CHANGED = ["1", "1", "1", "1", "1", "2"] +TOTAL_OPTIONS = ["8", "17", "17", "16", "17", "17", "21", "21"] +TOTAL_VALUES = ["6", "13", "13", "12", "13", "13", "16", "16"] +NUM_VALUE_NODES_CHANGED = ["0", "0", "1", "0", "0", "3", "0", "1"] +NUM_VALUE_NODES_ADDED = ["6", "7", "0", "0", "1", "0", "3", "0"] +NUM_VALUE_NODES_REMOVED = ["0", "0", "0", "1", "0", "0", "0", "0"] +NUM_CONFIG_FILES_CHANGED = ["1", "1", "1", "1", "1", "2", "1", "1"] +TOTAL_LINKS = ["0", "0", "0", "0", "0", "0", "1", "0"] +LINKS_ADDED = ["0", "0", "0", "0", "0", "0", "1", "0"] +LINKS_REMOVED = ["0", "0", "0", "0", "0", "0", "0", "1"] +CONFLICTS_DETECTED = ["0", "0", "0", "0", "0", "0", "0", "1"] @pytest.fixture(name="get_repo") @@ -65,7 +70,7 @@ def test_number_of_commits(get_csv_path): reader = csv.DictReader(csv_stats_file) rows = list(reader) - assert len(rows) == 6 + assert len(rows) == 8 def test_total_option_number(get_csv_path): @@ -89,12 +94,23 @@ def test_num_config_file_changed(get_csv_path): ) +def test_total_value_nodes(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + print("added: ", i) + assert rows[i]["total_value_nodes"] == TOTAL_VALUES[i] + + def test_num_value_nodes_added(get_csv_path): with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: reader = csv.DictReader(csv_stats_file) rows = list(reader) for i in range(len(rows)): + print("added: ", i) assert rows[i]["num_value_nodes_added"] == NUM_VALUE_NODES_ADDED[i] @@ -104,6 +120,7 @@ def test_num_value_nodes_removed(get_csv_path): rows = list(reader) for i in range(len(rows)): + print("removed: ", i) assert ( rows[i]["num_value_nodes_removed"] == NUM_VALUE_NODES_REMOVED[i] @@ -120,3 +137,39 @@ def test_num_value_nodes_changed(get_csv_path): rows[i]["num_value_nodes_changed"] == NUM_VALUE_NODES_CHANGED[i] ) + + +def test_total_links(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert rows[i]["total_links"] == TOTAL_LINKS[i] + + +def test_links_added(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert rows[i]["links_added"] == LINKS_ADDED[i] + + +def test_links_removed(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert rows[i]["links_removed"] == LINKS_REMOVED[i] + + +def test_conflicts_detected(get_csv_path): + with open(get_csv_path, "r", encoding="utf-8") as csv_stats_file: + reader = csv.DictReader(csv_stats_file) + rows = list(reader) + + for i in range(len(rows)): + assert rows[i]["conflicts_detected"] == CONFLICTS_DETECTED[i] diff --git a/tests/test_repos/commit_stats/0007-Add-pom.xml.patch b/tests/test_repos/commit_stats/0007-Add-pom.xml.patch new file mode 100644 index 0000000..481d318 --- /dev/null +++ b/tests/test_repos/commit_stats/0007-Add-pom.xml.patch @@ -0,0 +1,24 @@ +From edcfba50b5b62689a361127a42b273951d7eb239 Mon Sep 17 00:00:00 2001 +From: simisimon +Date: Fri, 18 Mar 2022 13:59:24 +0100 +Subject: [PATCH 7/8] Add pom.xml + +--- + pom.xml | 4 ++++ + 1 file changed, 4 insertions(+) + create mode 100644 pom.xml + +diff --git a/pom.xml b/pom.xml +new file mode 100644 +index 0000000..f9e682f +--- /dev/null ++++ b/pom.xml +@@ -0,0 +1,4 @@ ++ ++ 0.2.0 ++ 7777 ++ +\ No newline at end of file +-- +2.25.1 + diff --git a/tests/test_repos/commit_stats/0008-Destroy-one-link-between-Docker-and-Maven.patch b/tests/test_repos/commit_stats/0008-Destroy-one-link-between-Docker-and-Maven.patch new file mode 100644 index 0000000..d3e8f6a --- /dev/null +++ b/tests/test_repos/commit_stats/0008-Destroy-one-link-between-Docker-and-Maven.patch @@ -0,0 +1,23 @@ +From 9771f0dfc5f805cab98692410e229cc8f1000795 Mon Sep 17 00:00:00 2001 +From: simisimon +Date: Fri, 18 Mar 2022 14:00:10 +0100 +Subject: [PATCH 8/8] Destroy one link between Docker and Maven + +--- + pom.xml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/pom.xml b/pom.xml +index f9e682f..9a4f924 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -1,4 +1,4 @@ + + 0.2.0 +- 7777 ++ 9000 + +\ No newline at end of file +-- +2.25.1 + From a22369effa3d6af8da2b72feed7336baa6748fdf Mon Sep 17 00:00:00 2001 From: simisimon Date: Fri, 18 Mar 2022 16:14:31 +0100 Subject: [PATCH 4/7] Fix commit stats and add new statistics --- src/cfgnet/analyze/analyzer.py | 15 ++++-- src/cfgnet/utility/statistics.py | 87 +++++++++++++++++++------------- 2 files changed, 63 insertions(+), 39 deletions(-) diff --git a/src/cfgnet/analyze/analyzer.py b/src/cfgnet/analyze/analyzer.py index f053edb..12e4e42 100644 --- a/src/cfgnet/analyze/analyzer.py +++ b/src/cfgnet/analyze/analyzer.py @@ -86,7 +86,16 @@ def analyze_commit_history(self) -> None: self.commit_stats_file, "w+", encoding="utf-8" ) as stats_csv: CommitStatistics.setup_writer(stats_csv) - stats_prev = CommitStatistics() + initial_stats = CommitStatistics() + stats = CommitStatistics.calc_stats( + commit=commit, + commit_number=history.commit_index, + network=ref_network, + conflicts=conflicts, + prev=initial_stats, + ) + CommitStatistics.write_row(stats) + while history.has_next_commit(): commit = history.next_commit() commit_number = history.commit_index @@ -101,8 +110,8 @@ def analyze_commit_history(self) -> None: commit=commit, commit_number=commit_number, network=ref_network, - conflicts=conflicts, - prev=stats_prev, + conflicts=detected_conflicts, + prev=stats, ) CommitStatistics.write_row(stats) diff --git a/src/cfgnet/utility/statistics.py b/src/cfgnet/utility/statistics.py index 1260bb1..f55ed04 100644 --- a/src/cfgnet/utility/statistics.py +++ b/src/cfgnet/utility/statistics.py @@ -29,16 +29,19 @@ class CommitStatistics: def __init__(self, commit=None): self.commit_hash = None self.commit_number = 0 - self.total_artifact_nodes = 0 + self.total_num_artifact_nodes = 0 + self.num_configuration_files_changed = 0 + self.total_config_files = set() + self.config_files_changed = set() self.total_option_nodes = 0 + self.total_value_nodes = 0 self.num_value_nodes_added = 0 self.num_value_nodes_removed = 0 self.num_value_nodes_changed = 0 self.total_links = 0 - self.total_links_changed = 0 - self.num_configuration_files_changed = 0 - self.num_detected_conflicts = 0 - self.num_fixed_conflicts = 0 + self.links_added = 0 + self.links_removed = 0 + self.conflicts_detected = 0 self._value_node_ids = set() self._value_node_parent_ids = set() @@ -59,22 +62,37 @@ def calc_stats( ) -> "CommitStatistics": stats = CommitStatistics(commit=commit) - stats._value_node_ids = { - node.id for node in network.get_nodes(ValueNode) - } - stats._value_node_parent_ids = { - node.parent.id for node in network.get_nodes(ValueNode) - } - + # commit data stats.commit_number = commit_number + stats.commit_hash = commit.hexsha + + # artifact data artifact_nodes = network.get_nodes(ArtifactNode) - stats.total_artifact_nodes = len(artifact_nodes) - stats.total_option_nodes = len(network.get_nodes(OptionNode)) + stats.total_num_artifact_nodes = len(artifact_nodes) + stats.total_config_files = { + node.rel_file_path for node in artifact_nodes + } + files_changed = set(commit.stats.files.keys()) + stats.config_files_changed = stats.total_config_files.intersection( + files_changed + ) + stats.num_configuration_files_changed = len(stats.config_files_changed) + + # option data + option_nodes = network.get_nodes(OptionNode) + stats.total_option_nodes = len(option_nodes) + + # value data + value_nodes = network.get_nodes(ValueNode) + stats.total_value_nodes = len(value_nodes) + stats._value_node_ids = {node.id for node in value_nodes} + stats._value_node_parent_ids = {node.parent.id for node in value_nodes} + stats.num_value_nodes_added = len( - prev._value_node_ids.difference(stats._value_node_parent_ids) + stats._value_node_ids.difference(prev._value_node_ids) ) stats.num_value_nodes_removed = len( - stats._value_node_ids.difference(prev._value_node_parent_ids) + prev._value_node_ids.difference(stats._value_node_ids) ) stats.num_value_nodes_changed = 0 @@ -90,23 +108,17 @@ def same_parent(node_id, parent=node): value_new = list(filter(same_parent, stats._value_node_ids))[0] if value_prev != value_new: stats.num_value_nodes_changed += 1 + stats.num_value_nodes_added -= 1 + stats.num_value_nodes_removed -= 1 - stats._links = set(network.links) + # link data + stats._links = network.links stats.total_links = len(stats._links) - stats.total_links_changed = len( - prev._links.symmetric_difference(stats._links) - ) - - config_files = {node.rel_file_path for node in artifact_nodes} - files_changed = set(commit.stats.files.keys()) - config_files_changed = config_files.intersection(files_changed) - stats.num_configuration_files_changed = len(config_files_changed) + stats.links_added = len(stats._links.difference(prev._links)) + stats.links_removed = len(prev._links.difference(stats._links)) - stats._conflicts = set(conflicts) - new_conflicts = stats._conflicts.difference(prev._conflicts) - fixed_conflicts = prev._conflicts.difference(stats._conflicts) - stats.num_detected_conflicts = Conflict.count_total(new_conflicts) - stats.num_fixed_conflicts = Conflict.count_total(fixed_conflicts) + # conflict data + stats.conflicts_detected = len(list(conflicts)) return stats @@ -131,16 +143,19 @@ def data_dict(self): { "commit_number": self.commit_number, "commit_hash": self.commit_hash, + "total_num_artifact_nodes": self.total_num_artifact_nodes, + "num_configuration_files_changed": self.num_configuration_files_changed, + "total_config_files": sorted(self.total_config_files), + "config_files_changed": sorted(self.config_files_changed), "total_option_nodes": self.total_option_nodes, + "total_value_nodes": self.total_value_nodes, "num_value_nodes_changed": self.num_value_nodes_changed, - "total_artifact_nodes": self.total_artifact_nodes, - "total_links": self.total_links, - "total_links_changed": self.total_links_changed, - "num_configuration_files_changed": self.num_configuration_files_changed, - "num_detected_conflicts": self.num_detected_conflicts, - "num_fixed_conflicts": self.num_fixed_conflicts, "num_value_nodes_added": self.num_value_nodes_added, "num_value_nodes_removed": self.num_value_nodes_removed, + "total_links": self.total_links, + "links_added": self.links_added, + "links_removed": self.links_removed, + "conflicts_detected": self.conflicts_detected, } ) return data From bae6030b8a5d81b941ac57222948a472582344b9 Mon Sep 17 00:00:00 2001 From: simisimon Date: Sat, 19 Mar 2022 11:14:47 +0100 Subject: [PATCH 5/7] Revise commit statistics --- src/cfgnet/analyze/analyzer.py | 63 ++++++++++++++++---------------- src/cfgnet/utility/statistics.py | 24 +++++++----- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/src/cfgnet/analyze/analyzer.py b/src/cfgnet/analyze/analyzer.py index 12e4e42..dfdd0ce 100644 --- a/src/cfgnet/analyze/analyzer.py +++ b/src/cfgnet/analyze/analyzer.py @@ -17,7 +17,7 @@ import logging import time -from typing import Optional, Set +from typing import Optional, Set, List from cfgnet.vcs.git import Git from cfgnet.vcs.git_history import GitHistory from cfgnet.network.network import Network, NetworkConfiguration @@ -77,48 +77,45 @@ def analyze_commit_history(self) -> None: commit_hash_pre_analysis = repo.get_current_commit_hash() conflicts: Set = set() + commit_stats: List[CommitStatistics] = [] history = GitHistory(repo) commit = history.restore_initial_commit() try: ref_network = Network.init_network(cfg=self.cfg) - with open( - self.commit_stats_file, "w+", encoding="utf-8" - ) as stats_csv: - CommitStatistics.setup_writer(stats_csv) - initial_stats = CommitStatistics() - stats = CommitStatistics.calc_stats( - commit=commit, - commit_number=history.commit_index, - network=ref_network, - conflicts=conflicts, - prev=initial_stats, - ) - CommitStatistics.write_row(stats) + initial_stats = CommitStatistics() + stats = CommitStatistics.calc_stats( + commit=commit, + commit_number=history.commit_index, + network=ref_network, + conflicts=conflicts, + prev=initial_stats, + ) + commit_stats.append(stats) - while history.has_next_commit(): - commit = history.next_commit() - commit_number = history.commit_index + while history.has_next_commit(): + commit = history.next_commit() + commit_number = history.commit_index - detected_conflicts, ref_network = ref_network.validate( - commit.hexsha - ) + detected_conflicts, ref_network = ref_network.validate( + commit.hexsha + ) - conflicts.update(detected_conflicts) + conflicts.update(detected_conflicts) - stats = CommitStatistics.calc_stats( - commit=commit, - commit_number=commit_number, - network=ref_network, - conflicts=detected_conflicts, - prev=stats, - ) - CommitStatistics.write_row(stats) + stats = CommitStatistics.calc_stats( + commit=commit, + commit_number=commit_number, + network=ref_network, + conflicts=detected_conflicts, + prev=stats, + ) + commit_stats.append(stats) - self._print_progress(num_commit=commit_number + 1) + self._print_progress(num_commit=commit_number + 1) - if commit.hexsha == commit_hash_pre_analysis: - break + if commit.hexsha == commit_hash_pre_analysis: + break except Exception as error: logging.error( @@ -141,6 +138,8 @@ def analyze_commit_history(self) -> None: csv_path=self.conflicts_csv_path, conflicts=conflicts ) + CommitStatistics.write_stats_to_csv(self.commit_stats_file, commit_stats) + self._print_progress( num_commit=history.commit_index + 1, final=True ) diff --git a/src/cfgnet/utility/statistics.py b/src/cfgnet/utility/statistics.py index f55ed04..250d12d 100644 --- a/src/cfgnet/utility/statistics.py +++ b/src/cfgnet/utility/statistics.py @@ -15,7 +15,7 @@ import csv from collections import OrderedDict -from typing import Optional, Iterable +from typing import List, Iterable from cfgnet.network.network import Network from cfgnet.network.nodes import OptionNode, ArtifactNode, ValueNode @@ -24,8 +24,6 @@ class CommitStatistics: - writer: Optional[csv.DictWriter] = None - def __init__(self, commit=None): self.commit_hash = None self.commit_number = 0 @@ -124,15 +122,21 @@ def same_parent(node_id, parent=node): @staticmethod def setup_writer(commit_stats_file) -> None: - CommitStatistics.writer = csv.DictWriter( - commit_stats_file, fieldnames=CommitStatistics.fieldnames() - ) - CommitStatistics.writer.writeheader() + with open(commit_stats_file, "w+", encoding="utf-8") as stats_csv: + CommitStatistics.writer = csv.DictWriter( + stats_csv, fieldnames=CommitStatistics.fieldnames() + ) + CommitStatistics.writer.writeheader() @staticmethod - def write_row(stats): - if CommitStatistics.writer is not None: - CommitStatistics.writer.writerow(stats.data_dict()) + def write_stats_to_csv(file_path: str, commit_data: List["CommitStatistics"]) -> None: + with open(file_path, "w+", encoding="utf-8") as stats_csv: + writer = csv.DictWriter( + stats_csv, fieldnames=CommitStatistics.fieldnames() + ) + writer.writeheader() + for stats in commit_data: + writer.writerow(stats.data_dict()) @staticmethod def fieldnames(): From 79386d3274dca858b894e47617290a574562d3ab Mon Sep 17 00:00:00 2001 From: simisimon Date: Sat, 19 Mar 2022 16:02:57 +0100 Subject: [PATCH 6/7] Parse nodes for concept and write them to csv file --- src/cfgnet/analyze/analyzer.py | 11 +++- src/cfgnet/utility/statistics.py | 92 ++++++++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/src/cfgnet/analyze/analyzer.py b/src/cfgnet/analyze/analyzer.py index dfdd0ce..fc48254 100644 --- a/src/cfgnet/analyze/analyzer.py +++ b/src/cfgnet/analyze/analyzer.py @@ -51,6 +51,10 @@ def _setup_dirs(self) -> None: analysis_dir, f"commit_stats_{self.cfg.project_name()}.csv" ) + self.option_stats_file = os.path.join( + analysis_dir, f"option_stats_{self.cfg.project_name()}.csv" + ) + for path in [self.conflicts_csv_path, self.commit_stats_file]: if os.path.exists(path): os.remove(path) @@ -138,7 +142,12 @@ def analyze_commit_history(self) -> None: csv_path=self.conflicts_csv_path, conflicts=conflicts ) - CommitStatistics.write_stats_to_csv(self.commit_stats_file, commit_stats) + CommitStatistics.write_stats_to_csv( + self.commit_stats_file, commit_stats + ) + CommitStatistics.write_options_to_csv( + self.option_stats_file, commit_stats + ) self._print_progress( num_commit=history.commit_index + 1, final=True diff --git a/src/cfgnet/utility/statistics.py b/src/cfgnet/utility/statistics.py index 250d12d..adbd5cd 100644 --- a/src/cfgnet/utility/statistics.py +++ b/src/cfgnet/utility/statistics.py @@ -14,6 +14,7 @@ # this program. If not, see . import csv +import re from collections import OrderedDict from typing import List, Iterable @@ -41,6 +42,12 @@ def __init__(self, commit=None): self.links_removed = 0 self.conflicts_detected = 0 + self.docker_nodes = {} + self.nodejs_nodes = {} + self.maven_nodes = {} + self.travis_nodes = {} + self.docker_compose_nodes = {} + self._value_node_ids = set() self._value_node_parent_ids = set() self._links = set() @@ -118,30 +125,56 @@ def same_parent(node_id, parent=node): # conflict data stats.conflicts_detected = len(list(conflicts)) + # nodes data + stats.docker_nodes = CommitStatistics.parse_options( + network, r"Dockerfile" + ) + stats.nodejs_nodes = CommitStatistics.parse_options( + network, r"package.json" + ) + stats.maven_nodes = CommitStatistics.parse_options(network, r"pom.xml") + stats.docker_compose_nodes = CommitStatistics.parse_options( + network, r"docker-compose(.\w+)?.yml" + ) + stats.travis_nodes = CommitStatistics.parse_options( + network, r".travis.yml" + ) + return stats @staticmethod - def setup_writer(commit_stats_file) -> None: - with open(commit_stats_file, "w+", encoding="utf-8") as stats_csv: - CommitStatistics.writer = csv.DictWriter( - stats_csv, fieldnames=CommitStatistics.fieldnames() + def write_stats_to_csv( + file_path: str, commit_data: List["CommitStatistics"] + ) -> None: + with open(file_path, "w+", encoding="utf-8") as stats_csv: + writer = csv.DictWriter( + stats_csv, + fieldnames=CommitStatistics.commit_stats_fieldnames(), ) - CommitStatistics.writer.writeheader() + writer.writeheader() + for stats in commit_data: + writer.writerow(stats.data_dict()) @staticmethod - def write_stats_to_csv(file_path: str, commit_data: List["CommitStatistics"]) -> None: - with open(file_path, "w+", encoding="utf-8") as stats_csv: + def write_options_to_csv( + nodes_file_path: str, commit_data: List["CommitStatistics"] + ) -> None: + with open(nodes_file_path, "w+", encoding="utf-8") as stats_csv: writer = csv.DictWriter( - stats_csv, fieldnames=CommitStatistics.fieldnames() + stats_csv, fieldnames=CommitStatistics.nodes_fieldnames() ) writer.writeheader() for stats in commit_data: - writer.writerow(stats.data_dict()) + writer.writerow(stats.option_dict()) @staticmethod - def fieldnames(): + def commit_stats_fieldnames(): return list(CommitStatistics().data_dict().keys()) + @staticmethod + def nodes_fieldnames(): + return list(CommitStatistics().option_dict().keys()) + def data_dict(self): data = OrderedDict( { @@ -163,3 +196,42 @@ def data_dict(self): } ) return data + + def option_dict(self): + data = OrderedDict( + { + "docker_nodes": self.docker_nodes, + "docker_compose_nodes": self.docker_compose_nodes, + "maven_nodes": self.maven_nodes, + "nodejs_nodes": self.nodejs_nodes, + "travis_nodes": self.travis_nodes, + } + ) + return data + + @staticmethod + def parse_options(network: Network, file_name: str) -> dict: + artifacts = list( + filter( + lambda x: isinstance(x, ArtifactNode) + and re.compile(file_name).search(x.id), + network.get_nodes(node_type=ArtifactNode), + ) + ) + + data = {} + + for artifact in artifacts: + artifact_data = {} + options = filter( + lambda x: x.prevalue_node, + artifact.get_nodes(node_type=OptionNode), + ) + for option in options: + parts = option.id.split("::::") + option_name = "::::".join(parts[2:]) + artifact_data[f"{option_name}"] = option.children[0].name + + data[artifact.rel_file_path] = artifact_data + + return data From a30cb823a65f34fcb52b1abc5a994a2e1515a5a6 Mon Sep 17 00:00:00 2001 From: simisimon Date: Mon, 21 Mar 2022 15:28:23 +0100 Subject: [PATCH 7/7] Remove total config file column --- src/cfgnet/utility/statistics.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/cfgnet/utility/statistics.py b/src/cfgnet/utility/statistics.py index adbd5cd..da22788 100644 --- a/src/cfgnet/utility/statistics.py +++ b/src/cfgnet/utility/statistics.py @@ -30,7 +30,6 @@ def __init__(self, commit=None): self.commit_number = 0 self.total_num_artifact_nodes = 0 self.num_configuration_files_changed = 0 - self.total_config_files = set() self.config_files_changed = set() self.total_option_nodes = 0 self.total_value_nodes = 0 @@ -74,11 +73,9 @@ def calc_stats( # artifact data artifact_nodes = network.get_nodes(ArtifactNode) stats.total_num_artifact_nodes = len(artifact_nodes) - stats.total_config_files = { - node.rel_file_path for node in artifact_nodes - } + total_config_files = {node.rel_file_path for node in artifact_nodes} files_changed = set(commit.stats.files.keys()) - stats.config_files_changed = stats.total_config_files.intersection( + stats.config_files_changed = total_config_files.intersection( files_changed ) stats.num_configuration_files_changed = len(stats.config_files_changed) @@ -182,7 +179,6 @@ def data_dict(self): "commit_hash": self.commit_hash, "total_num_artifact_nodes": self.total_num_artifact_nodes, "num_configuration_files_changed": self.num_configuration_files_changed, - "total_config_files": sorted(self.total_config_files), "config_files_changed": sorted(self.config_files_changed), "total_option_nodes": self.total_option_nodes, "total_value_nodes": self.total_value_nodes,