From d52e7a24c3229783a4dd7fb72023aa295b611cbd Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 21 Aug 2024 15:50:35 -0700 Subject: [PATCH] Fix the report workflow to work with ci-group (#4960) Signed-off-by: Zelin Hao --- src/manifests/test_manifest.py | 1 + src/report_workflow/test_report_runner.py | 40 ++++---- .../test-manifest-opensearch-dashboards.yml | 1 + .../test_test_report_runner_dashboards.py | 99 +++++++++++-------- 4 files changed, 85 insertions(+), 56 deletions(-) diff --git a/src/manifests/test_manifest.py b/src/manifests/test_manifest.py index 428743454f..ff484780e9 100644 --- a/src/manifests/test_manifest.py +++ b/src/manifests/test_manifest.py @@ -31,6 +31,7 @@ class TestManifest(ComponentManifest['TestManifest', 'TestComponents']): - with-less-security additional-cluster-configs: - key : value + ci-group: 6 bwc-test: test-configs: - with-security diff --git a/src/report_workflow/test_report_runner.py b/src/report_workflow/test_report_runner.py index 5a395d0e9c..dd001bdc98 100644 --- a/src/report_workflow/test_report_runner.py +++ b/src/report_workflow/test_report_runner.py @@ -26,7 +26,7 @@ class TestReportRunner: test_manifest: TestManifest tests_dir: str test_report_manifest: TestReportManifest - test_run_data: dict + test_report_data: dict bundle_manifest: BundleManifest def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None: @@ -34,7 +34,7 @@ def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None: self.base_path = args.base_path self.test_manifest = test_manifest self.release_candidate = self.args.release_candidate - self.test_run_data = self.test_report_manifest_data_template("manifest") + self.test_report_data = self.test_report_manifest_data_template("manifest") self.product_name = test_manifest.__to_dict__().get("name") self.name = self.product_name.replace(" ", "-").lower() self.components = self.args.components @@ -50,19 +50,24 @@ def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None: self.test_components = self.test_manifest.components def update_data(self) -> dict: - self.test_run_data["name"] = self.product_name + self.test_report_data["name"] = self.product_name self.bundle_manifest = BundleManifest.from_urlpath(self.dist_manifest) - self.test_run_data["version"] = self.bundle_manifest.build.version - self.test_run_data["platform"] = self.bundle_manifest.build.platform - self.test_run_data["architecture"] = self.bundle_manifest.build.architecture - self.test_run_data["distribution"] = self.bundle_manifest.build.distribution - self.test_run_data["id"] = self.bundle_manifest.build.id - self.test_run_data["rc"] = self.release_candidate - self.test_run_data["test-run"] = self.update_test_run_data() + self.test_report_data["version"] = self.bundle_manifest.build.version + self.test_report_data["platform"] = self.bundle_manifest.build.platform + self.test_report_data["architecture"] = self.bundle_manifest.build.architecture + self.test_report_data["distribution"] = self.bundle_manifest.build.distribution + self.test_report_data["id"] = self.bundle_manifest.build.id + self.test_report_data["rc"] = self.release_candidate + self.test_report_data["test-run"] = self.update_test_run_data() for component in self.test_components.select(focus=self.args.components): if self.test_manifest.components[component.name].__to_dict__().get(self.test_type) is not None: - self.test_run_data["components"].append(self.component_entry(component.name)) - return self.test_run_data + component_ci_group = getattr(component, self.test_type.replace("-", "_")).get("ci-groups", None) + if component_ci_group: + for i in range(component_ci_group): + self.test_report_data["components"].append(self.component_entry(component.name, i + 1)) + else: + self.test_report_data["components"].append(self.component_entry(component.name)) + return self.test_report_data def update_test_run_data(self) -> dict: test_run_data = { @@ -80,9 +85,10 @@ def generate_report(self, data: dict, output_dir: str) -> Any: logging.info(f"Generating test-report.yml in {output_dir}") return test_report_manifest.to_file(test_report_manifest_file) - def component_entry(self, component_name: str) -> Any: + def component_entry(self, component_name: str, ci_group: int = None) -> Any: component = self.test_report_manifest_data_template("component") - component["name"] = component_name + test_report_component_name = component_name if not ci_group else f"{component_name}-ci-group-{ci_group}" + component["name"] = test_report_component_name component["command"] = generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths, component_name) test_component = self.test_manifest.components[component_name] @@ -94,7 +100,7 @@ def component_entry(self, component_name: str) -> Any: "name": config, } - component_yml_ref = generate_component_yml_ref(self.base_path, str(self.test_run_id), self.test_type, component_name, config) + component_yml_ref = generate_component_yml_ref(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config) logging.info(f"Loading {component_yml_ref}") try: if validators.url(component_yml_ref): @@ -111,8 +117,8 @@ def component_entry(self, component_name: str) -> Any: component_yml_ref = "URL not available" config_dict["yml"] = component_yml_ref config_dict["status"] = test_result - config_dict["cluster_stdout"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, component_name, config, self.name)[0] - config_dict["cluster_stderr"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, component_name, config, self.name)[1] + config_dict["cluster_stdout"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config, self.name)[0] + config_dict["cluster_stderr"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config, self.name)[1] component["configs"].append(config_dict) return component diff --git a/tests/tests_report_workflow/data/test-manifest-opensearch-dashboards.yml b/tests/tests_report_workflow/data/test-manifest-opensearch-dashboards.yml index 1422c00e30..d6a147ccb6 100644 --- a/tests/tests_report_workflow/data/test-manifest-opensearch-dashboards.yml +++ b/tests/tests_report_workflow/data/test-manifest-opensearch-dashboards.yml @@ -13,6 +13,7 @@ components: additional-cluster-configs: vis_builder.enabled: true data_source.enabled: true + ci-groups: 3 - name: alertingDashboards integ-test: test-configs: diff --git a/tests/tests_report_workflow/test_test_report_runner_dashboards.py b/tests/tests_report_workflow/test_test_report_runner_dashboards.py index 9f62d9482e..8c83bde7cb 100644 --- a/tests/tests_report_workflow/test_test_report_runner_dashboards.py +++ b/tests/tests_report_workflow/test_test_report_runner_dashboards.py @@ -29,18 +29,18 @@ def test_runner_dashboards_init(self, report_args_mock: MagicMock) -> None: report_args_mock.test_type = "integ-test" report_args_mock.release_candidate = "100" - test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS) - test_run_runner_data = test_run_runner.update_data() - self.assertEqual(test_run_runner.name, "opensearch-dashboards") - self.assertEqual(test_run_runner.test_run_id, 123) - self.assertEqual(test_run_runner.test_type, "integ-test") - self.assertEqual(test_run_runner.test_manifest_path, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH) - self.assertEqual(test_run_runner_data["version"], "1.3.18") - self.assertEqual(test_run_runner_data["platform"], "linux") - self.assertEqual(test_run_runner_data["architecture"], "x64") - self.assertEqual(test_run_runner_data["distribution"], "tar") - self.assertEqual(test_run_runner_data["id"], "7791") - self.assertEqual(test_run_runner_data["rc"], "100") + test_report_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS) + test_report_runner_data = test_report_runner.update_data() + self.assertEqual(test_report_runner.name, "opensearch-dashboards") + self.assertEqual(test_report_runner.test_run_id, 123) + self.assertEqual(test_report_runner.test_type, "integ-test") + self.assertEqual(test_report_runner.test_manifest_path, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH) + self.assertEqual(test_report_runner_data["version"], "1.3.18") + self.assertEqual(test_report_runner_data["platform"], "linux") + self.assertEqual(test_report_runner_data["architecture"], "x64") + self.assertEqual(test_report_runner_data["distribution"], "tar") + self.assertEqual(test_report_runner_data["id"], "7791") + self.assertEqual(test_report_runner_data["rc"], "100") @patch("report_workflow.report_args.ReportArgs") def test_generate_file(self, report_args_mock: MagicMock) -> None: @@ -51,14 +51,37 @@ def test_generate_file(self, report_args_mock: MagicMock) -> None: report_args_mock.test_type = "integ-test" report_args_mock.release_candidate = "100" - test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS) - test_run_runner_data = test_run_runner.update_data() + test_report_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS) + test_report_runner_data = test_report_runner.update_data() with TemporaryDirectory() as path: output_path = os.path.join(path.name, "test-report.yml") - test_run_runner.generate_report(test_run_runner_data, path.name) + test_report_runner.generate_report(test_report_runner_data, path.name) self.assertTrue(os.path.isfile(output_path)) + @patch("report_workflow.report_args.ReportArgs") + def test_ci_group(self, report_args_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH + report_args_mock.artifact_paths = {"opensearch-dashboards": self.DATA_DIR} + report_args_mock.test_run_id = 123 + report_args_mock.base_path = self.DATA_DIR + report_args_mock.test_type = "integ-test" + report_args_mock.release_candidate = "100" + + test_report_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS) + test_report_runner_data = test_report_runner.update_data() + + self.assertEqual(len(test_report_runner_data["components"]), 9) + for i in range(3): + self.assertEqual(test_report_runner_data["components"][i]["name"], f"OpenSearch-Dashboards-ci-group-{i + 1}") + + for i in range(len(self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS.components.__to_dict__())): + if self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS.components.__to_dict__()[i]["name"] == "OpenSearch-Dashboards": + continue + else: + self.assertEqual(self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS.components.__to_dict__()[i]["name"], + test_report_runner_data["components"][i + 2]["name"]) + @patch("yaml.safe_load") @patch("urllib.request.urlopen") @patch("validators.url") @@ -75,30 +98,28 @@ def test_runner_component_entry_url(self, report_args_mock: MagicMock, validator yaml_safe_load_mock.return_value = {"test_result": "PASS"} urlopen_mock.return_value = MagicMock() - test_run_component_dict = TestReportRunner(report_args_mock, - self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS).component_entry( - "alertingDashboards") + test_report_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS).component_entry("alertingDashboards") urlopen_mock.assert_has_calls([call( 'https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/alertingDashboards.yml')]) - self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS") - self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") - self.assertEqual(test_run_component_dict.get("configs")[0]["yml"], + self.assertEqual(test_report_component_dict.get("configs")[0]["status"], "PASS") + self.assertEqual(test_report_component_dict.get("configs")[0]["name"], "with-security") + self.assertEqual(test_report_component_dict.get("configs")[0]["yml"], "https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/alertingDashboards.yml") - self.assertEqual(test_run_component_dict.get("configs")[0]["cluster_stdout"][0], "https://ci.opensearch.org/ci" - "/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-0/stdout.txt") - self.assertEqual(test_run_component_dict.get("configs")[0]["cluster_stdout"][1], "https://ci.opensearch.org/ci" - "/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-1/stdout.txt") - self.assertEqual(test_run_component_dict.get("configs")[0]["cluster_stderr"][0], "https://ci.opensearch.org/ci" - "/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-0/stderr.txt") - self.assertEqual(test_run_component_dict.get("configs")[0]["cluster_stderr"][1], "https://ci.opensearch.org/ci" - "/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-1/stderr.txt") - - self.assertEqual(test_run_component_dict.get("configs")[1]["name"], "without-security") - self.assertEqual(test_run_component_dict.get("configs")[1]["cluster_stdout"][0], "https://ci.opensearch.org/ci" - "/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-2/stdout.txt") - self.assertEqual(test_run_component_dict.get("configs")[1]["cluster_stdout"][1], "https://ci.opensearch.org/ci" - "/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-3/stdout.txt") - self.assertEqual(test_run_component_dict.get("configs")[1]["cluster_stderr"][0], "https://ci.opensearch.org/ci" - "/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-2/stderr.txt") - self.assertEqual(test_run_component_dict.get("configs")[1]["cluster_stderr"][1], "https://ci.opensearch.org/ci" - "/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-3/stderr.txt") + self.assertEqual(test_report_component_dict.get("configs")[0]["cluster_stdout"][0], + "https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-0/stdout.txt") + self.assertEqual(test_report_component_dict.get("configs")[0]["cluster_stdout"][1], + "https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-1/stdout.txt") + self.assertEqual(test_report_component_dict.get("configs")[0]["cluster_stderr"][0], + "https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-0/stderr.txt") + self.assertEqual(test_report_component_dict.get("configs")[0]["cluster_stderr"][1], + "https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-1/stderr.txt") + + self.assertEqual(test_report_component_dict.get("configs")[1]["name"], "without-security") + self.assertEqual(test_report_component_dict.get("configs")[1]["cluster_stdout"][0], + "https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-2/stdout.txt") + self.assertEqual(test_report_component_dict.get("configs")[1]["cluster_stdout"][1], + "https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-3/stdout.txt") + self.assertEqual(test_report_component_dict.get("configs")[1]["cluster_stderr"][0], + "https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-2/stderr.txt") + self.assertEqual(test_report_component_dict.get("configs")[1]["cluster_stderr"][1], + "https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-3/stderr.txt")