Python test whl: Use zip files for data_model

.github/workflows/check-data-model-directory-updates.yaml

@@ -36,4 +36,25 @@ jobs:
           python3 scripts/dm_xml_ci_change_enforcement.py data_model/1.3
       - name: Check for changes to 1.4 data_model directory without a SHA update
         run: |
-          python3 scripts/dm_xml_ci_change_enforcement.py data_model/1.4
+          python3 scripts/dm_xml_ci_change_enforcement.py data_model/1.4
+
+  check-data_model-build-file:
+    name: Check that all data_model files are listed in the data_model_xmls.gni build file
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/project-chip/chip-build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup pip modules we use
+        run: |
+            python3 -m venv out/venv
+            out/venv/bin/pip3 install \
+                jinja2
+      - name: Generate build file (data_model_xmls.gni)
+        run: out/venv/bin/python3 src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py
+      - name: Ensure git works in current working directory
+        run: git config --global --add safe.directory `pwd`
+      - name: Check for uncommited changes
+        run: |
+          git diff --exit-code HEAD -- src/python_testing/matter_testing_infrastructure/data_model_xmls.gni

src/python_testing/matter_testing_infrastructure/BUILD.gn

@@ -18,6 +18,7 @@ import("//build_overrides/pigweed.gni")
 import("//src/python_testing/matter_testing_infrastructure/data_model_xmls.gni")
 import("$dir_pw_build/python.gni")
 import("$dir_pw_build/python_dist.gni")
+import("$dir_pw_build/zip.gni")
 
 pw_python_package("chip-testing-module") {
   generate_setup = {
@@ -52,6 +53,33 @@ pw_python_package("chip-testing-module") {
   ]
 }
 
+pw_zip("data_model_zip_1_3") {
+  inputs = []
+  foreach(file, data_model_XMLS_1_3) {
+    zip_path = rebase_path(file, "${chip_root}/data_model/1.3/", "/")
+    inputs += [ "${file} > /${zip_path}" ]
+  }
+  output = "${root_out_dir}/data_model/zip_1_3.zip"
+}
+
+pw_zip("data_model_zip_1_4") {
+  inputs = []
+  foreach(file, data_model_XMLS_1_4) {
+    zip_path = rebase_path(file, "${chip_root}/data_model/1.4/", "/")
+    inputs += [ "${file} > /${zip_path}" ]
+  }
+  output = "${root_out_dir}/data_model/zip_1_4.zip"
+}
+
+pw_zip("data_model_zip_master") {
+  inputs = []
+  foreach(file, data_model_XMLS_master) {
+    zip_path = rebase_path(file, "${chip_root}/data_model/master/", "/")
+    inputs += [ "${file} > /${zip_path}" ]
+  }
+  output = "${root_out_dir}/data_model/zip_master.zip"
+}
+
 pw_python_distribution("chip-testing") {
   packages = [ ":chip-testing-module" ]
 
@@ -62,9 +90,15 @@ pw_python_distribution("chip-testing") {
     include_extra_files_in_package_data = true
   }
 
-  # Use the imported data_model_XMLS directly
-  extra_files = []
-  foreach(file, data_model_XMLS) {
-    extra_files += [ "${file} > chip/testing/${file}" ]
-  }
+  public_deps = [
+    ":data_model_zip_1_3",
+    ":data_model_zip_1_4",
+    ":data_model_zip_master",
+  ]
+
+  extra_files = [
+    "${root_out_dir}/data_model/zip_1_3.zip > chip/testing/data_model/1.3/allfiles.zip",
+    "${root_out_dir}/data_model/zip_1_4.zip > chip/testing/data_model/1.4/allfiles.zip",
+    "${root_out_dir}/data_model/zip_master.zip > chip/testing/data_model/master/allfiles.zip",
+  ]
 }

src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py

@@ -20,6 +20,7 @@
 import logging
 import typing
 import xml.etree.ElementTree as ElementTree
+import zipfile
 from copy import deepcopy
 from dataclasses import dataclass
 from enum import Enum, auto
@@ -549,8 +550,11 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect
         return data_model_directory
 
     # If it's a prebuilt directory, build the path based on the version and data model level
-    return pkg_resources.files(importlib.import_module('chip.testing')).joinpath(
-        'data_model').joinpath(data_model_directory.dirname).joinpath(data_model_level.dirname)
+    zip_path = pkg_resources.files(importlib.import_module('chip.testing')).joinpath(
+        'data_model').joinpath(data_model_directory.dirname).joinpath('allfiles.zip')
+    path = zipfile.Path(zip_path)
+
+    return path.joinpath(data_model_level.dirname)
 
 
 def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> typing.Tuple[dict[int, dict], list]:

src/python_testing/matter_testing_infrastructure/data_model_xmls.gni

@@ -11,10 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+# This file is generated by
+# src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py
+#
+# Manually re-generate this file on any changes to the data_model directory.
 
 import("//build_overrides/chip.gni")
 
-data_model_XMLS = [
+data_model_XMLS_1_3 = [
   "${chip_root}/data_model/1.3/clusters/ACL-Cluster.xml",
   "${chip_root}/data_model/1.3/clusters/AccountLogin.xml",
   "${chip_root}/data_model/1.3/clusters/AdminCommissioningCluster.xml",
@@ -116,7 +121,6 @@ data_model_XMLS = [
   "${chip_root}/data_model/1.3/clusters/WindowCovering.xml",
   "${chip_root}/data_model/1.3/clusters/bridge-clusters-ActionsCluster.xml",
   "${chip_root}/data_model/1.3/clusters/bridge-clusters-BridgedDeviceBasicInformationCluster.xml",
-  "${chip_root}/data_model/1.3/clusters/cluster_ids.json",
   "${chip_root}/data_model/1.3/device_types/Aggregator.xml",
   "${chip_root}/data_model/1.3/device_types/AirPurifier.xml",
   "${chip_root}/data_model/1.3/device_types/AirQualitySensor.xml",
@@ -180,6 +184,9 @@ data_model_XMLS = [
   "${chip_root}/data_model/1.3/device_types/WaterValve.xml",
   "${chip_root}/data_model/1.3/device_types/WindowCovering.xml",
   "${chip_root}/data_model/1.3/device_types/WindowCoveringController.xml",
+]
+
+data_model_XMLS_1_4 = [
   "${chip_root}/data_model/1.4/clusters/ACL-Cluster.xml",
   "${chip_root}/data_model/1.4/clusters/AccountLogin.xml",
   "${chip_root}/data_model/1.4/clusters/AdminCommissioningCluster.xml",
@@ -291,7 +298,6 @@ data_model_XMLS = [
   "${chip_root}/data_model/1.4/clusters/WindowCovering.xml",
   "${chip_root}/data_model/1.4/clusters/bridge-clusters-ActionsCluster.xml",
   "${chip_root}/data_model/1.4/clusters/bridge-clusters-BridgedDeviceBasicInformationCluster.xml",
-  "${chip_root}/data_model/1.4/clusters/cluster_ids.json",
   "${chip_root}/data_model/1.4/device_types/Aggregator.xml",
   "${chip_root}/data_model/1.4/device_types/AirPurifier.xml",
   "${chip_root}/data_model/1.4/device_types/AirQualitySensor.xml",
@@ -366,6 +372,9 @@ data_model_XMLS = [
   "${chip_root}/data_model/1.4/device_types/WaterValve.xml",
   "${chip_root}/data_model/1.4/device_types/WindowCovering.xml",
   "${chip_root}/data_model/1.4/device_types/WindowCoveringController.xml",
+]
+
+data_model_XMLS_master = [
   "${chip_root}/data_model/master/clusters/ACL-Cluster.xml",
   "${chip_root}/data_model/master/clusters/AccountLogin.xml",
   "${chip_root}/data_model/master/clusters/AdminCommissioningCluster.xml",
@@ -483,7 +492,6 @@ data_model_XMLS = [
   "${chip_root}/data_model/master/clusters/bridge-clusters-ActionsCluster.xml",
   "${chip_root}/data_model/master/clusters/bridge-clusters-BridgedDeviceBasicInformationCluster.xml",
   "${chip_root}/data_model/master/clusters/bridge-clusters-EcosystemInformationCluster.xml",
-  "${chip_root}/data_model/master/clusters/cluster_ids.json",
   "${chip_root}/data_model/master/device_types/Aggregator.xml",
   "${chip_root}/data_model/master/device_types/AirPurifier.xml",
   "${chip_root}/data_model/master/device_types/AirQualitySensor.xml",

src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 """
-Generates a gni file containing all data_model files (generally XML and JSON files) 
-that are typically used by matter_testing_infrastructure.
+Generates a gni file containing all data_model files (XML)
+that are used by matter_testing_infrastructure.
 
 These files are to be bundled with whl packages of the matter_testing_infrastructure
 so that methods requiring data model files work just by installing the python
@@ -26,19 +26,10 @@
 
 # Set chip_root to be dynamically based on the script's location
 chip_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
-
-# Directories to search for .xml and .json files relative to chip_root
-directories = [
-    os.path.join(chip_root, "data_model/1.3/clusters/"),
-    os.path.join(chip_root, "data_model/1.3/device_types/"),
-    os.path.join(chip_root, "data_model/1.4/clusters/"),
-    os.path.join(chip_root, "data_model/1.4/device_types/"),
-    os.path.join(chip_root, "data_model/master/clusters/"),
-    os.path.join(chip_root, "data_model/master/device_types/"),
-]
+data_model_dir = os.path.join(chip_root, "data_model")
 
 # Template for generating the GNI file content with proper indentation
-GNI_TEMPLATE = """\
+GNI_HEADER = """\
 # Copyright (c) 2024 Project CHIP Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -52,43 +43,54 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+# This file is generated by
+# src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py
+#
+# Manually re-generate this file on any changes to the data_model directory.
 
 import("//build_overrides/chip.gni")
+"""
+
+GNI_TEMPLATE = """\
 
-data_model_XMLS = [
+data_model_XMLS_{{ dir }} = [
 {% for name in file_list %}
-    "{{ name }}",
+  "{{ name }}",
 {% endfor %}
 ]
-"""
 
-# Function to find and collect all .xml and .json files
+"""
 
 
-def get_data_model_file_names():
+def get_data_model_file_names(directory: str):
+    ''' Function to find and collect all .xml files'''
     file_list = []
-    for directory in directories:
-        for root, _, files in os.walk(directory):
-            for file in files:
-                if file.endswith(".xml") or file.endswith(".json"):
-                    # Replace absolute path with `${chip_root}` for GNI compatibility
-                    relative_path = os.path.join("${chip_root}", os.path.relpath(root, chip_root), file)
-                    file_list.append(relative_path)
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".xml"):
+                # Replace absolute path with `${chip_root}` for GNI compatibility
+                relative_path = os.path.join("${chip_root}", os.path.relpath(root, chip_root), file)
+                file_list.append(relative_path)
     # Sort files alphabetically
     file_list.sort()
     return file_list
 
-# Main function to generate the data_model_xmls.gni file
-
 
 def generate_gni_file():
-    # Step 1: Find all files and create the sorted file list
-    file_list = get_data_model_file_names()
-
-    # Step 2: Render the template with the file list
+    '''Main function to generate the data_model_xmls.gni file'''
     environment = jinja2.Environment(trim_blocks=True, lstrip_blocks=True)
     template = environment.from_string(GNI_TEMPLATE)
-    output_content = template.render(file_list=file_list)
+    output_content_per_dir = []
+
+    # Step 1: Find all files and create the sorted file list
+    dirs = sorted([d for d in os.listdir(data_model_dir) if os.path.isdir(os.path.join(data_model_dir, d))])
+    for directory in dirs:
+        file_list = get_data_model_file_names(os.path.join(data_model_dir, directory))
+        # Step 2: Render the template with the file list
+        output_content_per_dir.append(template.render(dir=directory.replace('.', '_'), file_list=file_list))
+
+    output_content = GNI_HEADER + "".join(output_content_per_dir)
 
     # Step 3: Dynamically generate the output file path
     # Get the script's directory (where this script is located)