From cafa374dd593d8b904ece8eafc2519f94059d6a9 Mon Sep 17 00:00:00 2001 From: Shane Brown Date: Tue, 12 Sep 2023 17:31:32 -0600 Subject: [PATCH] Loosen strictness of config validation --- buildrunner/validation/config.py | 7 +--- buildrunner/validation/step.py | 40 +++++-------------- .../test_validation_artifacts.py | 33 --------------- .../test_validation/test_validation_config.py | 32 +++++---------- 4 files changed, 21 insertions(+), 91 deletions(-) diff --git a/buildrunner/validation/config.py b/buildrunner/validation/config.py index c09f8a32..3dc6d6b6 100644 --- a/buildrunner/validation/config.py +++ b/buildrunner/validation/config.py @@ -6,7 +6,6 @@ with the terms of the Adobe license agreement accompanying it. """ -import logging from typing import Dict, List, Optional, Set, Union # pylint: disable=no-name-in-module @@ -15,8 +14,6 @@ from buildrunner.validation.errors import Errors, add_validation_errors from buildrunner.validation.step import Step, StepPushCommitDict -logger = logging.getLogger(__name__) - class Config(BaseModel): """ Top level config model """ @@ -44,7 +41,7 @@ class SSHKey(BaseModel, extra='forbid'): # Global config attributes env: Optional[Dict[str, str]] build_servers: Optional[Dict[str, Union[str, List[str]]]] = Field(alias='build-servers') - # Intentionally loose restrictions on ssh-keys since documentation isn't clear + # Intentionally has loose restrictions on ssh-keys since documentation isn't clear ssh_keys: Optional[Union[SSHKey, List[SSHKey]]] = Field(alias='ssh-keys') local_files: Optional[Dict[str, str]] = Field(alias='local-files') caches_root: Optional[str] = Field(alias='caches-root') @@ -62,7 +59,7 @@ def validate_steps(cls, values) -> None: ValueError | pydantic.ValidationError : If the config file is invalid """ - def validate_push(push: Union[StepPushCommitDict, List[Union[str, StepPushCommitDict]], str], + def validate_push(push: Union[StepPushCommitDict, str, List[Union[str, StepPushCommitDict]]], mp_push_tags: Set[str], step_name: str, update_mp_push_tags: bool = True): diff --git a/buildrunner/validation/step.py b/buildrunner/validation/step.py index f7793549..2c0d9c9c 100644 --- a/buildrunner/validation/step.py +++ b/buildrunner/validation/step.py @@ -6,7 +6,6 @@ with the terms of the Adobe license agreement accompanying it. """ -from enum import Enum from typing import Any, Dict, List, Optional, Union # pylint: disable=no-name-in-module @@ -22,45 +21,23 @@ class StepPypiPush(BaseModel): class Artifact(BaseModel): """ Artifact model """ - - class FormatTypes(Enum): - """ Format types """ - # pylint: disable=invalid-name - uncompressed = 'uncompressed' - - class CompressionTypes(Enum): - """ Compression types """ - # [compression: gz|bz2|xz|lzma|lzip|lzop|z] - # pylint: disable=invalid-name - gz = 'gz' - bz2 = 'bz2' - xz = 'xz' - lzma = 'lzma' - lzip = 'lzip' - lzop = 'lzop' - z = 'z' - - format: Optional[FormatTypes] + # Intentionally loose restrictions + format: Optional[str] type: Optional[Any] - compression: Optional[CompressionTypes] + compression: Optional[str] push: Optional[bool] class StepRun(BaseModel, extra='forbid'): """ Run model within a step """ - class ProvisionerTypes(Enum): - """ Provisioner types """ - # pylint: disable=invalid-name - shell: str = 'shell' - salt: str = 'salt' - xfail: Optional[bool] - services: Optional[Dict[str, str]] + services: Optional[Dict[str, Any]] image: Optional[str] cmd: Optional[str] cmds: Optional[List[str]] - provisioners: Optional[Dict[ProvisionerTypes, str]] + # Intentionally loose restrictions + provisioners: Optional[Dict[str, str]] shell: Optional[str] cwd: Optional[str] user: Optional[str] @@ -80,8 +57,9 @@ class ProvisionerTypes(Enum): systemd: Optional[bool] cap_add: Optional[Union[str, List[str]]] privileged: Optional[bool] - post_build: Optional[Union[str, Dict[str, str]]] = Field(alias='post-build') + post_build: Optional[Union[str, Dict[str, Any]]] = Field(alias='post-build') containers: Optional[List[str]] + no_cache: Optional[bool] = Field(alias='no-cache') class StepRemote(BaseModel, extra='forbid'): @@ -100,6 +78,8 @@ class StepBuild(BaseModel, extra='forbid'): platform: Optional[str] platforms: Optional[List[str]] inject: Optional[Dict[str, str]] + no_cache: Optional[bool] = Field(alias='no-cache') + buildargs: Optional[Dict[str, Any]] class StepPushCommitDict(BaseModel, extra='forbid'): diff --git a/tests/test_validation/test_validation_artifacts.py b/tests/test_validation/test_validation_artifacts.py index a2543ad2..665395e7 100644 --- a/tests/test_validation/test_validation_artifacts.py +++ b/tests/test_validation/test_validation_artifacts.py @@ -53,24 +53,6 @@ def test_step_artifacts_valid_compression(): assert errors is None -def test_step_artifacts_invalid_compression(): - config_yaml = """ - steps: - build-remote: - remote: - host: myserver.ut1 - cmd: docker build -t mytest-reg/buildrunner-test . - artifacts: - bogus/path/to/artifacts/*: - type: tar - compression: bogus - """ - config = yaml.load(config_yaml, Loader=yaml.Loader) - errors = validate_config(**config) - assert isinstance(errors, Errors) - assert errors.count() == 1 - - def test_step_run_format_valid(): config_yaml = """ steps: @@ -85,21 +67,6 @@ def test_step_run_format_valid(): assert errors is None -def test_step_run_format_invalid(): - config_yaml = """ - steps: - build-run: - run: - artifacts: - bogus/path/to/artifacts/*: - format: 134 - """ - config = yaml.load(config_yaml, Loader=yaml.Loader) - errors = validate_config(**config) - assert isinstance(errors, Errors) - assert errors.count() == 1 - - def test_step_type_valid(): # Checks zip type config_yaml = """ diff --git a/tests/test_validation/test_validation_config.py b/tests/test_validation/test_validation_config.py index 55956df2..afe6970a 100644 --- a/tests/test_validation/test_validation_config.py +++ b/tests/test_validation/test_validation_config.py @@ -376,22 +376,21 @@ def test_doc_confg(): assert errors is None -def test_full_real_confg(): +def test_full_confg(): # Minus template areas - # Taken from https://git.corp.company.com/analytics-platform/scds/blob/a627732f894537fbcdb66cf1d311d7ff657eb3b4/buildrunner.yaml config_yaml = """ steps: generate_files: run: image: docker.company.com/abc-xdm-proto-build:latest - ssh-keys: ["adobe-github"] + ssh-keys: ["company-github"] env: GIT_TOKEN: 'blahblahblahblahblahblah' - cmd: sbt clean generateContentAwareJsonFiles combineXDM generateProtobufFiles + cmd: sbt clean generateAwareJsonFiles combineXDM generateProtobufFiles artifacts: - 'target/protobufFiles/Analytics*.proto': - 'target/rawJson/Analytics*.json': - 'target/contentAwareJson/ContentAware.json': + 'target/protobufFiles/Database*.proto': + 'target/rawJson/Database*.json': + 'target/AwareJson/Aware.json': 'target/combinedXDM/complete-schema-template.schema.json': build-dev-rpm: build: @@ -400,7 +399,7 @@ def test_full_real_confg(): "buildrunner.results/generate_files/A*.json": "json/" "db_build/dms.repo.centos7": db_build/dms.repo dockerfile: | - FROM docker-xeng-release.dr.corp.company.com/xeng/centos-7-x86_64-obuild:latest + FROM docker-release.dr.corp.company.com/centos-7-x86_64-obuild:latest ADD db_build/dms.repo /etc/yum.repos.d/dms.repo RUN rpm --rebuilddb; yum clean all; yum install -y db-omniture-libs-protobuf-2.6.1 db-scds-proto-1.0 db-scds-json-1.0 ADD proto/*.proto /tmp/proto/ @@ -411,14 +410,6 @@ def test_full_real_confg(): - "echo ~ Compiling previous proto version..." - "mkdir -p /tmp/existingscds && for f in `ls -d /home/omniture/protobuf/scds/*.proto`; do protoc -I=/home/omniture/protobuf --cpp_out /tmp/existingscds $f; done" - "echo ~ Compiling current proto version..." - - "mkdir -p /tmp/currentstaging/scds && cp /tmp/proto/* /tmp/currentstaging/scds" - - "mkdir -p /tmp/currentscds && for f in `ls -d /tmp/currentstaging/scds/*.proto`; do protoc -I=/tmp/currentstaging --cpp_out /tmp/currentscds $f; done" - - "echo ~ Comparing code versions..." - - "rm -f /tmp/doredeploy" - - "for f in `ls /tmp/currentscds/scds/* | xargs -n1 basename`; do if ! diff /tmp/currentscds/scds/$f /tmp/existingscds/scds/$f ; then echo ======== $f: Protobuf versions DO NOT match: Generating RPM...; rm -f /tmp/doredeploy; break; else echo ======== $f: Protobuf versions match...; touch /tmp/doredeploy; fi; done" - - "for f in `ls /tmp/json/A*.json | xargs -n1 basename`; do if ! diff /home/omniture/json/scds/$f /tmp/json/$f ; then echo ======== $f: JSON versions DO NOT match: Generating RPM...; rm -f /tmp/doredeploy; break; else echo ======== $f: JSON version match...; touch /tmp/doredeploy; fi; done" - - "if [[ -f /tmp/doredeploy ]]; then echo Redeploying existing RPM...; mkdir -p /source/db_tmp/rpm/RPMS/noarch; chmod -R 777 db_tmp; yum remove -y db-scds-proto-1.0 db-scds-json-1.0; yum install -y --downloadonly --downloaddir=/source/db_tmp/rpm/RPMS/noarch db-scds-proto-1.0 db-scds-json-1.0 || true; ls -al /source/db_tmp/rpm/RPMS/noarch; fi" - - "if [[ ! -f /tmp/doredeploy ]]; then echo Building new RPM due to Protobuf or JSON mismatch...; su -l httpd -c 'cd /source && obuild'; fi" artifacts: # pull the log if rpmbuild fails "db_tmp/rpm/TMPDIR/*.log": {type: 'log'} @@ -444,7 +435,7 @@ def test_full_real_confg(): inject: "db_build/bin/*": "db_build/bin/" dockerfile: | - FROM docker-xeng-release.dr.corp.company.com/xeng/centos-7-x86_64-obuild + FROM docker-release.dr.corp.company.com/centos-7-x86_64-obuild ADD db_build/bin/* /tmp/ run: cmds: @@ -462,15 +453,10 @@ def test_full_real_confg(): inject: "buildrunner.results/generate_files/*.proto": "proto" "buildrunner.results/generate_files/*.json": "json" - "buildrunner.results/generate_files/complete-schema-template.schema.json": "xdmschema/complete-schema-template.schema.json" - "buildrunner.results/download-country/country_code_map.csv": "csv/country_code_map.csv" dockerfile: | FROM docker.company.com/abc-base-containers/protobuf-builder:java8-2.5.0 RUN apt-get update && apt-get -y install openssh-client ADD proto/*.proto /tmp/proto/scds/ - ADD json/ContentAware.json /tmp/json/contentAware/ - ADD xdmschema/complete-schema-template.schema.json /tmp/xdmschema/ - ADD csv/country_code_map.csv /tmp/csv/ run: env: ARTIFACTORY_USER: 'cool_user' @@ -492,7 +478,7 @@ def test_full_real_confg(): generate_docs: run: image: docker.company.com/abc-xdm-proto-build:latest - ssh-keys: ["adobe-github"] + ssh-keys: ["company-github"] env: GIT_TOKEN: 'blahblahblahblahblahblahblah' cmd: "sbt clean generateDocs ${BUILDRUNNER_DO_PUSH+publishGHPages}"