Loosen strictness of config validation

buildrunner/validation/config.py

@@ -6,7 +6,6 @@
 with the terms of the Adobe license agreement accompanying it.
 """
 
-import logging
 from typing import Dict, List, Optional, Set, Union
 
 # pylint: disable=no-name-in-module
@@ -15,8 +14,6 @@
 from buildrunner.validation.errors import Errors, add_validation_errors
 from buildrunner.validation.step import Step, StepPushCommitDict
 
-logger = logging.getLogger(__name__)
-
 
 class Config(BaseModel):
     """ Top level config model """
@@ -44,7 +41,7 @@ class SSHKey(BaseModel, extra='forbid'):
     # Global config attributes
     env: Optional[Dict[str, str]]
     build_servers: Optional[Dict[str, Union[str, List[str]]]] = Field(alias='build-servers')
-    #  Intentionally loose restrictions on ssh-keys since documentation isn't clear
+    #  Intentionally has loose restrictions on ssh-keys since documentation isn't clear
     ssh_keys: Optional[Union[SSHKey, List[SSHKey]]] = Field(alias='ssh-keys')
     local_files: Optional[Dict[str, str]] = Field(alias='local-files')
     caches_root: Optional[str] = Field(alias='caches-root')
@@ -62,7 +59,7 @@ def validate_steps(cls, values) -> None:
             ValueError | pydantic.ValidationError : If the config file is invalid
         """
 
-        def validate_push(push: Union[StepPushCommitDict, List[Union[str, StepPushCommitDict]], str],
+        def validate_push(push: Union[StepPushCommitDict, str, List[Union[str, StepPushCommitDict]]],
                           mp_push_tags: Set[str],
                           step_name: str,
                           update_mp_push_tags: bool = True):

buildrunner/validation/step.py

@@ -6,7 +6,6 @@
 with the terms of the Adobe license agreement accompanying it.
 """
 
-from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
 # pylint: disable=no-name-in-module
@@ -22,45 +21,23 @@ class StepPypiPush(BaseModel):
 
 class Artifact(BaseModel):
     """ Artifact model """
-
-    class FormatTypes(Enum):
-        """ Format types """
-        #  pylint: disable=invalid-name
-        uncompressed = 'uncompressed'
-
-    class CompressionTypes(Enum):
-        """ Compression types """
-        # [compression: gz|bz2|xz|lzma|lzip|lzop|z]
-        #  pylint: disable=invalid-name
-        gz = 'gz'
-        bz2 = 'bz2'
-        xz = 'xz'
-        lzma = 'lzma'
-        lzip = 'lzip'
-        lzop = 'lzop'
-        z = 'z'
-
-    format: Optional[FormatTypes]
+    # Intentionally loose restrictions
+    format: Optional[str]
     type: Optional[Any]
-    compression: Optional[CompressionTypes]
+    compression: Optional[str]
     push: Optional[bool]
 
 
 class StepRun(BaseModel, extra='forbid'):
     """ Run model within a step """
 
-    class ProvisionerTypes(Enum):
-        """ Provisioner types """
-        #  pylint: disable=invalid-name
-        shell: str = 'shell'
-        salt: str = 'salt'
-
     xfail: Optional[bool]
-    services: Optional[Dict[str, str]]
+    services: Optional[Dict[str, Any]]
     image: Optional[str]
     cmd: Optional[str]
     cmds: Optional[List[str]]
-    provisioners: Optional[Dict[ProvisionerTypes, str]]
+    # Intentionally loose restrictions
+    provisioners: Optional[Dict[str, str]]
     shell: Optional[str]
     cwd: Optional[str]
     user: Optional[str]
@@ -80,8 +57,9 @@ class ProvisionerTypes(Enum):
     systemd: Optional[bool]
     cap_add: Optional[Union[str, List[str]]]
     privileged: Optional[bool]
-    post_build: Optional[Union[str, Dict[str, str]]] = Field(alias='post-build')
+    post_build: Optional[Union[str, Dict[str, Any]]] = Field(alias='post-build')
     containers: Optional[List[str]]
+    no_cache: Optional[bool] = Field(alias='no-cache')
 
 
 class StepRemote(BaseModel, extra='forbid'):
@@ -100,6 +78,8 @@ class StepBuild(BaseModel, extra='forbid'):
     platform: Optional[str]
     platforms: Optional[List[str]]
     inject: Optional[Dict[str, str]]
+    no_cache: Optional[bool] = Field(alias='no-cache')
+    buildargs: Optional[Dict[str, Any]]
 
 
 class StepPushCommitDict(BaseModel, extra='forbid'):

tests/test_validation/test_validation_artifacts.py

@@ -53,24 +53,6 @@ def test_step_artifacts_valid_compression():
     assert errors is None
 
 
-def test_step_artifacts_invalid_compression():
-    config_yaml = """
-    steps:
-      build-remote:
-        remote:
-          host: myserver.ut1
-          cmd: docker build -t mytest-reg/buildrunner-test .
-          artifacts:
-            bogus/path/to/artifacts/*:
-              type: tar
-              compression: bogus
-    """
-    config = yaml.load(config_yaml, Loader=yaml.Loader)
-    errors = validate_config(**config)
-    assert isinstance(errors, Errors)
-    assert errors.count() == 1
-
-
 def test_step_run_format_valid():
     config_yaml = """
     steps:
@@ -85,21 +67,6 @@ def test_step_run_format_valid():
     assert errors is None
 
 
-def test_step_run_format_invalid():
-    config_yaml = """
-    steps:
-      build-run:
-        run:
-          artifacts:
-            bogus/path/to/artifacts/*:
-              format: 134
-    """
-    config = yaml.load(config_yaml, Loader=yaml.Loader)
-    errors = validate_config(**config)
-    assert isinstance(errors, Errors)
-    assert errors.count() == 1
-
-
 def test_step_type_valid():
     #  Checks zip type
     config_yaml = """

tests/test_validation/test_validation_config.py

@@ -376,22 +376,21 @@ def test_doc_confg():
     assert errors is None
 
 
-def test_full_real_confg():
+def test_full_confg():
     # Minus template areas
-    # Taken from https://git.corp.company.com/analytics-platform/scds/blob/a627732f894537fbcdb66cf1d311d7ff657eb3b4/buildrunner.yaml
     config_yaml = """
     steps:
       generate_files:
         run:
           image: docker.company.com/abc-xdm-proto-build:latest
-          ssh-keys: ["adobe-github"]
+          ssh-keys: ["company-github"]
           env:
             GIT_TOKEN: 'blahblahblahblahblahblah'
-          cmd: sbt clean generateContentAwareJsonFiles combineXDM generateProtobufFiles
+          cmd: sbt clean generateAwareJsonFiles combineXDM generateProtobufFiles
           artifacts:
-            'target/protobufFiles/Analytics*.proto':
-            'target/rawJson/Analytics*.json':
-            'target/contentAwareJson/ContentAware.json':
+            'target/protobufFiles/Database*.proto':
+            'target/rawJson/Database*.json':
+            'target/AwareJson/Aware.json':
             'target/combinedXDM/complete-schema-template.schema.json':
       build-dev-rpm:
         build:
@@ -400,7 +399,7 @@ def test_full_real_confg():
             "buildrunner.results/generate_files/A*.json": "json/"
             "db_build/dms.repo.centos7": db_build/dms.repo
           dockerfile: |
-            FROM docker-xeng-release.dr.corp.company.com/xeng/centos-7-x86_64-obuild:latest
+            FROM docker-release.dr.corp.company.com/centos-7-x86_64-obuild:latest
             ADD db_build/dms.repo /etc/yum.repos.d/dms.repo
             RUN rpm --rebuilddb; yum clean all; yum install -y db-omniture-libs-protobuf-2.6.1 db-scds-proto-1.0 db-scds-json-1.0
             ADD proto/*.proto /tmp/proto/
@@ -411,14 +410,6 @@ def test_full_real_confg():
             - "echo ~ Compiling previous proto version..."
             - "mkdir -p /tmp/existingscds && for f in `ls -d /home/omniture/protobuf/scds/*.proto`; do protoc -I=/home/omniture/protobuf --cpp_out /tmp/existingscds $f; done"
             - "echo ~ Compiling current proto version..."
-            - "mkdir -p /tmp/currentstaging/scds && cp /tmp/proto/* /tmp/currentstaging/scds"
-            - "mkdir -p /tmp/currentscds && for f in `ls -d /tmp/currentstaging/scds/*.proto`; do protoc -I=/tmp/currentstaging --cpp_out /tmp/currentscds $f; done"
-            - "echo ~ Comparing code versions..."
-            - "rm -f /tmp/doredeploy"
-            - "for f in `ls /tmp/currentscds/scds/* | xargs -n1 basename`; do if ! diff /tmp/currentscds/scds/$f /tmp/existingscds/scds/$f ; then echo ======== $f: Protobuf versions DO NOT match: Generating RPM...; rm -f /tmp/doredeploy; break; else echo ======== $f: Protobuf versions match...; touch /tmp/doredeploy; fi; done"
-            - "for f in `ls /tmp/json/A*.json | xargs -n1 basename`; do if ! diff /home/omniture/json/scds/$f /tmp/json/$f ; then echo ======== $f: JSON versions DO NOT match: Generating RPM...; rm -f /tmp/doredeploy; break; else echo ======== $f: JSON version match...; touch /tmp/doredeploy; fi; done"
-            - "if [[ -f /tmp/doredeploy ]]; then echo Redeploying existing RPM...; mkdir -p /source/db_tmp/rpm/RPMS/noarch; chmod -R 777 db_tmp; yum remove -y db-scds-proto-1.0 db-scds-json-1.0; yum install -y --downloadonly --downloaddir=/source/db_tmp/rpm/RPMS/noarch db-scds-proto-1.0 db-scds-json-1.0 || true; ls -al /source/db_tmp/rpm/RPMS/noarch; fi"
-            - "if [[ ! -f /tmp/doredeploy ]]; then echo Building new RPM due to Protobuf or JSON mismatch...; su -l httpd -c 'cd /source && obuild'; fi"
           artifacts:
             # pull the log if rpmbuild fails
             "db_tmp/rpm/TMPDIR/*.log": {type: 'log'}
@@ -444,7 +435,7 @@ def test_full_real_confg():
           inject:
             "db_build/bin/*": "db_build/bin/"
           dockerfile: |
-            FROM docker-xeng-release.dr.corp.company.com/xeng/centos-7-x86_64-obuild
+            FROM docker-release.dr.corp.company.com/centos-7-x86_64-obuild
             ADD db_build/bin/* /tmp/
         run:
           cmds:
@@ -462,15 +453,10 @@ def test_full_real_confg():
           inject:
             "buildrunner.results/generate_files/*.proto": "proto"
             "buildrunner.results/generate_files/*.json": "json"
-            "buildrunner.results/generate_files/complete-schema-template.schema.json": "xdmschema/complete-schema-template.schema.json"
-            "buildrunner.results/download-country/country_code_map.csv": "csv/country_code_map.csv"
           dockerfile: |
             FROM docker.company.com/abc-base-containers/protobuf-builder:java8-2.5.0
             RUN apt-get update && apt-get -y install openssh-client
             ADD proto/*.proto /tmp/proto/scds/
-            ADD json/ContentAware.json /tmp/json/contentAware/
-            ADD xdmschema/complete-schema-template.schema.json /tmp/xdmschema/
-            ADD csv/country_code_map.csv /tmp/csv/
         run:
           env:
             ARTIFACTORY_USER: 'cool_user'
@@ -492,7 +478,7 @@ def test_full_real_confg():
       generate_docs:
         run:
           image: docker.company.com/abc-xdm-proto-build:latest
-          ssh-keys: ["adobe-github"]
+          ssh-keys: ["company-github"]
           env:
             GIT_TOKEN: 'blahblahblahblahblahblahblah'
           cmd: "sbt clean generateDocs ${BUILDRUNNER_DO_PUSH+publishGHPages}"