Put projects in dynamic pipeline schema (#53)

* Put projects in dynamic pipeline schema * Remove jsonchema validation * Remove main_path * Add default branch option * Validate dynamic pipeline config * Update README.md example * Update diff commands
jwplayer · Apr 7, 2020 · cd13c1e · cd13c1e
1 parent 6234c6f
commit cd13c1e
Show file tree

Hide file tree

Showing 8 changed files with 209 additions and 138 deletions.
diff --git a/README.md b/README.md
@@ -15,33 +15,33 @@ Example
 steps:
   - label: ":buildkite:"
     plugins:
-      - jwplayer/buildpipe#v0.7.4:
+      - jwplayer/buildpipe#v0.8.0:
           dynamic_pipeline: dynamic_pipeline.yml
-          projects:
-           - label: project1
-             path: project1/  # changes in this dir will trigger steps for project1
-             skip:
-               - test  # skip steps with label test
-               - deploy*  # skip steps with label matching deploy* (e.g. deploy-prd)
-           - label: project2
-             skip: test
-             path: project2/
-           - label: project3
-             skip: deploy-stg
-             path:
-               - project3/
-               - project2/somedir/  # project3 steps will also be triggered by changes in this dir
 ```
 
 ### dynamic\_pipeline.yml
 
 ```yaml
-steps:
+projects:
+ - label: project1
+   path: project1/  # changes in this dir will trigger steps for project1
+   skip: deploy*  # skip steps with label matching deploy* (e.g. deploy-prd)
+ - label: project2
+   skip: test
+   path:
+      - project2/
+      - project1  # you can trigger a project using multiple paths
+ - label: project3
+   skip:  # you can skip a list of projects
+     - test
+     - deploy-stg
+   path: project3/somedir/  # subpaths can also be triggered
+steps:  # the same schema as regular buildkite pipeline steps
   - label: test
     env:
       BUILDPIPE_SCOPE: project  # this variable ensures a test step is generated for each project
     command:
-      - cd $$BUILDPIPE_PROJECT_PATH
+      - cd $$BUILDPIPE_PROJECT_PATH  # BUILDPIPE_PROJECT_PATH will be set by buildpipe
       - make test
   - wait
   - label: build
@@ -60,7 +60,7 @@ steps:
     command:
       - make tag-release
   - wait
-  - label: deploy-staging
+  - label: deploy-stg
     branches: "master"
     env:
       BUILDPIPE_SCOPE: project
@@ -71,7 +71,7 @@ steps:
   - block: ":rocket: Release!"
     branches: "master"
   - wait
-  - label: deploy-prod
+  - label: deploy-prd
     branches: "master"
     env:
       BUILDPIPE_SCOPE: project
@@ -99,14 +99,15 @@ Configuration
 
 ### Plugin
 
-| Option            | Required | Type   | Default | Description
-| ----------------- | -------- | ------ | ------- | -------------------------------------------------- |
-| dynamic\_pipeline | Yes      | string |         | The name including the path to the pipeline that contains all the actual steps |
-| diff              | No       | string |         | Can be used to override the default commands (see below for a better explanation of the defaults) |
-| log\_level        | No       | string | INFO    | The Level of logging to be used by the python script underneath; pass DEBUG for verbose logging if errors occur |
-| projects          | Yes      |  array |         | List of projects that buildpipe will run steps for |
+| Option           | Required | Type   | Default | Description
+| ---------------- | -------- | ------ | ------- | -------------------------------------------------- |
+| default_branch   | No       | string | master  | Default branch of repository |
+| diff_pr          | No       | string |         | Override command for non-default branch (see below for a better explanation of the defaults) |
+| diff_default     | No       | string |         | Override command for default branch (see below for a better explanation of the defaults) |
+| dynamic_pipeline | Yes      | string |         | The name including the path to the pipeline that contains all the actual steps |
+| log_level        | No       | string | INFO    | The Level of logging to be used by the python script underneath; pass DEBUG for verbose logging if errors occur |
 
-### Project
+### Project schema
 
 | Option | Required | Type   | Default | Description                           |
 | ------ | -------- | ------ | ------- | ------------------------------------- |
@@ -121,14 +122,24 @@ Other useful things to note:
 -   If multiple paths are specified, the environment variable
     `BUILDPIPE_PROJECT_PATH` will be the first path.
 
-`diff` command
---------------
+`diff_` commands
+----------------
 
 Depending on your [merge
 strategy](https://help.github.com/en/github/administering-a-repository/about-merge-methods-on-github),
 you might need to use different diff command.
 
-Buildpipe assumes you are using a merge strategy on the master branch.
+Buildpipe assumes you are using a merge strategy on the default branch, which is assumed to be `master`.
+
+The command for the non-default branch (e.g. when you have a PR up) is:
+```bash
+git log --name-only --no-merges --pretty=format: origin..HEAD
+```
+
+The command for the default branch you merge to is currently:
+```bash
+git log -m -1 --name-only --pretty=format: $BUILDKITE_COMMIT
+```
 
 
 Requirements

diff --git a/buildpipe/__main__.py b/buildpipe/__main__.py
@@ -3,18 +3,62 @@
 import argparse
 from fnmatch import fnmatch
 import io
+import json
 import logging
 import os
-import re
 import subprocess
 import sys
-from typing import List, Set
+from typing import List, Set, Union, Iterable
 
+import jsonschema
 from ruamel.yaml import YAML
 from ruamel.yaml.scanner import ScannerError
 
 from buildpipe import __version__
 
+pipeline_schema = json.loads("""
+{
+  "$schema": "http://json-schema.org/draft-06/schema#",
+  "title": "buildpipe schema",
+  "description": "buildpipe schema",
+  "type": "object",
+  "definitions": {
+    "Project": {
+      "type": "object",
+      "properties": {
+        "label": {
+          "type": "string",
+          "description": "Project label"
+        },
+        "path": {
+          "type": ["string", "array"],
+          "description": "Path of project"
+        },
+        "skip": {
+          "type": ["string", "array"],
+          "description": "Step labels to skip"
+        }
+      },
+      "required": ["label", "path"],
+      "additionalProperties": false
+    }
+  },
+  "properties": {
+    "projects": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/Project"
+      }
+    },
+    "steps": {
+      "type": "array"
+    }
+  },
+  "required": ["projects", "steps"],
+  "additionalProperties": false
+}
+""".strip())
+
 
 PLUGIN_PREFIX = "BUILDKITE_PLUGIN_BUILDPIPE_"
 
@@ -28,6 +72,21 @@
 yaml.representer.ignore_aliases = lambda *data: True
 
 
+class BuildpipeException(Exception):
+    pass
+
+
+def listify(arg: Union[None, str, List[str]]) -> List[str]:
+    if arg is None or len(arg) == 0:
+        return []
+    elif isinstance(arg, str):
+        return [arg]
+    elif isinstance(arg, Iterable):
+        return list(arg)
+    else:
+        raise ValueError(f"Argument is neither None, string nor list. Found {arg}")
+
+
 def dump_to_string(d):
     with io.StringIO() as buf:
         yaml.dump(d, buf)
@@ -51,16 +110,12 @@ def get_git_branch() -> str:
 def get_changed_files() -> Set[str]:
     branch = get_git_branch()
     logger.debug("Current branch: %s", branch)
-    deploy_branch = os.getenv(f"{PLUGIN_PREFIX}DEPLOY_BRANCH", "master")
-    commit = os.getenv("BUILDKITE_COMMIT") or branch
-    if branch == deploy_branch:
-        command = f"git log -m -1 --name-only --pretty=format: {commit}"
+    default_branch = os.getenv(f"{PLUGIN_PREFIX}DEFAULT_BRANCH", "master")
+    if branch == default_branch:
+        commit = os.getenv("BUILDKITE_COMMIT", branch)
+        command = os.getenv(f"{PLUGIN_PREFIX}DIFF_DEFAULT", f"git log -m -1 --name-only --pretty=format: {commit}")
     else:
-        diff = os.getenv(f"{PLUGIN_PREFIX}DIFF")
-        if diff:
-            command = diff
-        else:
-            command = "git log --name-only --no-merges --pretty=format: origin..HEAD"
+        command = os.getenv(f"{PLUGIN_PREFIX}DIFF_PR", "git log --name-only --no-merges --pretty=format: origin..HEAD")
 
     try:
         result = subprocess.run(command, stdout=subprocess.PIPE, shell=True)
@@ -69,7 +124,7 @@ def get_changed_files() -> Set[str]:
         logger.error(e)
         sys.exit(-1)
 
-    if branch == deploy_branch:
+    if branch == default_branch:
         try:
             first_merge_break = changed.index("")
             changed = changed[:first_merge_break]
@@ -87,7 +142,7 @@ def generate_project_steps(step: dict, projects: List[dict]) -> List[dict]:
                 "label": f'{step["label"]} {project["label"]}',
                 "env": {
                     "BUILDPIPE_PROJECT_LABEL": project["label"],
-                    "BUILDPIPE_PROJECT_PATH": project["main_path"],
+                    "BUILDPIPE_PROJECT_PATH": listify(project["path"])[0],
                     # Make sure step envs aren't overridden
                     **(step.get("env") or {}),
                 },
@@ -99,7 +154,7 @@ def generate_project_steps(step: dict, projects: List[dict]) -> List[dict]:
 
 
 def check_project_affected(project: dict, changed_files: Set[str]) -> bool:
-    for path in project["path"]:
+    for path in listify(project.get("path")):
         if path == ".":
             return True
         project_dirs = os.path.normpath(path).split("/")
@@ -121,14 +176,14 @@ def get_affected_projects(projects: List[dict]) -> List[dict]:
 
 
 def check_project_rules(step: dict, project: dict) -> bool:
-    for pattern in project.get("skip", []):
+    for pattern in listify(project.get("skip")):
         if fnmatch(step["label"], pattern):
             return False
 
     return True
 
 
-def generate_pipeline(steps: dict, projects: List[dict]) -> dict:
+def generate_pipeline(steps: List[dict], projects: List[dict]) -> dict:
     generated_steps = []
     for step in steps:
         if "env" in step and step.get("env", {}).get("BUILDPIPE_SCOPE") == "project":
@@ -139,7 +194,7 @@ def generate_pipeline(steps: dict, projects: List[dict]) -> dict:
     return {"steps": generated_steps}
 
 
-def load_steps() -> dict:
+def load_dynamic_pipeline() -> dict:
     filename = os.environ[f"{PLUGIN_PREFIX}DYNAMIC_PIPELINE"]
     try:
         with open(filename, "r") as f:
@@ -151,7 +206,7 @@ def load_steps() -> dict:
         logger.error("Invalid YAML in file %s: %s", filename, e)
         sys.exit(-1)
     else:
-        return pipeline["steps"]
+        return pipeline
 
 
 def upload_pipeline(pipeline: dict):
@@ -169,41 +224,13 @@ def upload_pipeline(pipeline: dict):
         logger.debug("Pipeline:\n%s", out)
 
 
-def get_projects() -> List[dict]:
-    re_label = re.compile(f"{PLUGIN_PREFIX}PROJECTS_[0-9]*_LABEL")
-    project_labels = {k: v for k, v in os.environ.items() if re.search(re_label, k)}
-    projects = []
-
-    for key, label in project_labels.items():
-        logger.debug("Checking %s", key)
-
-        project = {}
-        project_number = key.replace(f"{PLUGIN_PREFIX}PROJECTS_", "").replace(
-            "_LABEL", ""
-        )
-        project["label"] = label
-
-        main_path = os.getenv(f"{PLUGIN_PREFIX}PROJECTS_{project_number}_PATH")
-        if main_path is not None:
-            project["main_path"] = main_path
-        else:
-            # path is an array so choose the first path as the main one
-            project["main_path"] = os.environ[
-                f"{PLUGIN_PREFIX}PROJECTS_{project_number}_PATH_0"
-            ]
-
-        for option in ["PATH", "SKIP"]:
-            logger.debug("Checking %s", option)
-
-            re_option = re.compile(f"{PLUGIN_PREFIX}PROJECTS_{project_number}_{option}")
-
-            values = [v for k, v in os.environ.items() if re.search(re_option, k)]
-
-            logger.debug("Found patterns: (%s)", values)
-            project[option.lower()] = values
-        projects.append(project)
-
-    return projects
+def validate_dynamic_pipeline(pipeline: list) -> bool:
+    try:
+        jsonschema.validate(pipeline, pipeline_schema)
+    except jsonschema.exceptions.ValidationError as e:
+        raise BuildpipeException("Invalid projects schema") from e
+    else:
+        return True
 
 
 def create_parser():
@@ -215,11 +242,14 @@ def create_parser():
 def main():
     parser = create_parser()
     parser.parse_args()
-    projects = get_projects()
+    dynamic_pipeline = load_dynamic_pipeline()
+    validate_dynamic_pipeline(dynamic_pipeline)
+    steps, projects = dynamic_pipeline["steps"], dynamic_pipeline["projects"]
     affected_projects = get_affected_projects(projects)
+
     if not affected_projects:
         logger.info("No project was affected from changes")
         sys.exit(0)
-    steps = load_steps()
-    pipeline = generate_pipeline(steps, affected_projects)
-    upload_pipeline(pipeline)
+
+    generated_pipeline = generate_pipeline(steps, affected_projects)
+    upload_pipeline(generated_pipeline)
diff --git a/hooks/command b/hooks/command
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-buildpipe_version="${BUILDKITE_PLUGIN_BUILDPIPE_VERSION:-0.7.4}"
+buildpipe_version="${BUILDKITE_PLUGIN_BUILDPIPE_VERSION:-0.8.0}"
 is_test="${BUILDKITE_PLUGIN_BUILDPIPE_TEST_MODE:-false}"
 
 if [[ "$is_test" == "false" ]]; then

diff --git a/plugin.yml b/plugin.yml
@@ -6,22 +6,15 @@ requirements:
   - pip3
 configuration:
   properties:
-    diff:
+    default_branch:
+      type: string
+    diff_default:
+      type: string
+    diff_pr:
       type: string
     dynamic_pipeline:
       type: string
     log_level:
       type: string
-    projects:
-      type: [ array ]
-      minimum: 1
-      properties:
-        label:
-          type: string
-        path:
-          type: [ string, array ]
-        skip:
-          type: [ string, array ]
   required:
     - dynamic_pipeline
-    - projects