ci/cd: add azure-deploy json params (#2)

* add azure-deploy json params * add run-benchmark-prep job to docker.yml * fix typo * fix typo * Update azure-deploy.json Co-authored-by: Alex Axthelm <[email protected]> --------- Co-authored-by: CJ Yetman <[email protected]> Co-authored-by: Alex Axthelm <[email protected]>
RMI-PACTA · May 30, 2024 · 613a651 · 613a651
1 parent 9ebc0fd
commit 613a651
Show file tree

Hide file tree

Showing 4 changed files with 282 additions and 1 deletion.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -27,3 +27,9 @@ jobs:
         ]
       do-lint: false
       do-check-r-sysdeps: false
+  run-benchmark-prep:
+    uses: ./.github/workflows/run-benchmark-preparation.yml
+    needs: [docker]
+    secrets: inherit
+    with:
+      image-tag: ${{ needs.docker.outputs.full-image-name }}
diff --git a/.github/workflows/run-benchmark-preparation.yml b/.github/workflows/run-benchmark-preparation.yml
@@ -61,7 +61,7 @@ jobs:
             az --version
             IMAGE_TAG=$(echo $FULL_IMAGE_NAME | sed 's/^.*://')
             az deployment group create \
-              --name "workflow.scenario.preparation-$R_CONFIG_ACTIVE" \
+              --name "workflow.benchmark.preparation-$R_CONFIG_ACTIVE" \
               --resource-group "$RESOURCEGROUP" \
               --template-file azure-deploy.json \
               --parameters azure-deploy.rmi-pacta.parameters.json \

diff --git a/azure-deploy.json b/azure-deploy.json
@@ -0,0 +1,222 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "0.0.0.5",
+  "parameters": {
+    "containerGroupPrefix": {
+      "type": "string",
+      "defaultValue": "workflow.benchmark.preparation",
+      "metadata": {
+        "description": "The name of the container group."
+      }
+    },
+    "identity": {
+      "type": "string",
+      "metadata": {
+        "description": "The ID of the user assigned identity to use for the container group."
+      }
+    },
+    "location": {
+      "type": "string",
+      "defaultValue": "[resourceGroup().location]",
+      "metadata": {
+        "description": "Location for all resources."
+      }
+    },
+    "restartPolicy": {
+      "type": "string",
+      "defaultValue": "Never",
+      "allowedValues": [
+        "Always",
+        "Never",
+        "OnFailure"
+      ],
+      "metadata": {
+        "description": "The behavior of Azure runtime if container has stopped."
+      }
+    },
+    "starttime": {
+      "type": "string",
+      "defaultValue": "[utcNow()]",
+      "metadata": {
+        "description": "The time this template is deployed."
+      }
+    },
+    "imageTag": {
+      "type": "string",
+      "defaultValue": "main",
+      "metadata": {
+        "description": "Image tag for the loader container."
+      }
+    },
+    "logWorkspaceId": {
+      "type": "string",
+      "metadata": {
+        "description": "The ID for a Log Analytics Workspace."
+      }
+    },
+    "logWorkspaceKey": {
+      "type": "securestring",
+      "metadata": {
+        "description": "The key for a Log Analytics Workspace."
+      }
+    },
+    "storageAccountKeySources": {
+      "type": "securestring",
+      "metadata": {
+        "description": "The storage account key for the storage account for source files."
+      }
+    },
+    "storageAccountNameSources": {
+      "type": "string",
+      "metadata": {
+        "description": "The storage account name for the storage account for source files."
+      }
+    },
+    "storageAccountShareSources": {
+      "type": "string",
+      "metadata": {
+        "description": "The file share name for the source files."
+      }
+    },
+    "storageAccountKeyOutputs": {
+      "type": "securestring",
+      "metadata": {
+        "description": "The storage account key for the storage account for output files."
+      }
+    },
+    "storageAccountNameOutputs": {
+      "type": "string",
+      "metadata": {
+        "description": "The storage account name for the storage account for output files."
+      }
+    },
+    "storageAccountShareOutputs": {
+      "type": "string",
+      "metadata": {
+        "description": "The file share name for the output files."
+      }
+    },
+    "rConfigActive": {
+      "type": "string",
+      "allowedValues": [
+        "2022Q4",
+        "2023Q4"
+      ],
+      "metadata": {
+        "description": "Active configuration from config.yml"
+      }
+    }
+  },
+  "variables": {
+    "containerregistry": "ghcr.io/rmi-pacta",
+    "machineCpuCoresLimit": 1,
+    "machineCpuCoresRequest": 1,
+    "machineMemoryInGBLimit": 4,
+    "machineMemoryInGBRequest": 4,
+    "mountPathSources": "/mnt/benchmarks",
+    "mountPathOutputs": "/mnt/workflow-benchmark-preparation-outputs",
+    "containerGroupName": "[concat(parameters('containerGroupPrefix'), '-', parameters('rConfigActive'))]"
+  },
+  "functions": [],
+  "resources": [
+    {
+      "type": "Microsoft.ContainerInstance/containerGroups",
+      "apiVersion": "2021-09-01",
+      "name": "[variables('containerGroupName')]",
+      "location": "[parameters('location')]",
+      "identity": {
+        "type": "UserAssigned",
+        "userAssignedIdentities": {
+          "[parameters('identity')]": {}
+        }
+      },
+      "properties": {
+        "diagnostics": {
+          "logAnalytics": {
+            "logType": "ContainerInstanceLogs",
+            "workspaceId": "[parameters('logWorkspaceId')]",
+            "workspaceKey": "[parameters('logWorkspaceKey')]"
+          }
+        },
+        "containers": [
+          {
+            "name": "benchmark-prep-runner",
+            "properties": {
+              "image": "[concat(variables('containerregistry'),'/workflow.benchmark.preparation:', parameters('imageTag'))]",
+              "ports": [],
+              "resources": {
+                "limits": {
+                  "cpu": "[variables('machineCpuCoresLimit')]",
+                  "memoryInGB": "[variables('machineMemoryInGBLimit')]"
+                },
+                "requests": {
+                  "cpu": "[variables('machineCpuCoresRequest')]",
+                  "memoryInGB": "[variables('machineMemoryInGBRequest')]"
+                }
+              },
+              "environmentVariables": [
+                {
+                  "name": "DEPLOY_START_TIME",
+                  "value": "[parameters('starttime')]"
+                },
+                {
+                  "name": "MACHINE_CORES",
+                  "value": "[variables('machineCpuCoresRequest')]"
+                },
+                {
+                  "name": "LOG_LEVEL",
+                  "value": "TRACE"
+                },
+                {
+                  "name": "BENCHMARKS_PREPARATION_INPUTS_PATH",
+                  "value": "[variables('mountPathSources')]"
+                },
+                {
+                  "name": "BENCHMARKS_PREPARATION_OUTPUTS_PATH",
+                  "value": "[variables('mountPathOutputs')]"
+                },
+                {
+                  "name": "R_CONFIG_ACTIVE",
+                  "value": "[parameters('rConfigActive')]"
+                }
+              ],
+              "volumeMounts": [
+                {
+                  "name": "benchmarks",
+                  "mountPath": "[variables('mountPathSources')]"
+                },
+                {
+                  "name": "benchmark-preparation-outputs",
+                  "mountPath": "[variables('mountPathOutputs')]"
+                }
+              ]
+            }
+          }
+        ],
+        "restartPolicy": "[parameters('restartPolicy')]",
+        "osType": "Linux",
+        "volumes": [
+          {
+            "name": "benchmarks",
+            "azureFile": {
+              "shareName": "[parameters('storageAccountShareSources')]",
+              "readOnly": true,
+              "storageAccountName": "[parameters('storageAccountNameSources')]",
+              "storageAccountKey": "[parameters('storageAccountKeySources')]"
+            }
+          },
+          {
+            "name": "benchmark-preparation-outputs",
+            "azureFile": {
+              "shareName": "[parameters('storageAccountShareOutputs')]",
+              "readOnly": false,
+              "storageAccountName": "[parameters('storageAccountNameOutputs')]",
+              "storageAccountKey": "[parameters('storageAccountKeyOutputs')]"
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "outputs": {}
+}
diff --git a/azure-deploy.rmi-pacta.parameters.json b/azure-deploy.rmi-pacta.parameters.json
@@ -0,0 +1,53 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "identity": {
+      "value": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.ManagedIdentity/userAssignedIdentities/pacta-runner-dev"
+    },
+    "storageAccountKeySources": {
+      "reference": {
+        "keyVault": {
+          "id": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.KeyVault/vaults/pacta-vault-dev"
+        },
+        "secretName": "rawdata-storageaccountkey"
+      }
+    },
+    "storageAccountNameSources": {
+      "value": "pactarawdata"
+    },
+    "storageAccountShareSources": {
+      "value": "benchmarks"
+    },
+    "storageAccountKeyOutputs": {
+      "reference": {
+        "keyVault": {
+          "id": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.KeyVault/vaults/pacta-vault-dev"
+        },
+        "secretName": "pactadatadev-storageaccountkey"
+      }
+    },
+    "storageAccountNameOutputs": {
+      "value": "pactadatadev"
+    },
+    "storageAccountShareOutputs": {
+      "value": "workflow-benchmark-preparation-outputs"
+    },
+    "logWorkspaceId": {
+      "reference": {
+        "keyVault": {
+          "id": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.KeyVault/vaults/pacta-vault-dev"
+        },
+        "secretName": "LogWorkspaceID-Dev"
+      }
+    },
+    "logWorkspaceKey": {
+      "reference": {
+        "keyVault": {
+          "id": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.KeyVault/vaults/pacta-vault-dev"
+        },
+        "secretName": "LogWorkspaceKey-Dev"
+      }
+    }
+  }
+}