chore: securing terraform configs and bumping deps

.github/workflows/audit-terraform.yml

@@ -29,15 +29,21 @@ jobs:
 
     steps:
       - name: Clone repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
-      - name: Run tfsec
-        uses: aquasecurity/tfsec-sarif-action@21ded20e8ca120cd9d3d6ab04ef746477542a608
+      - name: Run Trivy vulnerability scanner in IaC mode for Terraform
+        uses: aquasecurity/trivy-action@b2933f565dbc598b29947660e66259e3c7bc8561 # 0.20.0
         with:
-          sarif_file: tfsec.sarif
+          scan-type: 'config'
+          scan-ref: './infra/terraform'
+          hide-progress: true
+          format: 'sarif'
+          output: 'trivy-terraform-results.sarif'
+          exit-code: '1'
+          ignore-unfixed: true
+          severity: 'CRITICAL,HIGH'
 
-      - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@v2
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@e14ec90e52a057614c707baecf2ed88a81b68bc9 # v2
         with:
-          # Path to SARIF file relative to the root of the repository
-          sarif_file: tfsec.sarif
+          sarif_file: 'trivy-terraform-results.sarif'

Makefile

@@ -203,8 +203,8 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
 KIND ?= $(LOCALBIN)/kind
 
 # tool versions
-KUSTOMIZE_VERSION ?= v5.1.1
-KIND_VERSION ?= v0.20.0
+KUSTOMIZE_VERSION ?= v5.4.3
+KIND_VERSION ?= v0.23.0
 
 # kustomize 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"

docs/azd.md

@@ -65,19 +65,30 @@ The following environment variables are used to define the deployment settings:
 
 | Variable | Description |
 |----------|-------------|
+| `AKS_VMSS_SKU` | The SKU of the virtual machine scale set nodes in the AKS cluster. The default is `Standard_DS2_v2`. |
 | `DEPLOY_AZURE_CONTAINER_REGISTRY` | By default, all application containers will be sourced from the [GitHub Container Registry](https://github.com/orgs/Azure-Samples/packages?repo_name=aks-store-demo). If you want to deploy apps from an Azure Container registry instead, set this environment variable to `true` to provision an Azure Container Registry and enable authentication from the AKS cluster. When this is set to true, you also have an option to set `BUILD_CONTAINERS` to `true` to build containers from source using the `az acr build command`; otherwise, the containers will be imported from the [GitHub Container Registry](https://github.com/orgs/Azure-Samples/packages?repo_name=aks-store-demo) using the `az acr import` command. |
 | `DEPLOY_AZURE_WORKLOAD_IDENTITY` | Set to `true` to deploy Azure Managed Identities for services that support it and enables workload identity and OIDC Issuer URL on AKS. |
 | `DEPLOY_AZURE_OPENAI` | Set to `true` to deploy Azure OpenAI, the `ai-service` microservice with workload identity authentication if that option was set to true. |
+| `AZURE_OPENAI_LOCATION` | The Azure region where the Azure OpenAI account will be deployed. Check [Provisioned deployment model availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#provisioned-deployment-model-availability) for availability. |
 | `DEPLOY_AZURE_OPENAI_DALL_E_MODEL` | Set to `true` to deploy the DALL-E 3 model on Azure OpenAI. |
 | `DEPLOY_AZURE_SERVICE_BUS` | Set to `true` to deploy Azure Service Bus and configures workload identity if that option is set to true. |
 | `DEPLOY_AZURE_COSMOSDB` | Set to `true` to deploy Azure Cosmos DB. When this is set to true, you can also set `AZURE_COSMOSDB_ACCOUNT_KIND` to `GlobalDocumentDB` to use the SQL API for Azure Cosmos DB; otherwise, MongoDB API will be used. The `makeline-service` supports both MongoDB and SQL API for accessing data in Azure CosmosDB. The default API is `MongoDB`, but if `DEPLOY_AZURE_WORKLOAD_IDENTITY` is set this will default to SQL API so that Azure RBAC authentication can be enabled for the Azure CosmosDB. |
+| `AZURE_COSMOSDB_FAILOVER_LOCATION` | The location to pair with the primary location as failover location for the Azure Cosmos DB account. Check [Azure paired regions](https://learn.microsoft.com/azure/reliability/cross-region-replication-azure). |
 | `DEPLOY_OBSERVABILITY_TOOLS` | Set to `true` to deploy Azure Log Analytics workspace, Azure Monitor managed service for Promethues, Azure Managed Grafana, and onboard the AKS cluster to Container Insights. |
 
 These environment variables listed above can be set with commands like this:
 
 ```bash
+# set the main deployment location
+azd env set AZURE_LOCATION eastus2
+
+# set the SKU of the virtual machine scale set nodes in the AKS cluster
+azd env set AKS_VMSS_SKU Standard_DS2_v3
+
 # deploys azure container registry and builds containers from source
 azd env set DEPLOY_AZURE_CONTAINER_REGISTRY true
+
+# builds containers from source using the az acr build command otherwise imports containers from the github container registry
 azd env set BUILD_CONTAINERS true
 
 # enables workload identity on the aks cluster and deploys managed identities
@@ -86,6 +97,9 @@ azd env set DEPLOY_AZURE_WORKLOAD_IDENTITY true
 # deploys azure openai
 azd env set DEPLOY_AZURE_OPENAI true
 
+# azure openai region
+azd env set AZURE_OPENAI_LOCATION eastus2
+
 # deploys the DALL-E 3 model on azure openai
 azd env set DEPLOY_AZURE_OPENAI_DALL_E_MODEL true
 
@@ -95,6 +109,9 @@ azd env set DEPLOY_AZURE_SERVICE_BUS true
 # deploys azure cosmos db with the sql api
 azd env set DEPLOY_AZURE_COSMOSDB true
 
+# choose the appropriate region pair for your preferred location
+azd env set AZURE_COSMOSDB_FAILOVER_LOCATION eastus2
+
 # note this is the default when DEPLOY_AZURE_WORKLOAD_IDENTITY is set to true
 azd env set AZURE_COSMOSDB_ACCOUNT_KIND GlobalDocumentDB
 

infra/terraform/cosmosdb.tf

@@ -7,7 +7,7 @@ resource "azurerm_cosmosdb_account" "example" {
   kind                               = local.cosmosdb_account_kind
   access_key_metadata_writes_enabled = !local.deploy_azure_workload_identity
   minimal_tls_version                = "Tls12"
-  automatic_failover_enabled         = false
+  automatic_failover_enabled         = true
 
   dynamic "capabilities" {
     for_each = local.cosmosdb_account_kind == "MongoDB" ? ["EnableAggregationPipeline", "mongoEnableDocLevelTTL", "MongoDBv3.4", "EnableMongo"] : ["EnableAggregationPipeline"]
@@ -23,13 +23,13 @@ resource "azurerm_cosmosdb_account" "example" {
   }
 
   geo_location {
-    location          = "eastus"
-    failover_priority = 1
+    location          = azurerm_resource_group.example.location
+    failover_priority = 0 # primary location
   }
 
   geo_location {
-    location          = "westus"
-    failover_priority = 0
+    location          = local.cosmosdb_failover_location
+    failover_priority = 1 # secondary location
   }
 }
 
@@ -72,7 +72,7 @@ resource "azurerm_cosmosdb_sql_container" "example" {
   resource_group_name   = azurerm_cosmosdb_account.example[0].resource_group_name
   account_name          = azurerm_cosmosdb_account.example[0].name
   database_name         = azurerm_cosmosdb_sql_database.example[0].name
-  partition_key_path    = "/storeId"
+  partition_key_paths   = ["/storeId"]
   partition_key_version = 1
   throughput            = 400
 }

infra/terraform/keyvault.tf

@@ -7,6 +7,16 @@ resource "azurerm_key_vault" "example" {
   tenant_id                   = data.azurerm_client_config.current.tenant_id
   sku_name                    = "standard"
   enable_rbac_authorization   = var.kv_rbac_enabled
+  purge_protection_enabled    = true
+  soft_delete_retention_days  = 7
+
+  network_acls {
+    default_action = "Deny"
+    bypass         = "AzureServices"
+    ip_rules = [
+      "${data.http.ifconfig.response_body}/32"
+    ]
+  }
 
   dynamic "access_policy" {
     for_each = var.kv_rbac_enabled ? [] : [1]

infra/terraform/kubernetes.tf

@@ -14,14 +14,32 @@ resource "azurerm_kubernetes_cluster" "example" {
 
   default_node_pool {
     name       = "system"
-    vm_size    = "Standard_D4s_v4"
+    vm_size    = local.aks_node_pool_vm_size
     node_count = 3
 
     upgrade_settings {
       max_surge = "10%"
     }
   }
 
+  azure_active_directory_role_based_access_control {
+    managed            = true
+    azure_rbac_enabled = true
+  }
+
+  api_server_access_profile {
+    authorized_ip_ranges = [
+      "${data.http.ifconfig.response_body}/32"
+    ]
+  }
+
+  network_profile {
+    network_plugin      = "azure"
+    network_plugin_mode = "overlay"
+    network_policy      = "cilium"
+    network_data_plane  = "cilium"
+  }
+
   identity {
     type = "SystemAssigned"
   }
@@ -40,12 +58,9 @@ resource "azurerm_kubernetes_cluster" "example" {
     }
   }
 
-  dynamic "oms_agent" {
-    for_each = local.deploy_observability_tools ? [1] : []
-    content {
-      log_analytics_workspace_id      = azurerm_log_analytics_workspace.example[0].id
-      msi_auth_for_monitoring_enabled = true
-    }
+  oms_agent {
+    log_analytics_workspace_id      = azurerm_log_analytics_workspace.example.id
+    msi_auth_for_monitoring_enabled = true
   }
 
   lifecycle {
@@ -57,6 +72,12 @@ resource "azurerm_kubernetes_cluster" "example" {
   }
 }
 
+resource "azurerm_role_assignment" "aks_cluster_admin" {
+  principal_id         = data.azurerm_client_config.current.object_id
+  role_definition_name = "Azure Kubernetes Service RBAC Cluster Admin"
+  scope                = azurerm_kubernetes_cluster.example.id
+}
+
 resource "azurerm_role_assignment" "example" {
   count                            = local.deploy_azure_container_registry ? 1 : 0
   principal_id                     = azurerm_kubernetes_cluster.example.kubelet_identity[0].object_id

infra/terraform/locals.tf

@@ -1,13 +1,16 @@
 locals {
   name                            = "${random_pet.example.id}${random_integer.example.result}"
   location                        = var.location
+  aks_node_pool_vm_size           = var.aks_node_pool_vm_size != "" ? var.aks_node_pool_vm_size : "Standard_DS2_v2"
+  deploy_azure_cosmosdb           = var.deploy_azure_cosmosdb == "true" ? true : false
   default_cosmosdb_account_kind   = var.deploy_azure_workload_identity == "true" ? "GlobalDocumentDB" : "MongoDB"
+  cosmosdb_failover_location      = var.deploy_azure_cosmosdb == "true" ? var.cosmosdb_failover_location : var.location
   cosmosdb_account_kind           = var.cosmosdb_account_kind != "" ? var.cosmosdb_account_kind : local.default_cosmosdb_account_kind
   deploy_azure_container_registry = var.deploy_azure_container_registry == "true" ? true : false
   deploy_azure_workload_identity  = var.deploy_azure_workload_identity == "true" ? true : false
   deploy_azure_openai             = var.deploy_azure_openai == "true" ? true : false
+  ai_location                     = var.ai_location == "" ? var.location : var.ai_location
   deploy_azure_openai_dalle_model = var.deploy_azure_openai_dalle_model == "true" ? true : false
   deploy_azure_servicebus         = var.deploy_azure_servicebus == "true" ? true : false
-  deploy_azure_cosmosdb           = var.deploy_azure_cosmosdb == "true" ? true : false
   deploy_observability_tools      = var.deploy_observability_tools == "true" ? true : false
 }

infra/terraform/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "=3.101.0"
+      version = "=3.113.0"
     }
 
     local = {
@@ -50,6 +50,10 @@ resource "random_pet" "example" {
   }
 }
 
+data "http" "ifconfig" {
+  url = "http://ifconfig.me"
+}
+
 data "azurerm_subscription" "current" {}
 data "azurerm_client_config" "current" {}
 

infra/terraform/main.tfvars.json

@@ -1,13 +1,15 @@
 {
   "resource_group_name_suffix": "${AZURE_ENV_NAME}",
   "location": "${AZURE_LOCATION}",
-  "ai_location": "${AZURE_LOCATION}",
+  "aks_node_pool_vm_size": "${AKS_NODE_POOL_VM_SIZE}",
+  "ai_location": "${AZURE_OPENAI_LOCATION}",
+  "deploy_azure_cosmosdb": "${DEPLOY_AZURE_COSMOSDB}",
   "cosmosdb_account_kind": "${AZURE_COSMOSDB_ACCOUNT_KIND}",
+  "cosmosdb_failover_location": "${AZURE_COSMOSDB_FAILOVER_LOCATION}",
   "deploy_azure_container_registry": "${DEPLOY_AZURE_CONTAINER_REGISTRY}",
   "deploy_azure_workload_identity": "${DEPLOY_AZURE_WORKLOAD_IDENTITY}",
   "deploy_azure_openai": "${DEPLOY_AZURE_OPENAI}",
   "deploy_azure_openai_dalle_model": "${DEPLOY_AZURE_OPENAI_DALL_E_MODEL}",
   "deploy_azure_servicebus": "${DEPLOY_AZURE_SERVICE_BUS}",
-  "deploy_azure_cosmosdb": "${DEPLOY_AZURE_COSMOSDB}",
   "deploy_observability_tools": "${DEPLOY_OBSERVABILITY_TOOLS}"
 }

infra/terraform/observability.tf

@@ -1,17 +1,16 @@
-resource "azurerm_monitor_workspace" "example" {
-  count               = local.deploy_observability_tools ? 1 : 0
-  name                = "amon-${local.name}"
+resource "azurerm_log_analytics_workspace" "example" {
+  name                = "alog-${local.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
+  sku                 = "PerGB2018"
+  retention_in_days   = 30
 }
 
-resource "azurerm_log_analytics_workspace" "example" {
+resource "azurerm_monitor_workspace" "example" {
   count               = local.deploy_observability_tools ? 1 : 0
-  name                = "alog-${local.name}"
+  name                = "amon-${local.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
-  sku                 = "PerGB2018"
-  retention_in_days   = 30
 }
 
 resource "azurerm_dashboard_grafana" "example" {
@@ -304,14 +303,14 @@ resource "azurerm_monitor_data_collection_rule" "example_msci" {
 
   destinations {
     log_analytics {
-      workspace_resource_id = azurerm_log_analytics_workspace.example[0].id
-      name                  = azurerm_log_analytics_workspace.example[0].name
+      workspace_resource_id = azurerm_log_analytics_workspace.example.id
+      name                  = azurerm_log_analytics_workspace.example.name
     }
   }
 
   data_flow {
     streams      = ["Microsoft-ContainerInsights-Group-Default"]
-    destinations = [azurerm_log_analytics_workspace.example[0].name]
+    destinations = [azurerm_log_analytics_workspace.example.name]
   }
 }