Several fixes, upgrades and questions

.gitignore

@@ -0,0 +1,4 @@
+.envrc
+.env.toml
+.terraform.lock.*
+.terraform

k8s/02_efs.sh

@@ -37,8 +37,8 @@ fi
 
 echo "Detected Security Group ID: $securityGroupId"
 
-subnetIdZoneA=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $13 }' | sort | head -1`
-subnetIdZoneB=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $13 }' | sort | tail -1`
+subnetIdZoneA=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $15 }' | sort | head -1`
+subnetIdZoneB=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $15 }' | sort | tail -1`
 
 echo "Detected Subnet: $subnetIdZoneA"
 echo "Detected Subnet: $subnetIdZoneB"

k8s/03_ebs.sh

@@ -31,7 +31,7 @@ terraform init -backend-config=bucket=$S3_BUCKET \
 
 terraform apply -var aws_region=$AWS_REGION -var aws_availability_zone=${AWS_REGION}a -auto-approve
 
-export TG_EBS_DATADIR_VOLUME_ID="aws://`terraform output availability_zone`/`terraform output volume_id`"
+export TG_EBS_DATADIR_VOLUME_ID="aws://`terraform output --raw availability_zone`/`terraform output --raw volume_id`"
 
 popd
 

k8s/cluster.yaml

@@ -36,13 +36,13 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-${ZONE}
+    - instanceGroup: master-${AWS_REGION}
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-${ZONE}
+    - instanceGroup: master-${AWS_REGION}
       name: a
     memoryRequest: 100Mi
     name: events
@@ -118,7 +118,7 @@ metadata:
   creationTimestamp: null
   labels:
     kops.k8s.io/cluster: ${CLUSTER_NAME}
-  name: master-${ZONE}
+  name: master-${AWS_REGION}
 spec:
   additionalUserData:
   - name: myscript.sh

k8s/delete_efs.sh

@@ -29,8 +29,8 @@ fi
 
 echo "Detected Security Group ID: $securityGroupId"
 
-subnetIdZoneA=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $13 }' | sort | head -1`
-subnetIdZoneB=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $13 }' | sort | tail -1`
+subnetIdZoneA=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $15 }' | sort | head -1`
+subnetIdZoneB=`aws ec2 describe-subnets --region=$AWS_REGION --output text | awk '/'$vpcId'/ { print $15 }' | sort | tail -1`
 
 echo "Detected Subnet: $subnetIdZoneA"
 echo "Detected Subnet: $subnetIdZoneB"

k8s/ebs-terraform/ebs.tf

@@ -1,6 +1,12 @@
-provider "aws" {
-  region  = var.aws_region
-  version = "~> 2.50"
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 3.0"
+    }
+  }
+
+  required_version = ">= 0.14"
 }
 
 # EBS for Testground daemon datadir

k8s/ebs-terraform/variables.tf

@@ -3,10 +3,11 @@ variable "aws_region" {}
 variable "aws_availability_zone" {}
 
 variable "default_tags" {
-  type = "map"
+  type = map
 
+  # this should be dynamic
   default = {
     Name              = "taas-daemon-datadir-volume"
-    KubernetesCluster = "anton-kops.k8s.local"
+    KubernetesCluster = "testground.k8s.local"
   }
 }

k8s/efs-terraform/efs.tf

@@ -1,6 +1,16 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 3.0"
+    }
+  }
+
+  required_version = ">= 0.14"
+}
+
 provider "aws" {
   region  = var.aws_region
-  version = "~> 2.50"
 }
 
 # EFS for Testground outputs

k8s/testground-daemon/config-map-env-toml.yml

@@ -6,7 +6,7 @@ metadata:
 data:
   .env.toml: |
     ["aws"]
-    region = "eu-west-2"
+    region = "us-east-1"
 
     [runners."cluster:k8s"]
     run_timeout_min             = 15

k8s/testground-daemon/deployment.yml

@@ -20,20 +20,21 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       nodeSelector:
         testground.node.role.infra: "true"
-      initContainers:
-      - name: iproute-add
-        image: busybox:1.31.1
-        securityContext:
-          privileged: true
-        command:
-        - sh
-        - -ac
-        - >
-          while [ "$GW" = "" ]; do export GW=$(ip route | grep cni0 | awk '{print $7}'); echo "Got GW: $GW"; sleep 5; done;
-          echo $GW &&
-          ip route &&
-          ip route add 100.64.0.0/16 via $GW &&
-          ip route || true;
+      # Why this? it doesn't work
+      # initContainers:
+      # - name: iproute-add
+      #   image: busybox:1.31.1
+      #   securityContext:
+      #     privileged: true
+      #   command:
+      #   - sh
+      #   - -ac
+      #   - >
+      #     while [ "$GW" = "" ]; do export GW=$(ip route | grep cni0 | awk '{print $7}'); echo "Got GW: $GW"; sleep 5; done;
+      #     echo $GW &&
+      #     ip route &&
+      #     ip route add 100.64.0.0/16 via $GW &&
+      #     ip route || true;
       containers:
       - name: goproxy
         image: iptestground/goproxy:2.0.2
@@ -44,8 +45,8 @@ spec:
           - name: envtoml
             mountPath: /root/testground/.env.toml
             subPath: .env.toml
-          - name: efs-pvc
-            mountPath: "/go"
+          # - name: efs-pvc
+          #   mountPath: "/go"
         resources:
           requests:
             memory: 512Mi
@@ -66,8 +67,8 @@ spec:
         volumeMounts:
           - name: daemon-datadir
             mountPath: "/root/testground/"
-          - name: efs-pvc
-            mountPath: "/efs"
+          # - name: efs-pvc
+          #   mountPath: "/efs"
           - name: dockersock
             mountPath: "/var/run/docker.sock"
           - name: envtoml
@@ -80,9 +81,9 @@ spec:
           limits:
             memory: 2048Mi
       volumes:
-        - name: efs-pvc
-          persistentVolumeClaim:
-            claimName: efs
+        # - name: efs-pvc
+        #   persistentVolumeClaim:
+        #     claimName: efs
         - name: daemon-datadir
           persistentVolumeClaim:
             claimName: testground-daemon-datadir-pvc
@@ -92,3 +93,17 @@ spec:
         - name: envtoml
           configMap:
             name: env-toml-cfg
+
+# missing policy
+# {
+#         "Sid": "kopsK8sEC2NodePerms",
+#         "Effect": "Allow",
+#         "Action": [
+#             "ec2:DescribeInstances",
+#             "ec2:AttachVolume",
+#             "ec2:DetachVolume"
+#         ],
+#         "Resource": [
+#             "*"
+#         ]
+#     },

k8s/testground-infra/requirements.lock

@@ -1,9 +1,9 @@
 dependencies:
 - name: redis
   repository: https://charts.bitnami.com/bitnami
-  version: 10.6.6
+  version: 15.6.4
 - name: testground-dashboards
   repository: ""
   version: 0.0.*
-digest: sha256:2a9e152a6d30e2aca28f44c2e5131469c014b505888c5e3a6eeac2af94ba0915
-generated: "2020-04-14T14:57:04.250358476-07:00"
+digest: sha256:2879642640ddf1ee3f5e19960dbac10dcdb9760a04a1b5dc6941e59dff566aff
+generated: "2021-12-14T12:48:36.199922+01:00"

k8s/testground-infra/values.yaml

@@ -11,18 +11,23 @@
 redis:
   metrics:
     enabled: true
-    serviceMonitor:
-      enabled: true
-      namespace: default
+    # This doesn work: Error: INSTALLATION FAILED: unable to build kubernetes objects from release manifest: unable to recognize "": no matches for kind "ServiceMonitor"
+    # This needs to be installed with: helm install testground-infra .
+    # serviceMonitor:
+    #   enabled: true
+    #   namespace: default
     resources:
       requests:
         memory: 256Mi
         cpu: 200m
       limits:
         memory: 256Mi
+  architecture: standalone
   cluster:
     enabled: false
   usePassword: false
+  auth:
+    enabled: false
   securityContext:
     sysctls:
       - name: net.core.somaxconn