Merge pull request #49 from magda-io/v4-upgrade-indexing

V4 Upgrade
magda-io · Jun 14, 2024 · 51ac8cc · 51ac8cc
2 parents 4b8976e + 6ec766a
commit 51ac8cc
Show file tree

Hide file tree

Showing 7 changed files with 76 additions and 104 deletions.
diff --git a/README.md b/README.md
@@ -16,6 +16,11 @@ helm upgrade -n dga-prod magda ./chart --timeout 30000s --install -f config.yaml
 
 # Upgrade Guide:
 
+## v4.0.0
+
+- Upgrades with improvements & bug fixes
+- Upgrade search engine to opensearch v2.14.0
+
 ## v3.0.1
 
 Upgrades with improvements & bug fixes

diff --git a/cert-issuer/README.md b/cert-issuer/README.md
@@ -2,38 +2,17 @@ Before you install this chart install `cert-manager` by following instruction he
 
 https://cert-manager.io/docs/installation/kubernetes/
 
-This requires you to create a secret for the route 53 credentials in the right namespace:
+We now (since June 2024) use Google Cloud DNS and authenticated via [Workload Identity Federation for GKE](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity). 
 
-```
-kubectl create secret generic prod-route53-credentials-secret --from-literal=secret-access-key=CHANGEME --namespace cert-manager
-```
-
-For route53 it also requires you to create the right IAM policy:
-
-```
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": "route53:GetChange",
-            "Resource": "arn:aws:route53:::change/*"
-        },
-        {
-            "Effect": "Allow",
-            "Action": "route53:ChangeResourceRecordSets",
-            "Resource": "arn:aws:route53:::hostedzone/*"
-        }
-    ]
-}
-```
+To setup, please refer to the following docs:
+- https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity
+- https://cert-manager.io/docs/configuration/acme/dns01/google/
 
-Also don't forget to specify `hostedZoneID` (the hosted zone for the domain) and `accessKeyID` (the access key for the user with the above IAM policy).
 
 Then finally install:
 
 ```
-helm upgrade --install --namespace cert-issuer cert-issuer ./cert-issuer --set hostedZoneID=CHANGEME,accessKeyID=CHANGEME,acmeEmail=CHANGEME,useStaging=SHOULDIUSESTAGING
+helm upgrade --install --namespace cert-issuer cert-issuer ./cert-issuer --set acmeEmail=CHANGEME,useStaging=SHOULDIUSESTAGING
 ```
 
 ## Values
@@ -42,7 +21,4 @@ helm upgrade --install --namespace cert-issuer cert-issuer ./cert-issuer --set h
 |-----|------|---------|-------------|
 | accessKeyID | string | `nil` |  |
 | acmeEmail | string | `nil` |  |
-| hostedZoneID | string | `nil` |  |
-| secretAccessKeySecretRef.key | string | `"secret-access-key"` |  |
-| secretAccessKeySecretRef.name | string | `"prod-route53-credentials-secret"` |  |
 | useStaging | bool | `true` |  |
diff --git a/cert-issuer/templates/cluster-issuer.yaml b/cert-issuer/templates/cluster-issuer.yaml
@@ -14,10 +14,5 @@ spec:
     solvers:
     # ACME DNS-01 provider configurations
     - dns01:
-        route53:
-          region: "ap-southeast-2"
-          hostedZoneID: {{ .Values.hostedZoneID | quote }}
-          accessKeyID: {{ .Values.accessKeyID | quote }}
-          secretAccessKeySecretRef:
-            name: {{ .Values.secretAccessKeySecretRef.name | quote }}
-            key: {{ .Values.secretAccessKeySecretRef.key | quote }}
+        cloudDNS:
+          project: "terriajs"
diff --git a/chart/Chart.lock b/chart/Chart.lock
@@ -1,7 +1,7 @@
 dependencies:
 - name: magda
   repository: oci://ghcr.io/magda-io/charts
-  version: 3.0.1
+  version: 4.0.0
 - name: ingress
   repository: file://../ingress
   version: 1.1.0
@@ -26,9 +26,6 @@ dependencies:
 - name: magda-ckan-connector
   repository: oci://ghcr.io/magda-io/charts
   version: 2.1.0
-- name: magda-ckan-connector
-  repository: oci://ghcr.io/magda-io/charts
-  version: 2.1.0
 - name: magda-project-open-data-connector
   repository: oci://ghcr.io/magda-io/charts
   version: 2.0.0
@@ -107,5 +104,5 @@ dependencies:
 - name: magda-project-open-data-connector
   repository: oci://ghcr.io/magda-io/charts
   version: 2.0.0
-digest: sha256:f9f69aa6e0faa5248ad5405050ea2a6c9ec66df528bf710d0136f7c48098fe57
-generated: "2024-04-19T15:17:58.068235+10:00"
+digest: sha256:b7ecc19451d180141dd313d4e42dbd4d1bf13c916c0719708313da2047234c5e
+generated: "2024-06-14T20:47:40.565497+10:00"
diff --git a/chart/Chart.yaml b/chart/Chart.yaml
@@ -1,11 +1,11 @@
 apiVersion: v2
 name: "magda-config"
 description: "DGA magda deployment chart"
-version: "3.0.1"
+version: "4.0.0"
 kubeVersion: ">= 1.16.0-0"
 dependencies:
   - name: magda
-    version: "3.0.1"
+    version: "4.0.0"
     repository: "oci://ghcr.io/magda-io/charts"
 
   - name: ingress
@@ -72,13 +72,14 @@ dependencies:
     tags:
       - connectors
       - connector-aurin
-  - name: magda-ckan-connector
-    version: "2.1.0"
-    alias: connector-brisbane
-    repository: "oci://ghcr.io/magda-io/charts"
-    tags:
-      - connectors
-      - connector-brisbane
+  # disable for now as brisbane city council has changed their system to opendatasoft
+  # - name: magda-ckan-connector
+  #   version: "2.1.0"
+  #   alias: connector-brisbane
+  #   repository: "oci://ghcr.io/magda-io/charts"
+  #   tags:
+  #     - connectors
+  #     - connector-brisbane
   - name: magda-project-open-data-connector
     version: "2.0.0"
     alias: connector-hobart

diff --git a/chart/values.yaml b/chart/values.yaml
@@ -31,9 +31,8 @@ global:
     ## for versions. https://github.com/magda-io/magda/releases/latest.
     ## By default this is the same version as the helm chart.
     # tag: 0.0.57-0
-    ## The docker repository to get the images from - defaults to the official
-    ## data61 docker hub repo
-    repository: docker.io/data61
+    ## The docker repository to get the images from - defaults to the official repo
+    ## repository: ghcr.io/magda-io
     ## The imagePullPolicy to use for images - generally unless you're actively
     ## trying to track development this should be "IfNotPresent", otherwise if
     ## you are it can be "Always". Defaults to "IfNotPresent"
@@ -145,7 +144,10 @@ tags:
   content-db: true
   content-api: true
   correspondence-api: false  # Disabled because it needs an SMTP server to run
-  elasticsearch: true
+  elasticsearch: false
+  opensearch: true
+  opensearch-dashboards: true
+
   gateway: true
   indexer: true
   preview-map: true
@@ -154,6 +156,7 @@ tags:
   search-api: true
   session-db: true
   web-server: true
+
 
   ## Whether to use an ingress, which is necessary for HTTPS and using the Google CDN with
   ## Google Kubernetes Engine. By default this is off, and access is provided to the cluster

diff --git a/config.yaml b/config.yaml
@@ -98,7 +98,9 @@ tags:
   content-db: true
   content-api: true
   correspondence-api: true
-  elasticsearch: true
+  elasticsearch: false
+  opensearch: true
+  opensearch-dashboards: true
   gateway: true
   indexer: true
   preview-map: true
@@ -108,7 +110,6 @@ tags:
   session-db: true
   web-server: true
   connectors: true
-  opa: true
 
   ## Whether to use an ingress, which is necessary for HTTPS and using the Google CDN with
   ## Google Kubernetes Engine. By default this is off, and access is provided to the cluster
@@ -203,13 +204,15 @@ connector-aurin:
     pageSize: 100
     schedule: 0 1 */3 * *
 
-connector-brisbane:
-  config:
-    id: brisbane
-    name: Brisbane City Council
-    sourceUrl: https://www.data.brisbane.qld.gov.au/data/
-    pageSize: 100
-    schedule: 10 * */1 * *
+# Brisbane City Council has switched to a system based on "opendatasoft"
+# disable the connector for now
+# connector-brisbane:
+#   config:
+#     id: brisbane
+#     name: Brisbane City Council
+#     sourceUrl: https://www.data.brisbane.qld.gov.au/data/
+#     pageSize: 100
+#     schedule: 10 * */1 * *
 
 connector-hobart:
   config:
@@ -695,58 +698,41 @@ magda:
         limits:
           cpu: 50m
 
-    ## The elasticsearch setup - this can consist of up to three different kinds of pods -
-    ## data nodes, master nodes and client nodes.
-    elasticsearch:
-      ## Whether to use client and master nodes, or just use data nodes.
-      production: true
-
-      ## All three kinds of nodes can be customised with these settings like so:
-      # nodetype:
-        ## The java heap size - this should be half the total memory request
-        # heapSize: 3000m
-        ## Elasticsearch plugins to install
-        # pluginsInstall: "repository-gcs"
-
-      ## Settings for the data nodes - this is a statefulset that actually holds the data,
-      ## and is the only kind of node that's strictly necessary. You can have multiple
-      ## replicas of these, they will discover each other and balance shards and replicas
-      ## between them.
-      data:
-        heapSize: 3000m
-        storage: 200Gi
-        # pluginsInstall: "repository-gcs" disabled until es backups are readded
-        storageClass: "fast-ssd"
+    opensearch:
+      master:
+        enabled: true
+        javaOpts: "-Xmx512M -Xms512M"
+        replicas: 3
         resources:
           requests:
-            cpu: 500m
-            memory: 5000Mi
+            cpu: 25m
+            memory: 1000Mi
           limits:
-            cpu: 1900m
-
-      ## Settings for the client nodes - this accepts HTTP connections from search-api and
-      ## indexer and forwards them to the data nodes.
+            cpu: 250m
+            memory: 1000Mi
       client:
+        javaOpts: "-Xmx1000M -Xms1000M"
+        enabled: true
         replicas: 2
-        heapSize: 900m
-        # pluginsInstall: "repository-gcs" disabled until es backups are readded
         resources:
           requests:
             cpu: 50m
             memory: 1500Mi
           limits:
             cpu: 250m
-
-      ## Settings for the master nodes - these keep track of the data nodes as they go up
-      ## and down and keep track of what the current state of the data should be.
-      master:
-        # pluginsInstall: "repository-gcs" disabled until es backups are readded
+            memory: 1500Mi
+      data:
+        replicas: 2
+        enabled: true
+        javaOpts: "-Xmx3000M -Xms3000M"
+        storage: 200Gi
         resources:
           requests:
-            cpu: 25m
-            memory: 1000Mi
+            cpu: 500m
+            memory: 5000Mi
           limits:
-            cpu: 250m
+            cpu: 1900m
+            memory: 5000Mi
 
     ## The gateway accepts incoming connections and directs them to the appropriate api pod,
     ## or the web server pod
@@ -843,14 +829,20 @@ magda:
 
     ## Configures the service that puts datasets and organisations into elasticsearch
     indexer:
+      # image:
+      #   tag: 3.0.2
+      # appConfig:
+      #   elasticSearch:
+      #     serverUrl: http://elasticsearch:9200
+      #     esInstanceSupport: true
       elasticsearch:
         ## How many shards and replicas to use when creating a new index. Generally, more
         ## shards mean that the index will be split among more data nodes, and more replicas
         ## allows elasticsearch to copy those shards among different nodes.
         ## In general you want a number of shards equal to the lowest number of data nodes
         ## possible, and a number of replicas equal to the highest number.
         shards: 2
-        replicas: 4
+        replicas: 1
       resources:
         requests:
           cpu: 100m
@@ -998,7 +990,10 @@ magda:
     ## Configures the API that connects to ElasticSearch to perform searches.
     search-api:
       # image:
-      #   tag: 0.0.55-RC1
+      #   tag: 3.0.1
+      # appConfig:
+      #   elasticSearch:
+      #     serverUrl: http://elasticsearch:9200
       replicas: 2
       resources:
         requests: