-
Notifications
You must be signed in to change notification settings - Fork 18
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Migration to kubernetes is rather fragile, with:
1. tasks running in `kubectl exec` or as `pod`.
2. the uyuni helm chart being deployed multiple times
3. `hostPath` mounts are used everywhere for the scripts to run and
data to read and force the script to run on the cluster node.
Here are the solutions to those problems:
1. Each step will run as a Job and those won't be deleted automatically
for the user to access their logs after.
2. Stop using the helm chart and deploy the resources when we need them.
This will allow more control of what runs when and reduces the number
of useless starts of the giant container.
Postgresql DB upgrade will disable SSL temporarily in the
postgresql.conf in order to not rely on the SSL certificates to be
migrated.
3. The scripts to run for each step will be passed directly as `sh -c`
parameter to the generated Jobs.
The migration data are be stored in a special volume and not on the
host.
As a collateral, SSH agent can no longer be used as that would require
running on a cluster node again. At the moment the user is required to
create a ConfigMap to stored the SSH config and known_hosts and a Secret
for a passwordless SSH key.
The PersistentVolumes are not destroyed after the end of the first job
and are then reused by the next ones and the final deployment.
Using Kubernetes API modules also helps for code reuse with a future
operator. However for this `kubectl` should be dropped completely in
favor of the official go client, but that could come in a future PR.
As a side effect kubernetes interactions would be easier to mock for
unit tests.
Note that the old postgresql database cannot be moved to a separate
PersistentVolumes. As we run a `db_upgrade --link`, the old database is
linked by the new one and cannot be disposed of.- Loading branch information
Showing
40 changed files
with
2,318 additions
and
283 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,157 @@ | ||
| // SPDX-FileCopyrightText: 2024 SUSE LLC | ||
| // | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build !nok8s | ||
|
|
||
| package kubernetes | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| "path" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/rs/zerolog" | ||
| "github.com/rs/zerolog/log" | ||
| "github.com/uyuni-project/uyuni-tools/shared/kubernetes" | ||
| . "github.com/uyuni-project/uyuni-tools/shared/l10n" | ||
| "github.com/uyuni-project/uyuni-tools/shared/types" | ||
| "github.com/uyuni-project/uyuni-tools/shared/utils" | ||
| "gopkg.in/yaml.v2" | ||
| core "k8s.io/api/core/v1" | ||
| meta "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| ) | ||
|
|
||
| // MigrationData represents the files and data extracted from the migration sync phase. | ||
| type MigrationData struct { | ||
| CaKey string | ||
| CaCert string | ||
| Data *utils.InspectResult | ||
| ServerCert string | ||
| ServerKey string | ||
| } | ||
|
|
||
| func extractMigrationData(namespace string, image string, volume types.VolumeMount) (*MigrationData, error) { | ||
| // Read the file from the volume from a container into stdout | ||
| mounts := kubernetes.ConvertVolumeMounts([]types.VolumeMount{volume}) | ||
| volumes := kubernetes.CreateVolumes([]types.VolumeMount{volume}) | ||
|
|
||
| podName := "uyuni-data-extractor" | ||
|
|
||
| // Use a pod here since this is a very simple task reading out a file from a volume | ||
| pod := core.Pod{ | ||
| TypeMeta: meta.TypeMeta{Kind: "Pod", APIVersion: "v1"}, | ||
| ObjectMeta: meta.ObjectMeta{Name: podName, Namespace: namespace}, | ||
| Spec: core.PodSpec{ | ||
| Containers: []core.Container{ | ||
| { | ||
| Name: "extractor", | ||
| Image: image, | ||
| ImagePullPolicy: core.PullIfNotPresent, | ||
| Command: []string{ | ||
| "sh", "-c", | ||
| "for f in /var/lib/uyuni-tools/*; do echo \"`basename $f`: |2\"; cat $f | sed 's/^/ /'; done", | ||
| }, | ||
| VolumeMounts: mounts, | ||
| }, | ||
| }, | ||
| Volumes: volumes, | ||
| RestartPolicy: core.RestartPolicyNever, | ||
| }, | ||
| } | ||
|
|
||
| tempDir, err := utils.TempDir() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer os.RemoveAll(tempDir) | ||
|
|
||
| // Run the pod | ||
| extractorPodPath := path.Join(tempDir, "extractor-pod.yaml") | ||
| if err := kubernetes.YamlFile([]runtime.Object{&pod}, extractorPodPath); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if err := utils.RunCmd("kubectl", "apply", "-f", extractorPodPath); err != nil { | ||
| return nil, utils.Errorf(err, L("failed to run the migration data extractor pod")) | ||
| } | ||
| if err := kubernetes.Apply( | ||
| []runtime.Object{&pod}, L("failed to run the migration data extractor pod"), | ||
| ); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if err := waitForPod(namespace, podName, 60); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| data, err := utils.RunCmdOutput(zerolog.DebugLevel, "kubectl", "logs", "-n", namespace, podName) | ||
| if err != nil { | ||
| return nil, utils.Errorf(err, L("failed to get the migration data extractor pod logs")) | ||
| } | ||
|
|
||
| defer func() { | ||
| if err := utils.RunCmd("kubectl", "delete", "pod", "-n", namespace, podName); err != nil { | ||
| log.Err(err).Msgf(L("failed to delete the uyuni-data-extractor pod")) | ||
| } | ||
| }() | ||
|
|
||
| // Parse the content | ||
| files := make(map[string]string) | ||
| if err := yaml.Unmarshal(data, &files); err != nil { | ||
| return nil, utils.Errorf(err, L("failed to parse data extractor pod output")) | ||
| } | ||
|
|
||
| var result MigrationData | ||
| for file, content := range files { | ||
| if file == "RHN-ORG-PRIVATE-SSL-KEY" { | ||
| result.CaKey = content | ||
| } else if file == "RHN-ORG-TRUSTED-SSL-CERT" { | ||
| result.CaCert = content | ||
| } else if file == "spacewalk.crt" { | ||
| result.ServerCert = content | ||
| } else if file == "spacewalk.key" { | ||
| result.ServerKey = content | ||
| } else if file == "data" { | ||
| parsedData, err := utils.ReadInspectDataString[utils.InspectResult]([]byte(content)) | ||
| if err != nil { | ||
| return nil, utils.Errorf(err, L("failed to parse migration data file")) | ||
| } | ||
| result.Data = parsedData | ||
| } | ||
| } | ||
|
|
||
| if result.Data == nil { | ||
| return nil, errors.New(L("found no data file after migration")) | ||
| } | ||
|
|
||
| return &result, nil | ||
| } | ||
|
|
||
| func waitForPod(namespace string, pod string, timeout int) error { | ||
| for i := 0; ; i++ { | ||
| out, err := utils.RunCmdOutput( | ||
| zerolog.DebugLevel, "kubectl", "get", "pod", "-n", namespace, pod, | ||
| "-o", "jsonpath={.status.containerStatuses[0].state.terminated.reason}", | ||
| ) | ||
| if err != nil { | ||
| return utils.Errorf(err, L("failed to get %s pod status"), pod) | ||
| } | ||
| status := strings.TrimSpace(string(out)) | ||
| if status != "" { | ||
| if status == "Completed" { | ||
| return nil | ||
| } | ||
| return fmt.Errorf(L("%[1]s pod failed with status %[2]s"), pod, status) | ||
| } | ||
|
|
||
| if timeout > 0 && i == timeout { | ||
| return fmt.Errorf(L("%[1]s pod failed to complete within %[2]d seconds"), pod, timeout) | ||
| } | ||
| time.Sleep(1 * time.Second) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| // SPDX-FileCopyrightText: 2024 SUSE LLC | ||
| // | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build !nok8s | ||
|
|
||
| package kubernetes | ||
|
|
||
| import ( | ||
| "github.com/rs/zerolog/log" | ||
| . "github.com/uyuni-project/uyuni-tools/shared/l10n" | ||
| "github.com/uyuni-project/uyuni-tools/shared/types" | ||
|
|
||
| "github.com/uyuni-project/uyuni-tools/mgradm/shared/templates" | ||
| "github.com/uyuni-project/uyuni-tools/shared/kubernetes" | ||
| batch "k8s.io/api/batch/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| ) | ||
|
|
||
| const dbFinalizeJobName = "uyuni-db-finalize" | ||
|
|
||
| func startDbFinalizeJob( | ||
| namespace string, | ||
| serverImage string, | ||
| pullPolicy string, | ||
| schemaUpdateRequired bool, | ||
| migration bool, | ||
| ) error { | ||
| log.Info().Msg(L("Running database finalization, this could be long depending on the size of the database…")) | ||
| job, err := getDbFinalizeJob(namespace, serverImage, pullPolicy, schemaUpdateRequired, migration) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return kubernetes.Apply([]runtime.Object{job}, L("failed to run the database finalization job")) | ||
| } | ||
|
|
||
| func getDbFinalizeJob( | ||
| namespace string, | ||
| image string, | ||
| pullPolicy string, | ||
| schemaUpdateRequired bool, | ||
| migration bool, | ||
| ) (*batch.Job, error) { | ||
| mounts := []types.VolumeMount{ | ||
| {MountPath: "/var/lib/pgsql", Name: "var-pgsql"}, | ||
| {MountPath: "/etc/rhn", Name: "etc-rhn"}, | ||
| } | ||
|
|
||
| // Prepare the script | ||
| scriptData := templates.FinalizePostgresTemplateData{ | ||
| RunAutotune: true, | ||
| RunReindex: true, | ||
| RunSchemaUpdate: schemaUpdateRequired, | ||
| Migration: migration, | ||
| Kubernetes: true, | ||
| } | ||
|
|
||
| return kubernetes.GetScriptJob(namespace, dbFinalizeJobName, image, pullPolicy, mounts, scriptData) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| // SPDX-FileCopyrightText: 2024 SUSE LLC | ||
| // | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| //go:build !nok8s | ||
|
|
||
| package kubernetes | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/rs/zerolog/log" | ||
| . "github.com/uyuni-project/uyuni-tools/shared/l10n" | ||
|
|
||
| "github.com/uyuni-project/uyuni-tools/mgradm/shared/templates" | ||
| "github.com/uyuni-project/uyuni-tools/shared/kubernetes" | ||
| "github.com/uyuni-project/uyuni-tools/shared/types" | ||
| "github.com/uyuni-project/uyuni-tools/shared/utils" | ||
| batch "k8s.io/api/batch/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| ) | ||
|
|
||
| const dbUpgradeJobName = "uyuni-db-upgrade" | ||
|
|
||
| func startDbUpgradeJob( | ||
| namespace string, | ||
| registry string, | ||
| image types.ImageFlags, | ||
| migrationImage types.ImageFlags, | ||
| oldPgsql string, | ||
| newPgsql string, | ||
| ) error { | ||
| log.Info().Msgf(L("Upgrading PostgreSQL database from %[1]s to %[2]s…"), oldPgsql, newPgsql) | ||
|
|
||
| var migrationImageUrl string | ||
| var err error | ||
| if migrationImage.Name == "" { | ||
| imageName := fmt.Sprintf("-migration-%s-%s", oldPgsql, newPgsql) | ||
| migrationImageUrl, err = utils.ComputeImage(registry, image.Tag, image, imageName) | ||
| } else { | ||
| migrationImageUrl, err = utils.ComputeImage(registry, image.Tag, migrationImage) | ||
| } | ||
| if err != nil { | ||
| return utils.Errorf(err, L("failed to compute image URL")) | ||
| } | ||
|
|
||
| log.Info().Msgf(L("Using database upgrade image %s"), migrationImageUrl) | ||
|
|
||
| job, err := getDbUpgradeJob(namespace, migrationImageUrl, image.PullPolicy, oldPgsql, newPgsql) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return kubernetes.Apply([]runtime.Object{job}, L("failed to run the database upgrade job")) | ||
| } | ||
|
|
||
| func getDbUpgradeJob( | ||
| namespace string, | ||
| image string, | ||
| pullPolicy string, | ||
| oldPgsql string, | ||
| newPgsql string, | ||
| ) (*batch.Job, error) { | ||
| mounts := []types.VolumeMount{ | ||
| {MountPath: "/var/lib/pgsql", Name: "var-pgsql"}, | ||
| } | ||
|
|
||
| // Prepare the script | ||
| scriptData := templates.PostgreSQLVersionUpgradeTemplateData{ | ||
| OldVersion: oldPgsql, | ||
| NewVersion: newPgsql, | ||
| } | ||
|
|
||
| return kubernetes.GetScriptJob(namespace, dbUpgradeJobName, image, pullPolicy, mounts, scriptData) | ||
| } |
Oops, something went wrong.