Better support for dev takeover (#75)

Per #65 we want to make it easy to use a RepoConfig and deploy from a
branch

* Add repo mappings to RepoConfig . This is a set of repo URLs to
rewrite the source branches in manifest sync
  this allows the application to be deployed from a branch.
  
* Also add a Pause option which can be used for a takeover.

* Fix #75 when matching globs against paths in a tarball we need to
strip any leading "/" in the path.

Add a version command  to hydros
* Update goreleaser to start setting the version

.goreleaser.yaml

@@ -17,8 +17,8 @@ builds:
 
     # Custom ldflags templates.
     # Default is `-s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.builtBy=goreleaser`.
-    #ldflags:
-      #- "-s -w -X github.com/jlewi/roboweb/kubedr/cmd/commands.version={{.Version}} -X github.com/jlewi/roboweb/kubedr/cmd/commands.commit={{.Commit}} -X github.com/jlewi/roboweb/kubedr/cmd/commands.date={{.Date}} -X github.com/jlewi/roboweb/kubedr/cmd/commands.builtBy=goreleaser"
+    ldflags:
+      - "-s -w -X github.com/jlewi/hydros/cmd/commands.version={{.Version}} -X github.com/jlewi/hydro/cmd/commands.commit={{.Commit}} -X github.com/jlewi/hydros/cmd/commands.date={{.Date}} -X github.com/jlewi/hydros/cmd/commands.builtBy=goreleaser"
 archives:
   # https://goreleaser.com/customization/archive/?h=archives
   - id: "binary"

Dockerfile

@@ -2,11 +2,15 @@ ARG BUILD_IMAGE=golang:1.19
 ARG RUNTIME_IMAGE=cgr.dev/chainguard/static:latest
 FROM ${BUILD_IMAGE} as builder
 
+# Build Args need to be after the FROM stage otherwise they don't get passed through to the RUN statment
+ARG VERSION=unknown
+ARG DATE=unknown
+ARG COMMIT=unknown
+
 WORKDIR /workspace/
 
 COPY . /workspace
 
-
 ## Build
 # N.B Disabling CGO can potentially cause problems on MacOSX and darwin builds because some networking requires
 # it https://github.com/golang/go/issues/16345. We use to build with CGO_ENABLED=- to disable it at Primer
@@ -15,7 +19,14 @@ COPY . /workspace
 # environment.
 #
 # TODO(jeremy): We should be setting version information here
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o hydros cmd/main.go
+# The LDFLAG can't be specified multiple times so we use an environment variable to build it up over multiple lines
+RUN LDFLAGS="-s -w -X github.com/jlewi/hydros/cmd/commands.version=${VERSION}" && \
+    LDFLAGS="${LDFLAGS} -X github.com/jlewi/hydros/cmd/commands.commit=${COMMIT}" && \
+    LDFLAGS="${LDFLAGS} -X github.com/jlewi/hydros/cmd/commands.date=${DATE}" && \
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on \
+    go build \
+    -ldflags "${LDFLAGS}" \
+    -a -o hydros cmd/main.go
 
 # TODO(jeremy): This won't be able to run Syncer until we update syncer to use GoGit and get rid of shelling
 # out to other tools.

api/v1alpha1/repo.go

@@ -2,6 +2,7 @@ package v1alpha1
 
 import (
 	"strings"
+	"time"
 )
 
 var (
@@ -13,6 +14,7 @@ var (
 )
 
 // RepoConfig specifies a repository that should be checked out and periodically sync'd.
+// TODO(jeremy): RepoConfig is a terrible name.
 type RepoConfig struct {
 	APIVersion string   `yaml:"apiVersion"`
 	Kind       string   `yaml:"kind"`
@@ -35,6 +37,23 @@ type RepoSpec struct {
 	// Selectors is one or more labelselectors used to filter resources
 	// to sync. A resource must match one of the label selectors in order to be included
 	Selectors []LabelSelector `yaml:"selectors,omitempty"`
+
+	// Pause causes the controller to pause regular ManifestSync hydration for the specified amount of type
+	// This causes the manifests to be hydrated in a takeover configuration
+	Pause string `yaml:"pause,omitempty"`
+
+	// RepoMappings is a list of one or more mappings from one repository to another repository(or branch).
+	// This is used to rewrite the sourceRepositories in ManifestSync resources in order to hydrate from a
+	// branch.
+	RepoMappings []RepoMapping `yaml:"repoMappings,omitempty"`
+}
+
+// RepoMapping is a mapping from a repository to a directory
+type RepoMapping struct {
+	// Input is the input URI of the repository to use.
+	Input string `yaml:"input"`
+	// Output is the output repostiroy to use.
+	Output string `yaml:"output"`
 }
 
 // IsValid returns true if the config is valid.
@@ -67,6 +86,13 @@ func (c *RepoConfig) IsValid() (string, bool) {
 		errors = append(errors, "At least one selector must be specified.")
 	}
 
+	if c.Spec.Pause != "" {
+		_, err := time.ParseDuration(c.Spec.Pause)
+		if err != nil {
+			errors = append(errors, "Pause is not a valid duration")
+		}
+	}
+
 	if len(errors) > 0 {
 		return "RepoConfig is invalid. " + strings.Join(errors, ". "), false
 	}

cmd/commands/apply.go

@@ -42,7 +42,7 @@ func NewApplyCmd() *cobra.Command {
 				log.Info("apply takes at least one argument which should be the file or directory YAML to apply.")
 				return
 			}
-
+			logVersion()
 			paths := []string{}
 
 			for _, resourcePath := range args {

cmd/commands/takeover.go

@@ -6,8 +6,6 @@ import (
 	"path/filepath"
 	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"github.com/go-logr/zapr"
 	"github.com/jlewi/hydros/api/v1alpha1"
 	"github.com/jlewi/hydros/pkg/files"
@@ -97,20 +95,11 @@ func TakeOver(args *TakeOverArgs) error {
 		return errors.Wrapf(err, "Failed to decode ManifestSync from file %v", manifestPath)
 	}
 
-	tEnd := time.Now().Add(args.Pause)
-
-	k8sTime := metav1.NewTime(tEnd)
-	v, err := k8sTime.MarshalJSON()
-	if err != nil {
-		return errors.Wrapf(err, "Failed to marshal time %v", tEnd)
-	}
-	m.Metadata.Annotations = map[string]string{
-		// We need to mark it as a takeover otherwise we won't override pauses.
-		v1alpha1.TakeoverAnnotation: "true",
-		v1alpha1.PauseAnnotation:    string(v),
+	if err := gitops.SetTakeOverAnnotations(m, args.Pause); err != nil {
+		return errors.Wrapf(err, "Failed to set takeover annotations")
 	}
 
-	log.Info("Pausing automatic syncs", "pauseUntil", string(v))
+	log.Info("Pausing automatic syncs")
 	syncer, err := gitops.NewSyncer(m, manager, gitops.SyncWithWorkDir(args.WorkDir), gitops.SyncWithLogger(log))
 	if err != nil {
 		return err

cmd/commands/version.go

@@ -0,0 +1,37 @@
+package commands
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/go-logr/zapr"
+	"go.uber.org/zap"
+
+	"github.com/spf13/cobra"
+)
+
+// N.B these will get set by goreleaser
+// https://goreleaser.com/cookbooks/using-main.version/?h=using+main.version
+var (
+	version = "dev"
+	commit  = "none"
+	date    = "unknown"
+	builtBy = "unknown"
+)
+
+func NewVersionCmd(name string, w io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "version",
+		Short:   "Return version",
+		Example: fmt.Sprintf("%s  version", name),
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Fprintf(w, "%s %s, commit %s, built at %s by %s\n", name, version, commit, date, builtBy)
+		},
+	}
+	return cmd
+}
+
+func logVersion() {
+	log := zapr.NewLogger(zap.L())
+	log.Info("binary version", "version", version, "commit", commit, "date", date, "builtBy", builtBy)
+}

cmd/main.go

@@ -122,6 +122,7 @@ func init() {
 	rootCmd.AddCommand(commands.NewTakeOverCmd())
 	rootCmd.AddCommand(commands.NewHydrosServerCmd())
 	rootCmd.AddCommand(commands.NewCloneCmd())
+	rootCmd.AddCommand(commands.NewVersionCmd("hydros", os.Stdout))
 
 	rootCmd.PersistentFlags().BoolVar(&gOptions.devLogger, "dev-logger", false, "If true configure the logger for development; i.e. non-json output")
 	rootCmd.PersistentFlags().StringVarP(&gOptions.level, "log-level", "", "info", "Log level: error info or debug")

pkg/github/uris.go

@@ -0,0 +1,23 @@
+package github
+
+import (
+	"net/url"
+
+	"github.com/jlewi/hydros/api/v1alpha1"
+)
+
+// GitHubRepoToURI converts a GitHubRepo to a URI in the gogetter form.
+// It assumes the protocol is https.
+func GitHubRepoToURI(repo v1alpha1.GitHubRepo) url.URL {
+	u := url.URL{
+		Scheme: "https",
+		Host:   "github.com",
+		Path:   "/" + repo.Org + "/" + repo.Repo + ".git",
+	}
+
+	if repo.Branch != "" {
+		u.RawQuery = "ref=" + repo.Branch
+	}
+
+	return u
+}

pkg/github/uris_test.go

@@ -0,0 +1,46 @@
+package github
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/jlewi/hydros/api/v1alpha1"
+)
+
+func Test_GitHubRepoToURL(t *testing.T) {
+	type testCase struct {
+		name string
+		repo v1alpha1.GitHubRepo
+
+		// Compare the actual URL to one parsed from a string
+		expected string
+	}
+
+	testCases := []testCase{
+		{
+			name: "main",
+			repo: v1alpha1.GitHubRepo{
+				Org:    "jlewi",
+				Repo:   "hydros",
+				Branch: "jlewi/cicd",
+			},
+			expected: "https://github.com/jlewi/hydros.git?ref=jlewi/cicd",
+		},
+	}
+
+	for _, c := range testCases {
+		t.Run(c.name, func(t *testing.T) {
+			actual := GitHubRepoToURI(c.repo)
+
+			expectedU, err := url.Parse(c.expected)
+			if err != nil {
+				t.Fatalf("Error parsing expected URL %v", err)
+			}
+
+			if d := cmp.Diff(expectedU, &actual); d != "" {
+				t.Errorf("Unexpected diff:\n%v", d)
+			}
+		})
+	}
+}

pkg/gitops/repocontroller.go

@@ -292,6 +292,24 @@ func (c *RepoController) applyManifest(ctx context.Context, r *resource) error {
 		return errors.Wrapf(err, "Error decoding manifest")
 	}
 
+	// Rewrite the source repo if necessary
+	if err := rewriteRepos(ctx, manifest, c.config.Spec.RepoMappings); err != nil {
+		return err
+	}
+
+	pause := c.config.Spec.Pause
+	if pause != "" {
+		pauseDur, err := time.ParseDuration(pause)
+		if err != nil {
+			return errors.Wrapf(err, "Error parsing pause duration %v", pause)
+		}
+
+		if err := SetTakeOverAnnotations(manifest, pauseDur); err != nil {
+			return errors.Wrapf(err, "Failed to set takeover annotations")
+		}
+		log.Info("Pausing automatic syncs; doing a takeover")
+	}
+
 	// Create a workDir for this syncer
 	// Each ManifestSync should get its own workDir
 	// This should be stable names so that they get reused on each sync

pkg/gitops/takeover.go

@@ -0,0 +1,77 @@
+package gitops
+
+import (
+	"context"
+	"net/url"
+	"time"
+
+	"github.com/jlewi/hydros/api/v1alpha1"
+	gh "github.com/jlewi/hydros/pkg/github"
+	"github.com/jlewi/hydros/pkg/github/ghrepo"
+	"github.com/jlewi/hydros/pkg/util"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// rewriteRepos rewrites the repos in the manifest to the new repos if necessary.
+func rewriteRepos(ctx context.Context, m *v1alpha1.ManifestSync, mappings []v1alpha1.RepoMapping) error {
+	log := util.LogFromContext(ctx)
+
+	if m == nil {
+		return errors.New("Manifest is nil")
+	}
+
+	if mappings == nil {
+		return nil
+	}
+
+	srcRepoURL := gh.GitHubRepoToURI(m.Spec.SourceRepo)
+
+	for _, mapping := range mappings {
+		if srcRepoURL.String() != mapping.Input {
+			continue
+		}
+
+		log.Info("Rewriting source repo", "old", srcRepoURL.String(), "new", mapping.Output, "manifestSync.Name", m.Metadata.Name)
+
+		u, err := url.Parse(mapping.Output)
+		if err != nil {
+			return errors.Wrapf(err, "Could not parse URL %v", mapping.Output)
+		}
+
+		r, err := ghrepo.FromURL(u)
+		if err != nil {
+			return errors.Wrapf(err, "Could not parse URL %v", srcRepoURL.String())
+		}
+
+		// ref parameter specifies the reference to checkout
+		// https://github.com/hashicorp/go-getter#protocol-specific-options
+		branch := u.Query().Get("ref")
+		if branch == "" {
+			return errors.Wrapf(err, "Branch is not specified in URL %v; it should be specified as a query argument e.g. ?ref=main", mapping.Output)
+		}
+		m.Spec.SourceRepo.Org = r.RepoOwner()
+		m.Spec.SourceRepo.Repo = r.RepoName()
+		m.Spec.SourceRepo.Branch = branch
+		return nil
+	}
+	return nil
+}
+
+// SetTakeOverAnnotations sets the takeover annotations on the manifest.
+func SetTakeOverAnnotations(m *v1alpha1.ManifestSync, pause time.Duration) error {
+	tEnd := time.Now().Add(pause)
+
+	k8sTime := metav1.NewTime(tEnd)
+	v, err := k8sTime.MarshalJSON()
+	if err != nil {
+		return errors.Wrapf(err, "Failed to marshal time %v", tEnd)
+	}
+	m.Metadata.Annotations = map[string]string{
+		// We need to mark it as a takeover otherwise we won't override pauses.
+		v1alpha1.TakeoverAnnotation: "true",
+		v1alpha1.PauseAnnotation:    string(v),
+	}
+
+	return nil
+}