From 0b6e96197f6ecd2c4dde7af4a68fd7c67d2b274e Mon Sep 17 00:00:00 2001 From: Barnaby Keene Date: Sun, 5 Apr 2020 19:47:09 +0100 Subject: [PATCH 1/3] Top level error handling (#56) * add a goroutine stack trace when exiting in debug mode * added error logging to the reconfigure daemon * added a couple of additional debug log entries to significant events * improved the top-level error handling so any point of failure ends the application --- main.go | 15 +++++++++++++ reconfigurer/git.go | 3 +++ service/service.go | 51 ++++++++++++++++++++++++--------------------- watcher/git.go | 2 ++ 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index e88cc11..988c6fd 100644 --- a/main.go +++ b/main.go @@ -2,8 +2,10 @@ package main import ( "context" + "log" "os" "os/signal" + "runtime" "time" _ "github.com/joho/godotenv/autoload" @@ -114,6 +116,19 @@ this repository has new commits, Pico will automatically reconfigure.`, }, } + if os.Getenv("DEBUG") != "" { + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, os.Interrupt) + buf := make([]byte, 1<<20) + for { + <-sigs + stacklen := runtime.Stack(buf, true) + log.Printf("\nPrinting goroutine stack trace because `DEBUG` was set.\n%s\n", buf[:stacklen]) + } + }() + } + err := app.Run(os.Args) if err != nil { zap.L().Fatal("exit", zap.Error(err)) diff --git a/reconfigurer/git.go b/reconfigurer/git.go index b8ef296..85ae584 100644 --- a/reconfigurer/git.go +++ b/reconfigurer/git.go @@ -88,6 +88,9 @@ func (p *GitProvider) reconfigure(w watcher.Watcher) (err error) { state.Env["HOSTNAME"] = p.hostname } + zap.L().Debug("setting state for watcher", + zap.Any("new_state", state)) + return w.SetState(state) } diff --git a/service/service.go b/service/service.go index 5fa0504..4fa0dca 100644 --- a/service/service.go +++ b/service/service.go @@ -10,7 +10,6 @@ import ( "github.com/eapache/go-resiliency/retrier" "github.com/pkg/errors" "go.uber.org/zap" - "golang.org/x/sync/errgroup" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/http" "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" @@ -110,39 +109,43 @@ func Initialise(c Config) (app *App, err error) { // Start launches the app and blocks until fatal error func (app *App) Start(ctx context.Context) error { - g, ctx := errgroup.WithContext(ctx) - - zap.L().Debug("starting service daemon") - - // TODO: Replace this errgroup with a more resilient solution. - // Not all of these tasks fail in the same way. Some don't fail at all. - // This needs to be rewritten to be more considerate of different failure - // states and potentially retry in some circumstances. Pico should be the - // kind of service that barely goes down, only when absolutely necessary. + errs := make(chan error) ce := executor.NewCommandExecutor(app.secrets, app.config.PassEnvironment, app.config.VaultConfig, "GLOBAL_") - g.Go(func() error { + go func() { ce.Subscribe(app.bus) - return nil - }) + }() - // TODO: gw can fail when setting up the gitwatch instance, it should retry. gw := app.watcher.(*watcher.GitWatcher) - g.Go(gw.Start) + go func() { + errs <- errors.Wrap(gw.Start(), "git watcher terminated fatally") + }() - // TODO: reconfigurer can also fail when setting up gitwatch. - g.Go(func() error { - return app.reconfigurer.Configure(app.watcher) - }) + go func() { + errs <- errors.Wrap(app.reconfigurer.Configure(app.watcher), "git watcher terminated fatally") + }() if s, ok := app.secrets.(*vault.VaultSecrets); ok { - g.Go(func() error { - return retrier.New(retrier.ConstantBackoff(3, 100*time.Millisecond), nil). - RunCtx(ctx, s.Renew) - }) + go func() { + errs <- errors.Wrap(retrier.New(retrier.ConstantBackoff(3, 100*time.Millisecond), nil).RunCtx(ctx, s.Renew), "git watcher terminated fatally") + }() } - return g.Wait() + handle := func() error { + select { + case err := <-errs: + return err + case <-ctx.Done(): + return context.Canceled + } + } + + zap.L().Debug("starting service main loop") + for { + if err := handle(); err != nil { + return err + } + } } func getAuthMethod(c Config, secretConfig map[string]string) (transport.AuthMethod, error) { diff --git a/watcher/git.go b/watcher/git.go index 0da26a7..7c39631 100644 --- a/watcher/git.go +++ b/watcher/git.go @@ -62,6 +62,8 @@ func NewGitWatcher( // Start runs the watcher loop and blocks until a fatal error occurs func (w *GitWatcher) Start() error { + zap.L().Debug("git watcher initialising, waiting for first state to be set") + // wait for the first config event to set the initial state <-w.initialise From 975266b14da047458d791f7179cb50342f2726b3 Mon Sep 17 00:00:00 2001 From: Barnaby Keene Date: Sun, 5 Apr 2020 20:28:38 +0100 Subject: [PATCH 2/3] clean readme (#58) * clean readme * add a size to the header image --- README.md | 210 ++++++++---------------------------------------------- 1 file changed, 29 insertions(+), 181 deletions(-) diff --git a/README.md b/README.md index 39f5301..ceb872e 100644 --- a/README.md +++ b/README.md @@ -1,181 +1,29 @@ -# Pico - -_The little git robot of automation!_ - -[![Build Status](https://travis-ci.org/picostack/pico.svg?branch=master)](https://travis-ci.org/picostack/pico) - -Pico is a git-driven task runner to automate the application of configs. - -## Overview - -Pico is a little tool for implementing [Git-Ops][git-ops] in single-server environments. It's analogous to -[kube-applier][kube-applier], [Terraform][terraform], [Ansible][ansible] but for automating lone servers that do not -need cluster-level orchestration. - -Instead, Pico aims to be extremely simple. You give it some Git repositories and tell it to run commands when those -Git repositories receive commits and that's about it. It also provides a way of safely passing in credentials from -[Hashicorp's Vault][vault]. - -## Install - -### Linux - -```sh -curl -s https://raw.githubusercontent.com/picostack/pico/master/install.sh | bash -``` - -Or via Docker: - -```sh -docker pull picostack/pico:v1 -``` - -See the docker section below and the image on [Docker Hub](https://hub.docker.com/r/picostack/pico). - -### Everything Else - -It's primarily a server side tool aimed at Linux servers, so there aren't any install scripts for other platforms. Most -Windows/Mac usage is probably just local testing so just use `go get` for these use-cases. - -## Usage - -Currently, Pico has a single command: `run` and it takes a single parameter: a Git URL. This Git URL defines the -"Config Repo" which contains Pico configuration files. These configuration files declare where Pico can find -"Target Repos" which are the repos that contain all the stuff you want to automate. The reason Pico is designed -this way instead of just using the target repos to define what Pico should do is 1. to consolidate Pico config -into one place, 2. separate the config of the tools from the applications and 3. keep your target repos clean. - -Pico also has a Docker image - see below for docker-specific information. - -### Configuration - -The precursor to Pico used JSON for configuration, this was fine for simple tasks but the ability to provide a -little bit of logic and variables for repetitive configurations is very helpful. Inspired by [StackExchange's -dnscontrol][dnscontrol], Pico uses JavaScript files as configuration. This provides a JSON-like environment with -the added benefit of conditional logic. - -Here's a simple example of a configuration that should exist in the Pico config repo that re-deploys a Docker -Compose stack whenever it changes: - -```js -T({ - name: "my_app", - url: "git@github.com:username/my-docker-compose-project", - branch: "prod", - up: ["docker-compose", "up", "-d"], - down: ["docker-compose", "down"] -}); -``` - -#### The `T` Function - -The `T` function declares a "Target" which is essentially a Git repository. In this example, the repository -`git@github.com:username/my-docker-compose-project` would contain a `docker-compose.yml` file for some application -stack. Every time you make a change to this file and push it, Pico will pull the new version and run the command -defined in the `up` attribute of the target, which is `docker-compose up -d`. - -You can put as many target declarations as you want in the config file, and as many config files as you want in the -config repo. You can also use variables to cut down on repeated things: - -```js -var GIT_HOST = "git@github.com:username/"; -T({ - name: "my_app", - url: GIT_HOST + "my-docker-compose-project", - up: ["docker-compose", "up", "-d"] -}); -``` - -Or, if you have a ton of Docker Compose projects and they all live on the same Git host, why not declare a function that -does all the hard work: - -```js -var GIT_HOST = "git@github.com:username/"; - -function Compose(name) { - return { - name: name, - url: GIT_HOST + name, - up: ["docker-compose", "up", "-d"] - }; -} - -T(Compose("homepage")); -T(Compose("todo-app")); -T(Compose("world-domination-scheme")); -``` - -The object passed to the `T` function accepts the following keys: - -- `name`: The name of the target -- `url`: The Git URL (ssh or https) -- `up`: The command to run on first-run and on changes -- `down`: The command to run when the target is removed -- `env`: Environment variables to pass to the target - -#### The `E` Function - -The only other function available in the configuration runtime is `E`, this declares an environment variable that will -be passed to the `up` and `down` commands for all targets. - -For example: - -```js -E("MOUNT_POINT", "/data"); -T({ name: "postgres", url: "...", up: "docker-compose", "up", "-d" }); -``` - -This would pass the environment variable `MOUNT_POINT=/data` to the `docker-compose` invocation. This is useful if you -have a bunch of compose configs that all mount data to some path on the machine, you then use -`${MOUNT_POINT}/postgres:/var/lib/postgres/data` as a volume declaration in your `docker-compose.yml`. - -## Usage as a Docker Container - -See the `docker-compose.yml` file for an example and read below for details. - -You can run Pico as a Docker container. If you're using it to deploy Docker containers via compose, this makes the -most sense. This is quite simple and is best done by writing a Docker Compose configuration for Pico in order to -bootstrap your deployment. - -The Pico image is built on the `docker/compose` image, since most use-cases will use Docker or Compose to deploy -services. This means you must mount the Docker API socket into the container, just like Portainer or cAdvisor or any of -the other Docker tools that also run inside a container. - -The socket is located by default at `/var/run/docker.sock` and the `docker/compose` image expects this path too, so you -just need to add a volume mount to your compose that specifies `/var/run/docker.sock:/var/run/docker.sock`. - -Another minor detail you should know is that Pico exposes a `HOSTNAME` variable for the configuration script. -However, when in a container, this hostname is a randomised string such as `b50fa67783ad`. This means, if your -configuration performs checks such as `if (HOSTNAME === 'server031')`, this won't work. To resolve this, Pico will -attempt to read the environment variable `HOSTNAME` and use that instead of using `/etc/hostname`. - -This means, you can bootstrap a Pico deployment with only two variables: - -```env -VAULT_TOKEN=abcxyz -HOSTNAME=server012 -``` - -### Docker Compose and `./` in Container Volume Mounts - -Another caveat to running Pico in a container to execute `docker-compose` is the container filesystem will not -match the host filesystem paths. - -If you mount directories from your repository - a common strategy for versioning configuration - `./` will be expanded -by Docker compose running inside the container, but this path may not be valid in the context of the Docker daemon, -which will be running on the host. - -The solution to this is both `DIRECTORY: "/cache"` and `/cache:/cache`: as long as the path used in the container also -exists on the host, Docker compose will expand `./` to the same path as the host and everything will work fine. - -This also means your config and target configurations will be persisted on the host's filesystem. - - - -[wadsworth]: https://i.imgur.com/RCYbkiq.png -[git-ops]: https://www.weave.works/blog/gitops-operations-by-pull-request -[kube-applier]: https://github.com/box/kube-applier -[terraform]: https://terraform.io -[ansible]: https://ansible.com -[vault]: https://vaultproject.io -[dnscontrol]: https://stackexchange.github.io/dnscontrol/ +

+ + + +

+ +

+ The little git robot of automation! +

+ +

+ GitHub Workflow Status + License +

+ +

+ Pico is a Git-driven task runner built to facilitate GitOps and + Infrastructure-as-Code while securely passing secrets to tasks. +

+ +

+ pico.sh +

From 40a642ed95c62c93109cf3de1b41966726414bd2 Mon Sep 17 00:00:00 2001 From: Barnaby Keene Date: Tue, 7 Apr 2020 10:47:16 +0100 Subject: [PATCH 3/3] remove the spin loop (#60) I was going to build a more complex restart system that would recover from certain errors, but it's a lot more work so I'm skipping it for now. --- service/service.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/service/service.go b/service/service.go index 4fa0dca..967bc9f 100644 --- a/service/service.go +++ b/service/service.go @@ -131,20 +131,11 @@ func (app *App) Start(ctx context.Context) error { }() } - handle := func() error { - select { - case err := <-errs: - return err - case <-ctx.Done(): - return context.Canceled - } - } - - zap.L().Debug("starting service main loop") - for { - if err := handle(); err != nil { - return err - } + select { + case err := <-errs: + return err + case <-ctx.Done(): + return context.Canceled } }