From 3d9ef06165edb37652fdb3e90958a28ff4723ca4 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Wed, 30 Oct 2019 18:42:46 +0000 Subject: [PATCH] Container Hosts/Projects register in Consul + refactoring Reorganized a lot of code to do with external connections and moved them out of the connections package Saves project metadata in Consul KV under its windlass_worker@hostname for below reasons Worker will check Consul KV for any projects associated with it on startup and spin up goroutine for health check Same topic, implemented Ping for LXD container host for said health check On creation of a project, it is also registered in Consul, KV and service LXD Host repo keeps in-memory the client TLS certs/keys while also persisting them in Vault for on next startup EstablishConnections checks connections on startup instead. Do we wanna change this to be in degraded state rather than outright not running? --- .gitignore | 1 + Dockerfile | 6 +- app/api/v1/projectsEndpoint.go | 2 +- app/connections/connections.go | 97 +----- app/connections/consulProvider.go | 153 --------- app/connections/vaultProvider.go | 34 -- .../containerHost/containerHostRepo.go | 19 +- .../containerHost/lxdContainerHostRepo.go | 79 ++++- app/repositories/providers/consulProvider.go | 306 ++++++++++++++++++ app/repositories/providers/kvProvider.go | 6 + app/repositories/providers/vaultProvider.go | 46 +++ app/repositories/tlsStorage/tlsStorageRepo.go | 24 +- .../tlsStorage/vaultTlsStorageRepo.go | 22 +- app/services/containerHostService.go | 41 ++- app/services/tlsCertService.go | 4 +- cmd/windlass-worker/main.go | 3 +- go.mod | 3 +- go.sum | 56 +++- 18 files changed, 564 insertions(+), 338 deletions(-) create mode 100644 .gitignore delete mode 100644 app/connections/consulProvider.go delete mode 100644 app/connections/vaultProvider.go create mode 100644 app/repositories/providers/consulProvider.go create mode 100644 app/repositories/providers/kvProvider.go create mode 100644 app/repositories/providers/vaultProvider.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a725465 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +vendor/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 73f022b..886352f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.12 AS dev +FROM golang:1.13 AS dev WORKDIR /windlass-worker @@ -6,8 +6,6 @@ RUN go get github.com/go-task/task/cmd/task \ github.com/derekparker/delve/cmd/dlv \ github.com/nomad-software/vend -ENV GO111MODULES=on - COPY go.mod . COPY go.sum . @@ -19,8 +17,6 @@ RUN go install github.com/UCCNetworkingSociety/Windlass-worker/cmd/windlass-work RUN go mod vendor && vend -ENV GO111MODULES=off - CMD [ "go", "run", "cmd/windlass-worker/main.go" ] FROM alpine diff --git a/app/api/v1/projectsEndpoint.go b/app/api/v1/projectsEndpoint.go index 6ccbb71..471d3d1 100644 --- a/app/api/v1/projectsEndpoint.go +++ b/app/api/v1/projectsEndpoint.go @@ -29,7 +29,7 @@ func NewProjectEndpoints(r chi.Router) { } r.Route("/projects", func(r chi.Router) { - r.Post("/", middleware.WithContext(projectEndpoint.createProject, time.Second*20)) + r.Post("/", middleware.WithContext(projectEndpoint.createProject, time.Second*40)) }) } diff --git a/app/connections/connections.go b/app/connections/connections.go index b8bd467..51f9f50 100644 --- a/app/connections/connections.go +++ b/app/connections/connections.go @@ -1,106 +1,39 @@ package connections import ( - consul "github.com/hashicorp/consul/api" - vault "github.com/hashicorp/vault/api" - - lxd "github.com/lxc/lxd/client" - "github.com/Strum355/log" - "github.com/spf13/viper" -) - -type Connections struct { - lxd lxd.ContainerServer - consul *ConsulProvider - vault *VaultProvider -} -var group Connections + "github.com/UCCNetworkingSociety/Windlass-worker/app/repositories/providers" +) -func EstablishConnections() error { +func TestConnections() error { var err error - if _, err = GetConsul(); err != nil { - return err - } - - if _, err = GetVault(); err != nil { - return err - } - - if err := group.consul.GetAndSetSharedSecret(); err != nil { + if err = testConsul(); err != nil { return err } - if _, err = GetLXD(); err != nil { + if err = testVault(); err != nil { return err } - log.Debug("connections established") + log.Debug("connections tested successfully") return nil } -func Close() { - +func testVault() error { + _, err := providers.NewVaultProvider() + return err } -func GetVault() (*VaultProvider, error) { - if group.vault != nil { - return group.vault, nil - } - - config := vault.Config{ - Address: viper.GetString("vault.url"), - } - - provider, err := newVaultProvider(&config) +func testConsul() error { + p, err := providers.NewConsulProvider() if err != nil { - return nil, err - } - - provider.client.SetToken(viper.GetString("vault.token")) - - return provider, nil -} - -func GetConsul() (*ConsulProvider, error) { - if group.consul != nil { - return group.consul, nil - } - - config := consul.Config{ - Address: viper.GetString("consul.url"), - Token: viper.GetString("consul.token"), - } - - provider, err := newConsulProvider(&config) - if err != nil { - return nil, NewConnectionError(err, "Consul") - } - - if err := provider.Register(); err != nil { - return nil, NewConnectionError(err, "Consul") - } - - group.consul = provider - - return group.consul, nil -} - -func GetLXD() (lxd.ContainerServer, error) { - if group.lxd != nil { - return group.lxd, nil + return err } - lxdConn, err := lxd.ConnectLXDUnix(viper.GetString("lxd.socket"), &lxd.ConnectionArgs{ - UserAgent: "Windlass", - }) - if err != nil { - return nil, NewConnectionError(err, "LXD") + if err := p.Register(); err != nil { + return err } - - group.lxd = lxdConn - - return lxdConn, nil + return p.GetAndSetSharedSecret() } diff --git a/app/connections/consulProvider.go b/app/connections/consulProvider.go deleted file mode 100644 index 6166545..0000000 --- a/app/connections/consulProvider.go +++ /dev/null @@ -1,153 +0,0 @@ -package connections - -import ( - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/Strum355/log" - - "github.com/spf13/viper" - - consul "github.com/hashicorp/consul/api" -) - -type ConsulProvider struct { - id string - address string - port int - client *consul.Client - deregisterCritical time.Duration - ttl time.Duration - refreshTTL time.Duration - ttlError error - secretGetRetry int -} - -func newConsulProvider(conf *consul.Config) (*ConsulProvider, error) { - client, err := consul.NewClient(conf) - if err != nil { - return nil, err - } - - return &ConsulProvider{ - client: client, - ttl: time.Second * 10, - refreshTTL: time.Second * 5, - secretGetRetry: 5, - }, nil -} - -// Register registers the worker and its associated projects with Consul. -// See registerWorker() and registerProjects() for specific details -func (p *ConsulProvider) Register() error { - p.id = fmt.Sprintf("windlass-worker@%s:%s", viper.GetString("http.address"), strconv.Itoa(viper.GetInt("http.port"))) - p.port = viper.GetInt("http.port") - p.address = viper.GetString("http.address") - - if err := p.registerWorker(); err != nil { - return err - } - - if err := p.registerProjects(); err != nil { - return err - } - - p.udpateWorkerTTL() - - p.updateProjectTTL() - - return nil -} - -// Registers this worker with Consul using a TTL based health check -func (p *ConsulProvider) registerWorker() error { - service := &consul.AgentServiceRegistration{ - ID: p.id, - Name: "windlass-worker", - Address: p.address, - Port: p.port, - Check: &consul.AgentServiceCheck{ - // Probably wanna keep failed workers around - // TODO: Need to purposely deregister on successful shutdown then - //DeregisterCriticalServiceAfter: p.deregisterCritical.String(), - TTL: p.ttl.String(), - }, - } - - return p.client.Agent().ServiceRegister(service) -} - -// Registers the projects associated with this worker with Consul using a TTL based health check -// It pings the Docker Daemon on each project, see https://docs.docker.com/engine/api/v1.37/#operation/SystemPing for details -func (p *ConsulProvider) registerProjects() error { - - return nil -} - -// Updates the TTL for this worker -func (p *ConsulProvider) udpateWorkerTTL() { - go func() { - ticker := time.NewTicker(p.ttl / 2) - for range ticker.C { - health := consul.HealthPassing - if viper.GetString("windlass.secret") == "" { - health = consul.HealthCritical - } - - err := p.client.Agent().UpdateTTL("service:"+p.id, "", health) - p.ttlError = err - if err != nil { - p.onFailedTTL() - log.WithError(err).Error("failed to update TTL") - } - } - }() -} - -// Updates the TTL for each project associated with this worker -func (p *ConsulProvider) updateProjectTTL() { - -} - -func (p *ConsulProvider) GetAndSetSharedSecret() error { - fn := func() error { - path := viper.GetString("consul.path") + "/secret" - kv, _, err := p.client.KV().Get(path, &consul.QueryOptions{}) - if err != nil { - return err - } - - if kv == nil { - return errors.New(fmt.Sprintf("key %s not set", path)) - } - - viper.Set("windlass.secret", kv.Value) - return nil - } - - count := p.secretGetRetry - var err error - for ; count > 0; count-- { - err = fn() - if err == nil { - return nil - } - log.WithFields(log.Fields{ - "limit": p.secretGetRetry, - "count": count, - }).WithError(err).Error("failed to get shared secret") - time.Sleep(time.Second * 3) - } - return err -} - -func (p *ConsulProvider) onFailedTTL() error { - if strings.HasSuffix(p.ttlError.Error(), "does not have associated TTL)") { - return p.registerWorker() - } - - return nil -} diff --git a/app/connections/vaultProvider.go b/app/connections/vaultProvider.go deleted file mode 100644 index 1fafb6f..0000000 --- a/app/connections/vaultProvider.go +++ /dev/null @@ -1,34 +0,0 @@ -package connections - -import ( - vault "github.com/hashicorp/vault/api" -) - -type VaultProvider struct { - client *vault.Client -} - -func newVaultProvider(conf *vault.Config) (*VaultProvider, error) { - client, err := vault.NewClient(conf) - if err != nil { - return nil, err - } - - return &VaultProvider{ - client: client, - }, nil -} - -func (p *VaultProvider) PushSecrets(pathPrefix string, kv map[string]interface{}) error { - _, err := p.client.Logical().Write(pathPrefix, kv) - return err -} - -func (p *VaultProvider) GetSecrets(pathPrefix string) (map[string]interface{}, error) { - s, err := p.client.Logical().Read(pathPrefix) - if err != nil { - return nil, err - } - - return s.Data, nil -} diff --git a/app/repositories/containerHost/containerHostRepo.go b/app/repositories/containerHost/containerHostRepo.go index e8176e4..d82f0b6 100644 --- a/app/repositories/containerHost/containerHostRepo.go +++ b/app/repositories/containerHost/containerHostRepo.go @@ -2,14 +2,14 @@ package host import ( "context" -) + "fmt" -type Pinger interface { - Ping(ctx context.Context) error -} + "github.com/spf13/viper" +) type ContainerHostRepository interface { - Pinger + Ping(ctx context.Context) error + UseCerts(clientKeyPEM, clientCertPEM, caPEM []byte) GetContainerHostIP(ctx context.Context, name string) (string, error) CreateContainerHost(ctx context.Context, opts ContainerHostCreateOptions) error DeleteContainerHost(ctx context.Context, opts ContainerHostDeleteOptions) error @@ -19,6 +19,15 @@ type ContainerHostRepository interface { RestartNGINX(ctx context.Context, name string) error } +func NewContainerHostRepository() ContainerHostRepository { + hostProvider := viper.GetString("containerHost.type") + + if hostProvider == "lxd" { + return NewLXDRepository() + } + panic(fmt.Sprintf("invalid container host %s", hostProvider)) +} + type ContainerName struct { Name string } diff --git a/app/repositories/containerHost/lxdContainerHostRepo.go b/app/repositories/containerHost/lxdContainerHostRepo.go index c1a9ee1..de31cc2 100644 --- a/app/repositories/containerHost/lxdContainerHostRepo.go +++ b/app/repositories/containerHost/lxdContainerHostRepo.go @@ -3,49 +3,87 @@ package host import ( "bytes" "context" - _ "errors" "fmt" "strings" "time" + docker "github.com/fsouza/go-dockerclient" "github.com/pkg/errors" "github.com/cenkalti/backoff" + lxd "github.com/lxc/lxd/client" + "go.uber.org/multierr" "github.com/Strum355/log" "github.com/spf13/viper" - "github.com/UCCNetworkingSociety/Windlass-worker/app/connections" "github.com/UCCNetworkingSociety/Windlass-worker/app/helpers" "github.com/UCCNetworkingSociety/Windlass-worker/utils/writecloser" lxdclient "github.com/lxc/lxd/client" "github.com/lxc/lxd/shared/api" ) -type LXDHost struct { - conn lxdclient.ContainerServer +var lxdConn lxd.ContainerServer + +type lxdHost struct { + conn lxdclient.ContainerServer + dockerConn *docker.Client + ip string + + // certs for interacting with the host's Docker daemon + clientKeyPEM []byte + clientCertPEM []byte + caPEM []byte +} + +func getLXD() (lxd.ContainerServer, error) { + if lxdConn != nil { + return lxdConn, nil + } + + lxdconn, err := lxd.ConnectLXDUnix(viper.GetString("lxd.socket"), &lxd.ConnectionArgs{ + UserAgent: "Windlass", + }) + if err != nil { + return nil, fmt.Errorf("couldnt connect to LXD socket: %v", err) + } + + lxdConn = lxdconn + + return lxdConn, nil } // TODO context tiemouts func NewLXDRepository() ContainerHostRepository { - lxdHost, err := connections.GetLXD() + lxdHostConn, err := getLXD() if err != nil { panic(fmt.Sprintf("error getting LXD host: %v", err)) } - return &LXDHost{ - conn: lxdHost, + return &lxdHost{ + conn: lxdHostConn, } } -func (lxd *LXDHost) Ping(ctx context.Context) error { - return nil +func (lxd *lxdHost) Ping(ctx context.Context) error { + if lxd.dockerConn == nil { + client, err := docker.NewTLSClientFromBytes("https://"+lxd.ip, lxd.clientCertPEM, lxd.clientKeyPEM, lxd.caPEM) + if err != nil { + return err + } + lxd.dockerConn = client + } + + log.WithFields(log.Fields{ + "ip": lxd.ip, + }).Info("pinging docker endpoint") + return lxd.dockerConn.PingWithContext(ctx) } -func (lxd *LXDHost) parseError(err error) error { +func (lxd *lxdHost) parseError(err error) error { if err == nil { return nil } @@ -55,7 +93,7 @@ func (lxd *LXDHost) parseError(err error) error { return err } -func (lxd *LXDHost) CreateContainerHost(ctx context.Context, opts ContainerHostCreateOptions) error { +func (lxd *lxdHost) CreateContainerHost(ctx context.Context, opts ContainerHostCreateOptions) error { log.WithFields(log.Fields{ "containerHostName": opts.Name, }).Debug("create container host request") @@ -89,7 +127,7 @@ func (lxd *LXDHost) CreateContainerHost(ctx context.Context, opts ContainerHostC return lxd.parseError(err) } -func (lxd *LXDHost) DeleteContainerHost(ctx context.Context, opts ContainerHostDeleteOptions) error { +func (lxd *lxdHost) DeleteContainerHost(ctx context.Context, opts ContainerHostDeleteOptions) error { op, err := lxd.conn.DeleteContainer(opts.Name) if err != nil { return err @@ -97,7 +135,7 @@ func (lxd *LXDHost) DeleteContainerHost(ctx context.Context, opts ContainerHostD return helpers.OperationTimeout(ctx, op) } -func (lxd *LXDHost) StartContainerHost(ctx context.Context, opts ContainerHostStartOptions) error { +func (lxd *lxdHost) StartContainerHost(ctx context.Context, opts ContainerHostStartOptions) error { op, err := lxd.conn.UpdateContainerState(opts.Name, api.ContainerStatePut{ Action: "start", Timeout: -1, @@ -109,7 +147,7 @@ func (lxd *LXDHost) StartContainerHost(ctx context.Context, opts ContainerHostSt return helpers.OperationTimeout(ctx, op) } -func (lxd *LXDHost) StopContainerHost(ctx context.Context, opts ContainerHostStopOptions) error { +func (lxd *lxdHost) StopContainerHost(ctx context.Context, opts ContainerHostStopOptions) error { op, err := lxd.conn.UpdateContainerState(opts.Name, api.ContainerStatePut{ Action: "stop", Timeout: -1, @@ -121,7 +159,7 @@ func (lxd *LXDHost) StopContainerHost(ctx context.Context, opts ContainerHostSto return helpers.OperationTimeout(ctx, op) } -func (lxd *LXDHost) GetContainerHostIP(ctx context.Context, name string) (string, error) { +func (lxd *lxdHost) GetContainerHostIP(ctx context.Context, name string) (string, error) { var ip string retry := backoff.WithContext(backoff.NewConstantBackOff(time.Millisecond*5), ctx) f := func() error { @@ -133,6 +171,7 @@ func (lxd *LXDHost) GetContainerHostIP(ctx context.Context, name string) (string for _, addr := range state.Network["eth0"].Addresses { if addr.Family == "inet" { ip = addr.Address + lxd.ip = ip return nil } } @@ -142,7 +181,7 @@ func (lxd *LXDHost) GetContainerHostIP(ctx context.Context, name string) (string return ip, backoff.Retry(f, retry) } -func (lxd *LXDHost) PushAuthCerts(ctx context.Context, opts ContainerPushCertsOptions, caPEM, serverKeyPEM, serverCertPEM []byte) error { +func (lxd *lxdHost) PushAuthCerts(ctx context.Context, opts ContainerPushCertsOptions, caPEM, serverKeyPEM, serverCertPEM []byte) error { err := multierr.Combine( errors.WithMessage(lxd.conn.CreateContainerFile(opts.Name, "/nginx/ca-cert.pem", lxdclient.ContainerFileArgs{ UID: 0, GID: 0, Content: bytes.NewReader(caPEM), Mode: 400, Type: "file", WriteMode: "overwrite", @@ -158,7 +197,13 @@ func (lxd *LXDHost) PushAuthCerts(ctx context.Context, opts ContainerPushCertsOp return err } -func (lxd *LXDHost) RestartNGINX(ctx context.Context, name string) error { +func (lxd *lxdHost) UseCerts(clientKeyPEM, clientCertPEM, caPEM []byte) { + lxd.clientKeyPEM = clientKeyPEM + lxd.clientCertPEM = clientCertPEM + lxd.caPEM = caPEM +} + +func (lxd *lxdHost) RestartNGINX(ctx context.Context, name string) error { exec := api.ContainerExecPost{ Command: []string{"systemctl", "restart", "nginx"}, WaitForWS: true, diff --git a/app/repositories/providers/consulProvider.go b/app/repositories/providers/consulProvider.go new file mode 100644 index 0000000..11b797e --- /dev/null +++ b/app/repositories/providers/consulProvider.go @@ -0,0 +1,306 @@ +package providers + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + host "github.com/UCCNetworkingSociety/Windlass-worker/app/repositories/containerHost" + + "github.com/Strum355/log" + + "github.com/spf13/viper" + + consul "github.com/hashicorp/consul/api" +) + +type ConsulProvider struct { + client *consul.Client + ttl time.Duration + mu *sync.Mutex + secretGetRetry int + projectIDtoCheck map[string]projectCheck +} + +type projectCheck struct { + check func(ip string) (string, bool) + ticker *time.Ticker +} + +type projectMeta struct { + ID string `json:"id"` + IP string `json:"ip_address"` +} + +func NewConsulProvider() (*ConsulProvider, error) { + config := consul.Config{ + Address: viper.GetString("consul.url"), + Token: viper.GetString("consul.token"), + } + + client, err := consul.NewClient(&config) + if err != nil { + return nil, err + } + + return &ConsulProvider{ + client: client, + ttl: time.Second * 10, + secretGetRetry: 5, + projectIDtoCheck: make(map[string]projectCheck), + mu: new(sync.Mutex), + }, nil +} + +func (p *ConsulProvider) id() string { + return fmt.Sprintf("windlass-worker@%s:%s", viper.GetString("http.hostname"), strconv.Itoa(viper.GetInt("http.port"))) +} + +func (p *ConsulProvider) address() string { + return viper.GetString("http.address") +} + +func (p *ConsulProvider) port() int { + return viper.GetInt("http.port") +} + +// Register registers the worker and its associated projects with Consul. +// See registerWorker() and registerProjects() for specific details +func (p *ConsulProvider) Register() error { + if err := p.registerWorker(); err != nil { + return fmt.Errorf("failed to register worker: %v", err) + } + + if err := p.registerProjects(); err != nil { + return fmt.Errorf("failed to register one or more associated projects: %v", err) + } + + p.udpateWorkerTTL() + + return nil +} + +// Registers this worker with Consul using a TTL based health check +func (p *ConsulProvider) registerWorker() error { + service := &consul.AgentServiceRegistration{ + ID: p.id(), + Name: "windlass_worker", + Address: p.address(), + Port: p.port(), + Check: &consul.AgentServiceCheck{ + // Probably wanna keep failed workers around + // TODO: Need to purposely deregister on successful shutdown then + //DeregisterCriticalServiceAfter: p.deregisterCritical.String(), + TTL: p.ttl.String(), + }, + } + + return p.client.Agent().ServiceRegister(service) +} + +// Registers the projects associated with this worker with Consul using a TTL based health check +// It pings the Docker Daemon on each project, see https://docs.docker.com/engine/api/v1.37/#operation/SystemPing for details +func (p *ConsulProvider) registerProjects() error { + pairs, _, err := p.client.KV().List(p.kvPath(), &consul.QueryOptions{}) + if err != nil { + return errors.New(fmt.Sprintf("failed to load KV at path %s", p.kvPath())) + } + + for _, pair := range pairs { + var meta projectMeta + err := json.Unmarshal(pair.Value, &meta) + if err != nil { + return err + } + + containerHost := host.NewContainerHostRepository() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + projectName := p.projectNameFromKVPath(pair.Key) + ip, err := containerHost.GetContainerHostIP(ctx, projectName) + if err != nil { + return errors.New(fmt.Sprintf("failed to get container IP for host %s: %v", projectName, err)) + } + + vault, err := NewVaultProvider() + if err != nil { + return err + } + + tls, err := vault.Get(viper.GetString("vault.path") + meta.ID) + if err != nil { + return err + } + + clientKey, _ := base64.StdEncoding.DecodeString(tls["client_key"].(string)) + clientCert, _ := base64.StdEncoding.DecodeString(tls["client_cert"].(string)) + clientCA, _ := base64.StdEncoding.DecodeString(tls["server_ca"].(string)) + + containerHost.UseCerts([]byte(clientKey), []byte(clientCert), []byte(clientCA)) + + p.RegisterProject(p.projectNameFromKVPath(pair.Key), ip, func(ip string) (string, bool) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := containerHost.Ping(ctx) + if err != nil { + return err.Error(), false + } + return "Remote Docker daemon reachable", true + }) + } + + return nil +} + +// RegisterProject registers a single project +func (p *ConsulProvider) RegisterProject(projectName string, ip string, check func(ip string) (string, bool)) error { + projectM := projectMeta{ID: projectName, IP: ip} + + projectService := &consul.AgentServiceRegistration{ + ID: projectName, + Name: "windlass_worker_projects", + Tags: []string{fmt.Sprintf("Worker:%s", p.kvPath())}, + Address: ip, + Port: 443, + Check: &consul.AgentServiceCheck{ + TTL: (p.ttl * 5).String(), + }, + } + + err := p.client.Agent().ServiceRegister(projectService) + if err != nil { + return err + } + + err = p.SaveProjectMeta(projectM) + if err != nil { + return err + } + + p.updateProjectTTL(projectName, ip, check) + + return nil +} + +// Updates the TTL for this worker +func (p *ConsulProvider) udpateWorkerTTL() { + go func() { + ticker := time.NewTicker(p.ttl / 2) + for range ticker.C { + health := consul.HealthPassing + if viper.GetString("windlass.secret") == "" { + health = consul.HealthCritical + } + + err := p.client.Agent().UpdateTTL("service:"+p.id(), "", health) + if err != nil { + p.onFailedWorkerTTL(err) + log.WithError(err).Error("failed to update TTL") + } + } + }() +} + +// Updates the TTL for each project associated with this worker +// runs `check` on each tick which returns a string for health message and a bool true if healthy and false if not +func (p *ConsulProvider) updateProjectTTL(id, ip string, check func(ip string) (string, bool)) { + ticker := time.NewTicker((p.ttl * 5) / 2) + p.projectIDtoCheck[id] = projectCheck{ + check: check, ticker: ticker, + } + + go func() { + for range ticker.C { + health := consul.HealthPassing + msg, healthy := check(ip) + log.WithFields(log.Fields{ + "msg": msg, + "healthy": healthy, + }).Info("docker daemon health check ping response") + if !healthy { + health = consul.HealthCritical + } + err := p.client.Agent().UpdateTTL("service:"+id, msg, health) + if err != nil { + + } + } + }() +} + +func (p *ConsulProvider) GetAndSetSharedSecret() error { + fn := func() error { + path := viper.GetString("consul.path") + "/secret" + kv, _, err := p.client.KV().Get(path, &consul.QueryOptions{}) + if err != nil { + return err + } + + if kv == nil { + return errors.New(fmt.Sprintf("key %s not set", path)) + } + + viper.Set("windlass.secret", kv.Value) + return nil + } + + count := p.secretGetRetry + var err error + for ; count > 0; count-- { + err = fn() + if err == nil { + return nil + } + log.WithFields(log.Fields{ + "limit": p.secretGetRetry, + "count": count, + }).WithError(err).Error("failed to get shared secret") + time.Sleep(time.Second * 3) + } + return err +} + +func (p *ConsulProvider) SaveProjectMeta(projectMetadata projectMeta) error { + b, err := json.Marshal(projectMetadata) + if err != nil { + return err + } + + p.mu.Lock() + _, err = p.client.KV().Put(&consul.KVPair{ + Key: fmt.Sprintf("%s/%s", p.kvPath(), projectMetadata.ID), + Value: b, + }, &consul.WriteOptions{}) + return err +} + +func (p *ConsulProvider) kvPath() string { + return fmt.Sprintf("windlass_worker@%s", viper.GetString("http.hostname")) +} + +func (p *ConsulProvider) onFailedWorkerTTL(err error) error { + if strings.HasSuffix(err.Error(), "does not have associated TTL)") { + return p.registerWorker() + } + + return nil +} + +func (p *ConsulProvider) onFailedProjectTTL(id, ip string, err error) error { + if strings.HasPrefix(err.Error(), "does not have associated TTL") { + return p.RegisterProject(id, ip, p.projectIDtoCheck[id].check) + } + return nil +} + +func (p *ConsulProvider) projectNameFromKVPath(path string) string { + return strings.TrimSuffix(strings.TrimPrefix(path, p.kvPath()+"/"), "/") +} diff --git a/app/repositories/providers/kvProvider.go b/app/repositories/providers/kvProvider.go new file mode 100644 index 0000000..00bef01 --- /dev/null +++ b/app/repositories/providers/kvProvider.go @@ -0,0 +1,6 @@ +package providers + +type KVProvider interface { + Put(pathPrefix string, kv map[string]interface{}) error + Get(pathPrefix string) (map[string]interface{}, error) +} diff --git a/app/repositories/providers/vaultProvider.go b/app/repositories/providers/vaultProvider.go new file mode 100644 index 0000000..0190815 --- /dev/null +++ b/app/repositories/providers/vaultProvider.go @@ -0,0 +1,46 @@ +package providers + +import ( + "errors" + + vault "github.com/hashicorp/vault/api" + "github.com/spf13/viper" +) + +type VaultProvider struct { + client *vault.Client +} + +func NewVaultProvider() (*VaultProvider, error) { + config := vault.Config{ + Address: viper.GetString("vault.url"), + } + client, err := vault.NewClient(&config) + if err != nil { + return nil, err + } + + client.SetToken(viper.GetString("vault.token")) + + return &VaultProvider{ + client: client, + }, nil +} + +func (p *VaultProvider) Put(pathPrefix string, kv map[string]interface{}) error { + _, err := p.client.Logical().Write(pathPrefix, kv) + return err +} + +func (p *VaultProvider) Get(pathPrefix string) (map[string]interface{}, error) { + s, err := p.client.Logical().Read(pathPrefix) + if err != nil { + return nil, err + } + + if s == nil { + return nil, errors.New("nothing found in Vault at given prefix") + } + + return s.Data, nil +} diff --git a/app/repositories/tlsStorage/tlsStorageRepo.go b/app/repositories/tlsStorage/tlsStorageRepo.go index ac330da..4f8d06f 100644 --- a/app/repositories/tlsStorage/tlsStorageRepo.go +++ b/app/repositories/tlsStorage/tlsStorageRepo.go @@ -2,8 +2,30 @@ package tlsstorage import ( "context" + "fmt" + + "github.com/UCCNetworkingSociety/Windlass-worker/app/repositories/providers" + "github.com/spf13/viper" ) type TLSStorageRepo interface { - PushAuthCerts(ctx context.Context, key string, caPEM, serverKeyPEM, serverCertPEM, clientKeyPEM, clientCertPEM []byte) error + PushAuthCerts(ctx context.Context, key string, serverCAPEM, clientCAPEM, serverKeyPEM, serverCertPEM, clientKeyPEM, clientCertPEM []byte) error +} + +func NewTLSStorageRepo() TLSStorageRepo { + if viper.GetBool("vault.enabled") { + vault, err := providers.NewVaultProvider() + if err != nil { + panic(fmt.Errorf("failed to create Vault client: %w", err)) + } + + repo, err := NewVaultTLSStorageRepo(vault) + if err != nil { + panic(fmt.Errorf("failed to get TLS storage repo: %w", err)) + } + return repo + } else { + // TODO: consul + } + panic("vault currently required") } diff --git a/app/repositories/tlsStorage/vaultTlsStorageRepo.go b/app/repositories/tlsStorage/vaultTlsStorageRepo.go index d0a2a0f..e78d69b 100644 --- a/app/repositories/tlsStorage/vaultTlsStorageRepo.go +++ b/app/repositories/tlsStorage/vaultTlsStorageRepo.go @@ -3,27 +3,23 @@ package tlsstorage import ( "context" - "github.com/UCCNetworkingSociety/Windlass-worker/app/connections" + "github.com/UCCNetworkingSociety/Windlass-worker/app/repositories/providers" + "github.com/spf13/viper" ) -type VaultTLSStorageRepo struct { - vault *connections.VaultProvider +type vaultTLSStorageRepo struct { + vault *providers.VaultProvider } -func NewVaultTLSStorageRepo() (*VaultTLSStorageRepo, error) { - vault, err := connections.GetVault() - if err != nil { - return nil, err - } - - return &VaultTLSStorageRepo{ +func NewVaultTLSStorageRepo(vault *providers.VaultProvider) (TLSStorageRepo, error) { + return &vaultTLSStorageRepo{ vault: vault, }, nil } -func (v *VaultTLSStorageRepo) PushAuthCerts(ctx context.Context, key string, caPEM, serverKeyPEM, serverCertPEM, clientKeyPEM, clientCertPEM []byte) error { - return v.vault.PushSecrets(viper.GetString("vault.path")+key, map[string]interface{}{ - "ca_cert": caPEM, "server_key": serverKeyPEM, "server_cert": serverCertPEM, "client_key": clientKeyPEM, "client_cert": clientCertPEM, +func (v *vaultTLSStorageRepo) PushAuthCerts(ctx context.Context, key string, serverCAPEM, clientCAPEM, serverKeyPEM, serverCertPEM, clientKeyPEM, clientCertPEM []byte) error { + return v.vault.Put(viper.GetString("vault.path")+key, map[string]interface{}{ + "server_ca": serverCAPEM, "client_ca": clientCAPEM, "server_key": serverKeyPEM, "server_cert": serverCertPEM, "client_key": clientKeyPEM, "client_cert": clientCertPEM, }) } diff --git a/app/services/containerHostService.go b/app/services/containerHostService.go index 1728d5a..6d6525c 100644 --- a/app/services/containerHostService.go +++ b/app/services/containerHostService.go @@ -3,14 +3,17 @@ package services import ( "context" "fmt" + "time" + + "github.com/UCCNetworkingSociety/Windlass-worker/app/repositories/providers" host "github.com/UCCNetworkingSociety/Windlass-worker/app/repositories/containerHost" tlsstorage "github.com/UCCNetworkingSociety/Windlass-worker/app/repositories/tlsStorage" - "github.com/spf13/viper" ) type ContainerHostService struct { repo host.ContainerHostRepository + consul *providers.ConsulProvider tlsService *TLSCertService tlsStorageRepo tlsstorage.TLSStorageRepo } @@ -20,23 +23,15 @@ func NewContainerHostService() *ContainerHostService { tlsService: NewTLSCertService(), } - hostProvider := viper.GetString("containerHost.type") + hostService.repo = host.NewContainerHostRepository() - if hostProvider == "lxd" { - hostService.repo = host.NewLXDRepository() - } else { - panic(fmt.Sprintf("invalid container host %s", hostProvider)) - } + hostService.tlsStorageRepo = tlsstorage.NewTLSStorageRepo() - if viper.GetBool("vault.enabled") { - repo, err := tlsstorage.NewVaultTLSStorageRepo() - if err != nil { - panic(fmt.Errorf("failed to get TLS storage repo: %w", err)) - } - hostService.tlsStorageRepo = repo - } else { - panic("vault currently required") + consul, err := providers.NewConsulProvider() + if err != nil { + panic(fmt.Sprintf("failed to get consul provider: %v", err)) } + hostService.consul = consul return hostService } @@ -64,7 +59,7 @@ func (service *ContainerHostService) CreateHost(ctx context.Context, name string return err } - if err := service.repo.PushAuthCerts(ctx, host.ContainerPushCertsOptions{containerName}, pems.CAPEM, pems.ServerKeyPEM, pems.ServerCertPEM); err != nil { + if err := service.repo.PushAuthCerts(ctx, host.ContainerPushCertsOptions{containerName}, pems.ServerCAPEM, pems.ServerKeyPEM, pems.ServerCertPEM); err != nil { return err } @@ -72,9 +67,21 @@ func (service *ContainerHostService) CreateHost(ctx context.Context, name string return err } - if err := service.tlsStorageRepo.PushAuthCerts(ctx, containerName.Name, pems.CAPEM, pems.ServerKeyPEM, pems.ServerCertPEM, pems.ClientKeyPEM, pems.ClientCertPEM); err != nil { + service.repo.UseCerts(pems.ClientKeyPEM, pems.ClientCertPEM, pems.ClientCAPEM) + + if err := service.tlsStorageRepo.PushAuthCerts(ctx, containerName.Name, pems.ServerCAPEM, pems.ClientCAPEM, pems.ServerKeyPEM, pems.ServerCertPEM, pems.ClientKeyPEM, pems.ClientCertPEM); err != nil { return err } + service.consul.RegisterProject(containerName.Name, ip, func(ip string) (string, bool) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := service.repo.Ping(ctx) + if err != nil { + return err.Error(), false + } + return "Remote Docker daemon reachable", true + }) + return nil } diff --git a/app/services/tlsCertService.go b/app/services/tlsCertService.go index a6cf91c..2e4f423 100644 --- a/app/services/tlsCertService.go +++ b/app/services/tlsCertService.go @@ -16,7 +16,7 @@ const ( ) type PEMContainer struct { - CAPEM, ServerKeyPEM, ServerCertPEM, ClientKeyPEM, ClientCertPEM []byte + ServerCAPEM, ClientCAPEM, ServerKeyPEM, ServerCertPEM, ClientKeyPEM, ClientCertPEM []byte } type TLSCertService struct { @@ -128,5 +128,5 @@ func (t *TLSCertService) CreatePEMs(serverIP string) (*PEMContainer, error) { return nil, err } - return &PEMContainer{caCertPEM, serverKeyPEM, serverCertPEM, clientKeyPEM, clientCertPEM}, nil + return &PEMContainer{caCertPEM, clientCACertPEM, serverKeyPEM, serverCertPEM, clientKeyPEM, clientCertPEM}, nil } diff --git a/cmd/windlass-worker/main.go b/cmd/windlass-worker/main.go index d779da0..5fbd119 100644 --- a/cmd/windlass-worker/main.go +++ b/cmd/windlass-worker/main.go @@ -25,8 +25,7 @@ func main() { must.Do(config.Load) - must.Do(connections.EstablishConnections) - defer connections.Close() + must.Do(connections.TestConnections) config.PrintSettings() diff --git a/go.mod b/go.mod index 43cf61f..7d33f4b 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Strum355/log v1.0.1 github.com/cenkalti/backoff v2.2.1+incompatible github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 // indirect - github.com/go-check/check v1.0.0-20180628173108-788fd7840127 // indirect + github.com/fsouza/go-dockerclient v1.5.0 github.com/go-chi/chi v4.0.2+incompatible github.com/go-chi/render v1.0.1 github.com/hashicorp/consul/api v1.1.0 @@ -24,7 +24,6 @@ require ( github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.4.0 // indirect go.uber.org/multierr v1.1.0 - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect gopkg.in/errgo.v1 v1.0.1 // indirect diff --git a/go.sum b/go.sum index ce937a2..4c3728c 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Strum355/log v1.0.1 h1:iU1K3d0gALy1EnI1ldN01x1n3Cm8vMeQttSRNtzxibk= github.com/Strum355/log v1.0.1/go.mod h1:5wP2IZ86aXjSO/xlH/9lNaN3G0K8u0baaHujSiIFtqA= @@ -18,6 +24,10 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/containerd/containerd v1.3.0 h1:xjvXQWABwS2uiv3TWgQt5Uth60Gu86LTGZXMJkjc7rY= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M= +github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -28,6 +38,15 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20190927142053-ada3c14355ce h1:H3csZuxZESJeeEiOxq4YXPNmLFbjl7u2qVBrAAGX/sA= +github.com/docker/docker v1.4.2-0.20190927142053-ada3c14355ce/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc= @@ -38,10 +57,11 @@ github.com/frankban/quicktest v1.2.2 h1:xfmOhhoH5fGPgbEAlhLpJH9p0z/0Qizio9osmvn9 github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsouza/go-dockerclient v1.5.0 h1:7OtayOe5HnoG+KWMHgyyPymwaodnB2IDYuVfseKyxbA= +github.com/fsouza/go-dockerclient v1.5.0/go.mod h1:AqZZK/zFO3phxYxlTsAaeAMSdQ9mgHuhy+bjN034Qds= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-check/check v1.0.0-20180628173108-788fd7840127 h1:3dbHpVjNKf7Myfit4Xmw4BA0JbCt47OJPhMQ5w8O3E8= -github.com/go-check/check v1.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= @@ -59,6 +79,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -69,6 +90,9 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42 h1:q3pnF5JFBNRz8sRD+IRj7Y6DMyYGTNqnZ9axTbSfoNI= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -146,6 +170,7 @@ github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -157,6 +182,7 @@ github.com/lxc/lxd v0.0.0-20190717210919-0ecd38c37220 h1:R4XJGEeDYAPhsfyZbxNby0F github.com/lxc/lxd v0.0.0-20190717210919-0ecd38c37220/go.mod h1:2BaZflfwsv8a3uy3/Vw+de4Avn4DSrAiqaHJjCIXMV4= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= @@ -177,9 +203,17 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -215,6 +249,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= @@ -252,8 +288,8 @@ golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad h1:5E5raQxcv+6CZ11RrBYQe5WRbUIWpScjh0kvHZkZIrQ= +golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -261,6 +297,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/net v0.0.0-20150829230318-ea47fc708ee3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -277,6 +314,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -287,6 +326,7 @@ golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -302,14 +342,19 @@ golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 h1:xtNn7qFlagY2mQNFHMSRPjT2RkOV4OXM7P5TVy9xATo= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= @@ -336,4 +381,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=