Using IP in Consul

5 step retry for getting shared secret
Updated log

app/config/defaults.go

@@ -10,7 +10,8 @@ import (
 
 func InitDefaults() {
 	viper.SetDefault("http.port", "9786")
-	viper.SetDefault("http.address", getFQDN())
+	viper.SetDefault("http.hostname", getFQDN())
+	viper.SetDefault("http.address", getOutboundIP().String())
 
 	// Consul settings
 	viper.SetDefault("consul.host", "127.0.0.1:8500")
@@ -50,3 +51,15 @@ func getFQDN() string {
 	}
 	return hostname
 }
+
+func getOutboundIP() net.IP {
+	conn, err := net.Dial("udp", "8.8.8.8:80")
+	if err != nil {
+		return nil
+	}
+	defer conn.Close()
+
+	localAddr := conn.LocalAddr().(*net.UDPAddr)
+
+	return localAddr.IP
+}

app/connections/consul/provider.go

@@ -3,8 +3,8 @@ package consul
 import (
 	"errors"
 	"fmt"
-	"net"
 	"strconv"
+	"strings"
 	"time"
 
 	log "github.com/UCCNetworkingSociety/Windlass-worker/utils/logging"
@@ -22,6 +22,8 @@ type Provider struct {
 	deregisterCritical time.Duration
 	ttl                time.Duration
 	refreshTTL         time.Duration
+	ttlError           error
+	secretGetRetry     int
 }
 
 func NewProvider(conf *api.Config) (*Provider, error) {
@@ -31,9 +33,10 @@ func NewProvider(conf *api.Config) (*Provider, error) {
 	}
 
 	return &Provider{
-		client:     client,
-		ttl:        time.Second * 10,
-		refreshTTL: time.Second * 5,
+		client:         client,
+		ttl:            time.Second * 10,
+		refreshTTL:     time.Second * 5,
+		secretGetRetry: 5,
 	}, nil
 }
 
@@ -87,8 +90,16 @@ func (p *Provider) udpateWorkerTTL() {
 	go func() {
 		ticker := time.NewTicker(p.ttl / 2)
 		for range ticker.C {
-			if err := p.client.Agent().UpdateTTL("service:"+p.id, "cool and well", api.HealthPassing); err != nil {
-				log.Error(err, "failed to update TTL")
+			health := api.HealthPassing
+			if viper.GetString("windlass.secret") == "" {
+				health = api.HealthCritical
+			}
+
+			err := p.client.Agent().UpdateTTL("service:"+p.id, "", health)
+			p.ttlError = err
+			if err != nil {
+				p.onFailedTTL()
+				log.WithError(err).Error("failed to update TTL")
 			}
 		}
 	}()
@@ -99,33 +110,42 @@ func (p *Provider) updateProjectTTL() {
 
 }
 
-func (p *Provider) getIP() (string, error) {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return "", err
-	}
+func (p *Provider) GetAndSetSharedSecret() error {
+	fn := func() error {
+		path := viper.GetString("consul.path") + "/secret"
+		kv, _, err := p.client.KV().Get(path, &api.QueryOptions{})
+		if err != nil {
+			return err
+		}
 
-	for _, a := range addrs {
-		if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To4() != nil {
-				return ipnet.IP.String(), nil
-			}
+		if kv == nil {
+			return errors.New(fmt.Sprintf("key %s not set", path))
 		}
+
+		viper.Set("windlass.secret", kv.Value)
+		return nil
 	}
-	return "", errors.New("couldnt find IP")
-}
 
-func (p *Provider) GetAndSetSharedSecret() error {
-	path := viper.GetString("consul.path") + "/secret"
-	kv, _, err := p.client.KV().Get(path, &api.QueryOptions{})
-	if err != nil {
-		return err
+	count := p.secretGetRetry
+	var err error
+	for ; count > 0; count-- {
+		err = fn()
+		if err == nil {
+			return nil
+		}
+		log.WithFields(log.Fields{
+			"limit": p.secretGetRetry,
+			"count": count,
+		}).WithError(err).Error("failed to get shared secret")
+		time.Sleep(time.Second * 3)
 	}
+	return err
+}
 
-	if kv == nil {
-		return errors.New(fmt.Sprintf("key %s not set", path))
+func (p *Provider) onFailedTTL() error {
+	if strings.HasSuffix(p.ttlError.Error(), "does not have associated TTL)") {
+		return p.registerWorker()
 	}
 
-	viper.Set("windlass.secret", kv.Value)
 	return nil
 }

app/models/project/project.go

@@ -2,6 +2,7 @@ package project
 
 import (
 	"errors"
+	"net/http"
 	"regexp"
 
 	"github.com/UCCNetworkingSociety/Windlass-worker/app/models/container"
@@ -22,7 +23,7 @@ type Project struct {
 	Containers container.Containers `json:"containers"`
 }
 
-func (p Project) Validate() error {
+func (p *Project) Bind(r *http.Request) error {
 	if !projectName.MatchString(p.Namespace + "_" + p.Name) {
 		return ErrInvalidFormat
 	}

cmd/main.go

@@ -30,6 +30,6 @@ func main() {
 	log.Info("API server started")
 
 	if err := http.ListenAndServe(":"+viper.GetString("http.port"), r); err != nil {
-		log.Error(err, "error starting server")
+		log.WithError(err).Error("error starting server")
 	}
 }

go.mod

@@ -3,6 +3,7 @@ module github.com/UCCNetworkingSociety/Windlass-worker
 go 1.12
 
 require (
+	github.com/UCCNetworkingSociety/Windlass v0.0.0-20190721173644-3b02ae196c9c // indirect
 	github.com/bwmarrin/lit v0.0.0-20190510005413-9c5ce4f3cafc
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1 // indirect
@@ -11,6 +12,7 @@ require (
 	github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 // indirect
 	github.com/go-check/check v1.0.0-20180628173108-788fd7840127 // indirect
 	github.com/go-chi/chi v4.0.2+incompatible
+	github.com/go-chi/render v1.0.1 // indirect
 	github.com/golang/protobuf v1.3.2 // indirect
 	github.com/google/go-cmp v0.3.0 // indirect
 	github.com/hashicorp/consul/api v1.1.0
@@ -22,14 +24,8 @@ require (
 	github.com/juju/webbrowser v0.0.0-20180907093207-efb9432b2bcb // indirect
 	github.com/lxc/lxd v0.0.0-20190717210919-0ecd38c37220
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
-	github.com/pelletier/go-toml v1.3.0 // indirect
 	github.com/rogpeppe/fastuuid v1.2.0 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.4.0
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
-	golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
-	golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect
 	gopkg.in/errgo.v1 v1.0.1 // indirect
 	gopkg.in/httprequest.v1 v1.2.0 // indirect
 	gopkg.in/macaroon-bakery.v2 v2.1.0 // indirect

go.sum

@@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/UCCNetworkingSociety/Windlass v0.0.0-20190721173644-3b02ae196c9c h1:qRU1wihyuzrozQBMfIT5o53Z7vCNA9/PTeyN79w5Xd8=
+github.com/UCCNetworkingSociety/Windlass v0.0.0-20190721173644-3b02ae196c9c/go.mod h1:FBCVM2pRcQlrFx+YENRHAq/FHVYHyyyHi/3/sEaQ3GY=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -50,6 +52,8 @@ github.com/go-check/check v1.0.0-20180628173108-788fd7840127 h1:3dbHpVjNKf7Myfit
 github.com/go-check/check v1.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
 github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=

utils/logging/log.go

@@ -2,7 +2,7 @@ package log
 
 import (
 	"fmt"
-	"os"
+	"sort"
 	"strings"
 
 	"github.com/bwmarrin/lit"
@@ -16,6 +16,7 @@ type Fields map[string]interface{}
 
 type Entry struct {
 	fields Fields
+	err    error
 }
 
 var emptyEntry = &Entry{}
@@ -30,18 +31,43 @@ func init() {
 }
 
 func WithFields(f Fields) *Entry {
-	return &Entry{f}
+	return &Entry{fields: f}
+}
+
+func (e *Entry) WithFields(f Fields) *Entry {
+	if e.fields == nil {
+		e.fields = make(Fields)
+	}
+	for k, v := range f {
+		e.fields[k] = v
+	}
+	return e
 }
 
 func (f Fields) format() string {
-	if f == nil {
+	if f == nil || len(f) == 0 {
 		return ""
 	}
 
+	keys := make([]string, 0, len(f))
+	for k := range f {
+		keys = append(keys, k)
+	}
+
+	sort.Strings(keys)
+
 	var builder strings.Builder
 	builder.WriteRune('\t')
-	for k, v := range f {
-		builder.WriteString(fmt.Sprintf("%s=%v", k, v))
+
+	var iterCount int
+	for _, k := range keys {
+		v := f[k]
+		s := fmt.Sprintf("%s=%v", k, v)
+		if iterCount < len(f)-1 {
+			s += " "
+		}
+		builder.WriteString(s)
+		iterCount++
 	}
 	builder.WriteRune('\n')
 	return builder.String()
@@ -52,34 +78,53 @@ func Debug(format string, a ...interface{}) {
 }
 
 func (e *Entry) Debug(format string, a ...interface{}) {
-	lit.Custom(os.Stdout, lit.LogDebug, callDepth, format, a...)
-	fmt.Print(e.fields.format())
+	builder := new(strings.Builder)
+	lit.Custom(builder, lit.LogDebug, callDepth, format, a...)
+	builder.WriteString(e.fields.format())
+	fmt.Print(builder.String())
 }
 
 func Info(format string, a ...interface{}) {
 	emptyEntry.Info(format, a...)
 }
 
 func (e *Entry) Info(format string, a ...interface{}) {
-	lit.Custom(os.Stdout, lit.LogInformational, callDepth, format, a...)
-	fmt.Print(e.fields.format())
+	builder := new(strings.Builder)
+	lit.Custom(builder, lit.LogInformational, callDepth, format, a...)
+	builder.WriteString(e.fields.format())
+	fmt.Print(builder.String())
 }
 
 func Warn(format string, a ...interface{}) {
 	emptyEntry.Warn(format, a...)
 }
 
 func (e *Entry) Warn(format string, a ...interface{}) {
-	lit.Custom(os.Stdout, lit.LogWarning, callDepth, format, a...)
-	fmt.Print(e.fields.format())
+	builder := new(strings.Builder)
+	lit.Custom(builder, lit.LogWarning, callDepth, format, a...)
+	builder.WriteString(e.fields.format())
+	fmt.Print(builder.String())
+}
+
+func Error(format string, a ...interface{}) {
+	emptyEntry.Error(format, a...)
+}
+
+func (e *Entry) Error(format string, a ...interface{}) {
+	if e.err != nil {
+		e.fields["error"] = e.err
+	}
+	builder := new(strings.Builder)
+	lit.Custom(builder, lit.LogError, callDepth, format, a...)
+	builder.WriteString(e.fields.format())
+	fmt.Print(builder.String())
 }
 
-func Error(err error, format string, a ...interface{}) {
-	emptyEntry.Error(err, format, a...)
+func WithError(err error) *Entry {
+	return &Entry{err: err}
 }
 
-func (e *Entry) Error(err error, format string, a ...interface{}) {
-	format += fmt.Sprintf(": %v", err)
-	lit.Custom(os.Stderr, lit.LogError, callDepth, format, a...)
-	fmt.Print(e.fields.format())
+func (e *Entry) WithError(err error) *Entry {
+	e.err = err
+	return e
 }