From 3df7eb05c5f0c0d79c18bca01795cca3871e8874 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sat, 7 Mar 2020 16:45:30 -0500 Subject: [PATCH 01/15] Updated client-go to v0.17.0 --- glide.lock | 324 ++++++++++++++++----------------- glide.yaml | 13 +- reaper/reaper.go | 7 +- rules/chaos.go | 2 +- rules/chaos_test.go | 2 +- rules/container_status.go | 2 +- rules/container_status_test.go | 2 +- rules/duration.go | 2 +- rules/duration_test.go | 6 +- rules/pod_status.go | 2 +- rules/pod_status_test.go | 2 +- rules/rules.go | 2 +- rules/rules_test.go | 9 +- rules/unready.go | 2 +- rules/unready_test.go | 6 +- 15 files changed, 182 insertions(+), 201 deletions(-) diff --git a/glide.lock b/glide.lock index e194322..412d7c6 100644 --- a/glide.lock +++ b/glide.lock @@ -1,223 +1,155 @@ -hash: 956aef653d24b11b29e6ac247018b5d5ce43e47395a1637b9dd0bd3fee9c4e5a -updated: 2020-02-28T17:08:38.4997436Z +hash: b7c11caa604dd7eb1eba1de72321086b340f380d4ccfec817d3882d897d89115 +updated: 2020-03-07T16:43:00.6334477-05:00 imports: -- name: cloud.google.com/go - version: 3b1ae45394a234c385be014e9a488f2bb6eef821 - subpackages: - - compute/metadata - - internal -- name: github.com/blang/semver - version: 31b736133b98f26d5e078ec9eb591666edfd091f -- name: github.com/coreos/go-oidc - version: 5644a2f50e2d2d5ba0b474bc5bc55fea1925936d - subpackages: - - http - - jose - - key - - oauth2 - - oidc -- name: github.com/coreos/pkg - version: fa29b1d70f0beaddd4c7021607cc3c3be8ce94b8 - subpackages: - - health - - httputil - - timeutil - name: github.com/davecgh/go-spew - version: 346938d642f2ec3594ed81d874461961cd0faa76 + version: 8991bc29aa16c548c550c7ff78260e27b9ab7c73 subpackages: - spew - name: github.com/docker/distribution version: 2461543d988979529609e8cb6fca9ca190dc48da - subpackages: - - digestset - - reference -- name: github.com/emicklei/go-restful - version: 09691a3b6378b740595c1002f40c34dd5f218a22 - subpackages: - - log - - swagger -- name: github.com/ghodss/yaml - version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee -- name: github.com/go-openapi/jsonpointer - version: 46af16f9f7b149af66e5d1bd010e3574dc06de98 -- name: github.com/go-openapi/jsonreference - version: 13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272 -- name: github.com/go-openapi/spec - version: 6aced65f8501fe1217321abf0749d354824ba2ff -- name: github.com/go-openapi/swag - version: 1d0bd113de87027671077d3c71eb3ac5d7dbba72 - name: github.com/gogo/protobuf - version: c0656edd0d9eab7c66d1eb0c568f9039345796f7 + version: 65acae22fc9d1fe290b33faa2bd64cdc20a463a0 subpackages: - proto - sortkeys -- name: github.com/golang/glog - version: 44145f04b68cf362d9c4df2182967c2275eaefed - name: github.com/golang/protobuf - version: 4bd1920723d7b7c925de087aa32e2187708897f7 + version: 6c65a5562fc06764971b7c5d05c76c75e84bdbf7 subpackages: - proto + - ptypes + - ptypes/any + - ptypes/duration + - ptypes/timestamp - name: github.com/google/gofuzz - version: 44d81051d367757e1c7c6a5a86423ece9afcf63c -- name: github.com/jonboulle/clockwork - version: 72f9bd7c4e0c2a40055ab3d0f09654f730cce982 + version: f140a6486e521aad38f5917de355cbf147cc0496 +- name: github.com/googleapis/gnostic + version: 0c5108395e2debce0d731cf0287ddf7242066aba + subpackages: + - OpenAPIv2 + - compiler + - extensions - name: github.com/joonix/log version: f5f056244ba320717491aa9a073a2cb4fdc2ff30 -- name: github.com/juju/ratelimit - version: 77ed1c8a01217656d2080ad51981f6e99adaa177 +- name: github.com/json-iterator/go + version: 03217c3e97663914aec3faafde50d081f197a0a2 - name: github.com/konsorten/go-windows-terminal-sequences version: f55edac94c9bbba5d6182a4be46d86a2c9b5b50e -- name: github.com/mailru/easyjson - version: d5b7844b561a7bc640052f1b935f7b800330d7e0 - subpackages: - - buffer - - jlexer - - jwriter -- name: github.com/opencontainers/go-digest - version: dd78d7521eee4ff6016d9b6273dc328308ac5a1c -- name: github.com/pborman/uuid - version: ca53cad383cad2479bbba7f7a1a05797ec1386e4 -- name: github.com/PuerkitoBio/purell - version: 8a290539e2e8629dbc4e6bad948158f790ec31f4 -- name: github.com/PuerkitoBio/urlesc - version: 5bd2802263f21d8788851d5305584c82a5c75d7e +- name: github.com/modern-go/concurrent + version: bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 +- name: github.com/modern-go/reflect2 + version: 94122c33edd36123c84d5368cfb2b69df93a0ec8 - name: github.com/robfig/cron version: b41be1df696709bb6395fe435af20370037c0b4c - name: github.com/sirupsen/logrus version: 839c75faf7f98a33d445d181f3018b5c3409a45e -- name: github.com/spf13/pflag - version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 - name: github.com/stretchr/testify version: 3ebf1ddaeb260c4b1ae502a01c7844fa8c1fa0e9 subpackages: - assert -- name: github.com/ugorji/go - version: ded73eae5db7e7a0ef6f55aace87a2873c5d2b74 +- name: golang.org/x/crypto + version: c2843e01d9a2bc60bb26ad24e09734fdc2d9ec58 subpackages: - - codec + - ssh/terminal - name: golang.org/x/net - version: f2499483f923065a842d38eb4c7f1927e6fc6e6d + version: 13f9640d40b9cc418fb53703dfbd177679788ceb subpackages: - context - context/ctxhttp + - http/httpguts - http2 - http2/hpack - idna - - lex/httplex - name: golang.org/x/oauth2 - version: 3c3a985cb79f52a3190fbc056984415ca6763d01 + version: 0f29369cfe4552d0e4bcddc57cc75f4d7e672a33 subpackages: - - google - internal - - jws - - jwt - name: golang.org/x/sys - version: d5e6a3e2c0ae16fc7480523ebcb7fd4dd3215489 + version: fde4db37ae7ad8191b03d30d27f258b5291ae4e3 subpackages: - unix + - windows - name: golang.org/x/text - version: 2910a502d2bf9e43193af9d68ca516529614eed3 + version: 342b2e1fbaa52c93f31447ad2c6abc048c63e475 subpackages: - - cases - - internal/tag - - language - - runes - secure/bidirule - - secure/precis - transform - unicode/bidi - unicode/norm - - width +- name: golang.org/x/time + version: 9d24e82272b4f38b78bc8cff74fa936d31ccd8ef + subpackages: + - rate - name: google.golang.org/appengine - version: 4f7eeb5305a4ba1966344836ba4af9996b7b4e05 + version: 54a98f90d1c46b7731eb8fb305d2a321c30ef610 subpackages: - internal - - internal/app_identity - internal/base - internal/datastore - internal/log - - internal/modules - internal/remote_api - internal/urlfetch - urlfetch - name: gopkg.in/inf.v0 - version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 + version: d2d2541c53f18d2a059457998ce2876cc8e67cbf - name: gopkg.in/yaml.v2 - version: 53feefa2559fb8dfa8d81baad31be332c97d6c77 + version: f221b8435cfb71e54062f6c6e99e9ade30b124d5 +- name: k8s.io/api + version: 4c9a86741a7ab3890dd9e0777e85d8eee48bf59c + subpackages: + - admissionregistration/v1 + - admissionregistration/v1beta1 + - apps/v1 + - apps/v1beta1 + - apps/v1beta2 + - auditregistration/v1alpha1 + - authentication/v1 + - authentication/v1beta1 + - authorization/v1 + - authorization/v1beta1 + - autoscaling/v1 + - autoscaling/v2beta1 + - autoscaling/v2beta2 + - batch/v1 + - batch/v1beta1 + - batch/v2alpha1 + - certificates/v1beta1 + - coordination/v1 + - coordination/v1beta1 + - core/v1 + - discovery/v1alpha1 + - discovery/v1beta1 + - events/v1beta1 + - extensions/v1beta1 + - flowcontrol/v1alpha1 + - networking/v1 + - networking/v1beta1 + - node/v1alpha1 + - node/v1beta1 + - policy/v1beta1 + - rbac/v1 + - rbac/v1alpha1 + - rbac/v1beta1 + - scheduling/v1 + - scheduling/v1alpha1 + - scheduling/v1beta1 + - settings/v1alpha1 + - storage/v1 + - storage/v1alpha1 + - storage/v1beta1 - name: k8s.io/apimachinery - version: c1b2a29e2e9d121b9b7d818cb0d84d8098783236 - subpackages: - - pkg/labels - - pkg/selection - - pkg/util/sets - - pkg/util/validation -- name: k8s.io/client-go - version: e121606b0d09b2e1c467183ee46217fa85a6b672 + version: 79c2a76c473a20cdc4ce59cae4b72529b5d9d16b subpackages: - - discovery - - kubernetes - - kubernetes/typed/apps/v1beta1 - - kubernetes/typed/authentication/v1beta1 - - kubernetes/typed/authorization/v1beta1 - - kubernetes/typed/autoscaling/v1 - - kubernetes/typed/batch/v1 - - kubernetes/typed/batch/v2alpha1 - - kubernetes/typed/certificates/v1alpha1 - - kubernetes/typed/core/v1 - - kubernetes/typed/extensions/v1beta1 - - kubernetes/typed/policy/v1beta1 - - kubernetes/typed/rbac/v1alpha1 - - kubernetes/typed/storage/v1beta1 - - pkg/api - pkg/api/errors - - pkg/api/install - pkg/api/meta - - pkg/api/meta/metatypes - pkg/api/resource - - pkg/api/unversioned - - pkg/api/v1 - - pkg/api/validation/path - - pkg/apimachinery - - pkg/apimachinery/announced - - pkg/apimachinery/registered - - pkg/apis/apps - - pkg/apis/apps/install - - pkg/apis/apps/v1beta1 - - pkg/apis/authentication - - pkg/apis/authentication/install - - pkg/apis/authentication/v1beta1 - - pkg/apis/authorization - - pkg/apis/authorization/install - - pkg/apis/authorization/v1beta1 - - pkg/apis/autoscaling - - pkg/apis/autoscaling/install - - pkg/apis/autoscaling/v1 - - pkg/apis/batch - - pkg/apis/batch/install - - pkg/apis/batch/v1 - - pkg/apis/batch/v2alpha1 - - pkg/apis/certificates - - pkg/apis/certificates/install - - pkg/apis/certificates/v1alpha1 - - pkg/apis/extensions - - pkg/apis/extensions/install - - pkg/apis/extensions/v1beta1 - - pkg/apis/policy - - pkg/apis/policy/install - - pkg/apis/policy/v1beta1 - - pkg/apis/rbac - - pkg/apis/rbac/install - - pkg/apis/rbac/v1alpha1 - - pkg/apis/storage - - pkg/apis/storage/install - - pkg/apis/storage/v1beta1 - - pkg/auth/user + - pkg/apis/meta/v1 + - pkg/apis/meta/v1/unstructured - pkg/conversion - pkg/conversion/queryparams - pkg/fields - - pkg/genericapiserver/openapi/common - pkg/labels - pkg/runtime + - pkg/runtime/schema - pkg/runtime/serializer - pkg/runtime/serializer/json - pkg/runtime/serializer/protobuf @@ -225,42 +157,94 @@ imports: - pkg/runtime/serializer/streaming - pkg/runtime/serializer/versioning - pkg/selection - - pkg/third_party/forked/golang/reflect - - pkg/third_party/forked/golang/template - pkg/types - - pkg/util - - pkg/util/cert - pkg/util/clock - pkg/util/errors - - pkg/util/flowcontrol - pkg/util/framer - - pkg/util/integer - pkg/util/intstr - pkg/util/json - - pkg/util/jsonpath - - pkg/util/labels + - pkg/util/naming - pkg/util/net - - pkg/util/parsers - - pkg/util/rand - pkg/util/runtime - pkg/util/sets - - pkg/util/uuid - pkg/util/validation - pkg/util/validation/field - - pkg/util/wait - pkg/util/yaml - pkg/version - pkg/watch - - pkg/watch/versioned - - plugin/pkg/client/auth - - plugin/pkg/client/auth/gcp - - plugin/pkg/client/auth/oidc + - third_party/forked/golang/reflect +- name: k8s.io/client-go + version: c68b62b1efa14564a47d67c07f013dc3553937b9 + subpackages: + - api/core/v1 + - discovery + - kubernetes + - kubernetes/scheme + - kubernetes/typed/admissionregistration/v1 + - kubernetes/typed/admissionregistration/v1beta1 + - kubernetes/typed/apps/v1 + - kubernetes/typed/apps/v1beta1 + - kubernetes/typed/apps/v1beta2 + - kubernetes/typed/auditregistration/v1alpha1 + - kubernetes/typed/authentication/v1 + - kubernetes/typed/authentication/v1beta1 + - kubernetes/typed/authorization/v1 + - kubernetes/typed/authorization/v1beta1 + - kubernetes/typed/autoscaling/v1 + - kubernetes/typed/autoscaling/v2beta1 + - kubernetes/typed/autoscaling/v2beta2 + - kubernetes/typed/batch/v1 + - kubernetes/typed/batch/v1beta1 + - kubernetes/typed/batch/v2alpha1 + - kubernetes/typed/certificates/v1beta1 + - kubernetes/typed/coordination/v1 + - kubernetes/typed/coordination/v1beta1 + - kubernetes/typed/core/v1 + - kubernetes/typed/discovery/v1alpha1 + - kubernetes/typed/discovery/v1beta1 + - kubernetes/typed/events/v1beta1 + - kubernetes/typed/extensions/v1beta1 + - kubernetes/typed/flowcontrol/v1alpha1 + - kubernetes/typed/networking/v1 + - kubernetes/typed/networking/v1beta1 + - kubernetes/typed/node/v1alpha1 + - kubernetes/typed/node/v1beta1 + - kubernetes/typed/policy/v1beta1 + - kubernetes/typed/rbac/v1 + - kubernetes/typed/rbac/v1alpha1 + - kubernetes/typed/rbac/v1beta1 + - kubernetes/typed/scheduling/v1 + - kubernetes/typed/scheduling/v1alpha1 + - kubernetes/typed/scheduling/v1beta1 + - kubernetes/typed/settings/v1alpha1 + - kubernetes/typed/storage/v1 + - kubernetes/typed/storage/v1alpha1 + - kubernetes/typed/storage/v1beta1 + - pkg/apis/clientauthentication + - pkg/apis/clientauthentication/v1alpha1 + - pkg/apis/clientauthentication/v1beta1 + - pkg/version + - plugin/pkg/client/auth/exec - rest + - rest/watch - tools/clientcmd/api - tools/metrics + - tools/reference - transport + - util/cert + - util/connrotation + - util/flowcontrol + - util/keyutil +- name: k8s.io/klog + version: 2ca9ad30301bf30a8a6e0fa2110db6b8df699a91 +- name: k8s.io/utils + version: e782cd3c129fc98ee807f3c889c0f26eb7c9daf5 + subpackages: + - integer +- name: sigs.k8s.io/yaml + version: fd68e9863619f6ec2fdd8625fe1f02e7c877e480 testImports: - name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d + version: 792786c7400a136282c1664665ae0a8db921c6c2 subpackages: - difflib diff --git a/glide.yaml b/glide.yaml index ac3135b..1a8bc8e 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,21 +1,16 @@ package: github.com/target/pod-reaper import: - package: k8s.io/apimachinery - version: c1b2a29e2e9d121b9b7d818cb0d84d8098783236 + version: =v0.17.0 subpackages: - /pkg/labels - /pkg/selection - package: k8s.io/client-go - version: =v2.0.0 + version: =v0.17.0 subpackages: - kubernetes - - pkg/api - - pkg/api/unversioned + - core/v1 - rest -- package: k8s.io/client-go - version: =v2.0.0 - subpackages: - - pkg/api/unversioned - package: github.com/stretchr/testify version: ^1.1.4 - package: github.com/robfig/cron @@ -27,4 +22,4 @@ import: - package: github.com/joonix/log version: =v0.1 - package: github.com/davecgh/go-spew - version: =v1.1.0 + version: ^v1.1.1 diff --git a/reaper/reaper.go b/reaper/reaper.go index 9336bcc..2a14642 100644 --- a/reaper/reaper.go +++ b/reaper/reaper.go @@ -5,9 +5,10 @@ import ( "github.com/robfig/cron" "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/rest" ) @@ -46,7 +47,7 @@ func newReaper() reaper { func (reaper reaper) getPods() *v1.PodList { coreClient := reaper.clientSet.CoreV1() pods := coreClient.Pods(reaper.options.namespace) - listOptions := v1.ListOptions{} + listOptions := metav1.ListOptions{} if reaper.options.labelExclusion != nil || reaper.options.labelRequirement != nil { selector := labels.NewSelector() if reaper.options.labelExclusion != nil { @@ -70,7 +71,7 @@ func (reaper reaper) reapPod(pod v1.Pod, reasons []string) { "pod": pod.Name, "reasons": reasons, }).Info("reaping pod") - deleteOptions := &v1.DeleteOptions{ + deleteOptions := &metav1.DeleteOptions{ GracePeriodSeconds: reaper.options.gracePeriod, } err := reaper.clientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, deleteOptions) diff --git a/rules/chaos.go b/rules/chaos.go index 154ee02..12973bd 100644 --- a/rules/chaos.go +++ b/rules/chaos.go @@ -7,7 +7,7 @@ import ( "strconv" "time" - "k8s.io/client-go/pkg/api/v1" + "k8s.io/api/core/v1" ) const envChaosChance = "CHAOS_CHANCE" diff --git a/rules/chaos_test.go b/rules/chaos_test.go index 1840f9a..0330416 100644 --- a/rules/chaos_test.go +++ b/rules/chaos_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "k8s.io/client-go/pkg/api/v1" + v1 "k8s.io/api/core/v1" ) func TestChaosLoad(t *testing.T) { diff --git a/rules/container_status.go b/rules/container_status.go index 4d0ce8b..57fc7e0 100644 --- a/rules/container_status.go +++ b/rules/container_status.go @@ -5,7 +5,7 @@ import ( "os" "strings" - "k8s.io/client-go/pkg/api/v1" + "k8s.io/api/core/v1" ) const envContainerStatus = "CONTAINER_STATUSES" diff --git a/rules/container_status_test.go b/rules/container_status_test.go index 49169e9..5b8515f 100644 --- a/rules/container_status_test.go +++ b/rules/container_status_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "k8s.io/client-go/pkg/api/v1" + "k8s.io/api/core/v1" ) func testWaitContainerState(reason string) v1.ContainerState { diff --git a/rules/duration.go b/rules/duration.go index 9cc9773..95b0359 100644 --- a/rules/duration.go +++ b/rules/duration.go @@ -5,7 +5,7 @@ import ( "os" "time" - "k8s.io/client-go/pkg/api/v1" + "k8s.io/api/core/v1" ) const envMaxDuration = "MAX_DURATION" diff --git a/rules/duration_test.go b/rules/duration_test.go index 4825cad..e703f5a 100644 --- a/rules/duration_test.go +++ b/rules/duration_test.go @@ -6,14 +6,14 @@ import ( "time" "github.com/stretchr/testify/assert" - "k8s.io/client-go/pkg/api/unversioned" - "k8s.io/client-go/pkg/api/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func testDurationPod(startTime *time.Time) v1.Pod { pod := v1.Pod{} if startTime != nil { - setTime := unversioned.NewTime(*startTime) + setTime := metav1.NewTime(*startTime) pod.Status.StartTime = &setTime } return pod diff --git a/rules/pod_status.go b/rules/pod_status.go index fb7fa71..eb5ec3a 100644 --- a/rules/pod_status.go +++ b/rules/pod_status.go @@ -5,7 +5,7 @@ import ( "os" "strings" - "k8s.io/client-go/pkg/api/v1" + "k8s.io/api/core/v1" ) const envPodStatus = "POD_STATUSES" diff --git a/rules/pod_status_test.go b/rules/pod_status_test.go index 0f92541..4b83b27 100644 --- a/rules/pod_status_test.go +++ b/rules/pod_status_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "k8s.io/client-go/pkg/api/v1" + "k8s.io/api/core/v1" ) func testPodFromReason(reason string) v1.Pod { diff --git a/rules/rules.go b/rules/rules.go index 19d2ed1..8307532 100644 --- a/rules/rules.go +++ b/rules/rules.go @@ -4,7 +4,7 @@ import ( "errors" "github.com/sirupsen/logrus" - "k8s.io/client-go/pkg/api/v1" + "k8s.io/api/core/v1" ) // Rule is an interface defining the two functions needed for pod reaper to use the rule. diff --git a/rules/rules_test.go b/rules/rules_test.go index 1cf55a0..11a20bc 100644 --- a/rules/rules_test.go +++ b/rules/rules_test.go @@ -5,11 +5,12 @@ import ( "testing" "time" + "io/ioutil" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "io/ioutil" - "k8s.io/client-go/pkg/api/unversioned" - "k8s.io/client-go/pkg/api/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func init() { @@ -17,7 +18,7 @@ func init() { } func testPod() v1.Pod { - startTime := unversioned.NewTime(time.Now().Add(-2 * time.Minute)) + startTime := metav1.NewTime(time.Now().Add(-2 * time.Minute)) return v1.Pod{ Status: v1.PodStatus{ StartTime: &startTime, diff --git a/rules/unready.go b/rules/unready.go index 048448d..511873d 100644 --- a/rules/unready.go +++ b/rules/unready.go @@ -5,7 +5,7 @@ import ( "os" "time" - "k8s.io/client-go/pkg/api/v1" + "k8s.io/api/core/v1" ) const envMaxUnready = "MAX_UNREADY" diff --git a/rules/unready_test.go b/rules/unready_test.go index bc47c60..6be44ee 100644 --- a/rules/unready_test.go +++ b/rules/unready_test.go @@ -6,14 +6,14 @@ import ( "time" "github.com/stretchr/testify/assert" - "k8s.io/client-go/pkg/api/unversioned" - "k8s.io/client-go/pkg/api/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func testUnreadyPod(lastTransitionTime *time.Time) v1.Pod { pod := v1.Pod{} if lastTransitionTime != nil { - setTime := unversioned.NewTime(*lastTransitionTime) + setTime := metav1.NewTime(*lastTransitionTime) pod.Status.Conditions = []v1.PodCondition{ v1.PodCondition{ Type: v1.PodReady, From 35591a451190617505ff321cb49bef992a57be7a Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sun, 8 Mar 2020 14:21:23 -0400 Subject: [PATCH 02/15] Standardized project layout https://github.com/golang-standards/project-layout --- Makefile | 21 +++ {rules => internal/pkg/rules}/chaos.go | 0 {rules => internal/pkg/rules}/chaos_test.go | 0 .../pkg/rules}/container_status.go | 0 .../pkg/rules}/container_status_test.go | 0 {rules => internal/pkg/rules}/duration.go | 0 .../pkg/rules}/duration_test.go | 0 {rules => internal/pkg/rules}/pod_status.go | 0 .../pkg/rules}/pod_status_test.go | 0 {rules => internal/pkg/rules}/rules.go | 0 {rules => internal/pkg/rules}/rules_test.go | 0 {rules => internal/pkg/rules}/unready.go | 0 {rules => internal/pkg/rules}/unready_test.go | 0 reaper/main.go | 52 ----- reaper/main_test.go | 71 ------- reaper/options.go | 138 -------------- reaper/options_test.go | 178 ------------------ reaper/reaper.go | 117 ------------ 18 files changed, 21 insertions(+), 556 deletions(-) create mode 100644 Makefile rename {rules => internal/pkg/rules}/chaos.go (100%) rename {rules => internal/pkg/rules}/chaos_test.go (100%) rename {rules => internal/pkg/rules}/container_status.go (100%) rename {rules => internal/pkg/rules}/container_status_test.go (100%) rename {rules => internal/pkg/rules}/duration.go (100%) rename {rules => internal/pkg/rules}/duration_test.go (100%) rename {rules => internal/pkg/rules}/pod_status.go (100%) rename {rules => internal/pkg/rules}/pod_status_test.go (100%) rename {rules => internal/pkg/rules}/rules.go (100%) rename {rules => internal/pkg/rules}/rules_test.go (100%) rename {rules => internal/pkg/rules}/unready.go (100%) rename {rules => internal/pkg/rules}/unready_test.go (100%) delete mode 100644 reaper/main.go delete mode 100644 reaper/main_test.go delete mode 100644 reaper/options.go delete mode 100644 reaper/options_test.go delete mode 100644 reaper/reaper.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6e95a61 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +.PHONY: test + +VERSION?=$(shell git describe --tags) +IMAGE:=pod-reaper:$(VERSION) + +all: build + +build: + CGO_ENABLED=0 go build -o _output/bin/pod-reaper github.com/target/pod-reaper/cmd/pod-reaper + +image: + docker build -t $(IMAGE) . + +clean: + rm -rf _output + +test-unit: + ./test/run-unit-tests.sh + +test-e2e: + ./test/run-e2e-tests.sh \ No newline at end of file diff --git a/rules/chaos.go b/internal/pkg/rules/chaos.go similarity index 100% rename from rules/chaos.go rename to internal/pkg/rules/chaos.go diff --git a/rules/chaos_test.go b/internal/pkg/rules/chaos_test.go similarity index 100% rename from rules/chaos_test.go rename to internal/pkg/rules/chaos_test.go diff --git a/rules/container_status.go b/internal/pkg/rules/container_status.go similarity index 100% rename from rules/container_status.go rename to internal/pkg/rules/container_status.go diff --git a/rules/container_status_test.go b/internal/pkg/rules/container_status_test.go similarity index 100% rename from rules/container_status_test.go rename to internal/pkg/rules/container_status_test.go diff --git a/rules/duration.go b/internal/pkg/rules/duration.go similarity index 100% rename from rules/duration.go rename to internal/pkg/rules/duration.go diff --git a/rules/duration_test.go b/internal/pkg/rules/duration_test.go similarity index 100% rename from rules/duration_test.go rename to internal/pkg/rules/duration_test.go diff --git a/rules/pod_status.go b/internal/pkg/rules/pod_status.go similarity index 100% rename from rules/pod_status.go rename to internal/pkg/rules/pod_status.go diff --git a/rules/pod_status_test.go b/internal/pkg/rules/pod_status_test.go similarity index 100% rename from rules/pod_status_test.go rename to internal/pkg/rules/pod_status_test.go diff --git a/rules/rules.go b/internal/pkg/rules/rules.go similarity index 100% rename from rules/rules.go rename to internal/pkg/rules/rules.go diff --git a/rules/rules_test.go b/internal/pkg/rules/rules_test.go similarity index 100% rename from rules/rules_test.go rename to internal/pkg/rules/rules_test.go diff --git a/rules/unready.go b/internal/pkg/rules/unready.go similarity index 100% rename from rules/unready.go rename to internal/pkg/rules/unready.go diff --git a/rules/unready_test.go b/internal/pkg/rules/unready_test.go similarity index 100% rename from rules/unready_test.go rename to internal/pkg/rules/unready_test.go diff --git a/reaper/main.go b/reaper/main.go deleted file mode 100644 index 3838af7..0000000 --- a/reaper/main.go +++ /dev/null @@ -1,52 +0,0 @@ -package main - -import ( - "os" - - joonix "github.com/joonix/log" - "github.com/sirupsen/logrus" -) - -const envLogLevel = "LOG_LEVEL" -const envLogFormat = "LOG_FORMAT" -const fluentdFormat = "Fluentd" -const logrusFormat = "Logrus" -const defaultLogLevel = logrus.InfoLevel - -func main() { - logLevel := getLogLevel() - logrus.SetLevel(logLevel) - logFormat := getLogFormat() - logrus.SetFormatter(logFormat) - - reaper := newReaper() - reaper.harvest() - logrus.Info("pod reaper is exiting") -} - -func getLogLevel() logrus.Level { - levelString, exists := os.LookupEnv(envLogLevel) - if !exists { - return defaultLogLevel - } - - level, err := logrus.ParseLevel(levelString) - if err != nil { - logrus.Errorf("error parsing %s: %v", envLogLevel, err) - return defaultLogLevel - } - - return level -} - -func getLogFormat() logrus.Formatter { - formatString, exists := os.LookupEnv(envLogFormat) - if !exists || formatString == logrusFormat { - return &logrus.JSONFormatter{} - } else if formatString == fluentdFormat { - return &joonix.FluentdFormatter{} - } else { - logrus.Errorf("unknown %s: %v", envLogFormat, formatString) - return &logrus.JSONFormatter{} - } -} diff --git a/reaper/main_test.go b/reaper/main_test.go deleted file mode 100644 index cce9753..0000000 --- a/reaper/main_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "os" - "reflect" - "testing" - - joonix "github.com/joonix/log" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" -) - -var levelTests = []struct { - str string - level logrus.Level -}{ - {"PANIC", logrus.PanicLevel}, - {"FATAL", logrus.FatalLevel}, - {"ERROR", logrus.ErrorLevel}, - {"WARNING", logrus.WarnLevel}, - {"DEBUG", logrus.DebugLevel}, - {"INFO", logrus.InfoLevel}, -} - -func TestGetLogLevel(t *testing.T) { - t.Run("default not set", func(t *testing.T) { - os.Clearenv() - level := getLogLevel() - assert.Equal(t, level, defaultLogLevel) - }) - t.Run("default invalid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envLogLevel, "foo") - level := getLogLevel() - assert.Equal(t, level, defaultLogLevel) - }) - for _, tt := range levelTests { - t.Run(tt.str, func(t *testing.T) { - os.Clearenv() - os.Setenv(envLogLevel, tt.str) - level := getLogLevel() - assert.Equal(t, level, tt.level) - }) - } -} - -func TestGetLogFormat(t *testing.T) { - t.Run("default not set", func(t *testing.T) { - os.Clearenv() - format := getLogFormat() - assert.Equal(t, reflect.TypeOf(format), reflect.TypeOf(&logrus.JSONFormatter{})) - }) - t.Run("default invalid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envLogFormat, "foo") - format := getLogFormat() - assert.Equal(t, reflect.TypeOf(format), reflect.TypeOf(&logrus.JSONFormatter{})) - }) - t.Run("logrus", func(t *testing.T) { - os.Clearenv() - os.Setenv(envLogFormat, "Logrus") - format := getLogFormat() - assert.Equal(t, reflect.TypeOf(format), reflect.TypeOf(&logrus.JSONFormatter{})) - }) - t.Run("fluentd", func(t *testing.T) { - os.Clearenv() - os.Setenv(envLogFormat, "Fluentd") - format := getLogFormat() - assert.Equal(t, reflect.TypeOf(format), reflect.TypeOf(&joonix.FluentdFormatter{})) - }) -} diff --git a/reaper/options.go b/reaper/options.go deleted file mode 100644 index ab83cf8..0000000 --- a/reaper/options.go +++ /dev/null @@ -1,138 +0,0 @@ -package main - -import ( - "fmt" - "os" - "strings" - "time" - - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - - "github.com/target/pod-reaper/rules" -) - -// environment variable names -const envNamespace = "NAMESPACE" -const envGracePeriod = "GRACE_PERIOD" -const envScheduleCron = "SCHEDULE" -const envRunDuration = "RUN_DURATION" -const envExcludeLabelKey = "EXCLUDE_LABEL_KEY" -const envExcludeLabelValues = "EXCLUDE_LABEL_VALUES" -const envRequireLabelKey = "REQUIRE_LABEL_KEY" -const envRequireLabelValues = "REQUIRE_LABEL_VALUES" - -type options struct { - namespace string - gracePeriod *int64 - schedule string - runDuration time.Duration - labelExclusion *labels.Requirement - labelRequirement *labels.Requirement - rules rules.Rules -} - -func namespace() string { - return os.Getenv(envNamespace) -} - -func gracePeriod() (*int64, error) { - envGraceDuration, exists := os.LookupEnv(envGracePeriod) - if !exists { - return nil, nil - } - duration, err := time.ParseDuration(envGraceDuration) - if err != nil { - return nil, fmt.Errorf("invalid %s: %s", envGracePeriod, err) - } - seconds := int64(duration.Seconds()) - return &seconds, nil -} - -func envDuration(key string, defValue string) (time.Duration, error) { - envDuration, exists := os.LookupEnv(key) - if !exists { - envDuration = defValue - } - duration, err := time.ParseDuration(envDuration) - if err != nil { - return duration, fmt.Errorf("invalid %s: %s", key, err) - } - return duration, nil -} - -func schedule() string { - schedule, exists := os.LookupEnv(envScheduleCron) - if !exists { - schedule = "@every 1m" - } - return schedule -} - -func runDuration() (time.Duration, error) { - return envDuration(envRunDuration, "0s") -} - -func labelExclusion() (*labels.Requirement, error) { - labelKey, labelKeyExists := os.LookupEnv(envExcludeLabelKey) - labelValue, labelValuesExist := os.LookupEnv(envExcludeLabelValues) - if labelKeyExists && !labelValuesExist { - return nil, fmt.Errorf("specified %s but not %s", envExcludeLabelKey, envExcludeLabelValues) - } else if !labelKeyExists && labelValuesExist { - return nil, fmt.Errorf("did not specify %s but did specify %s", envExcludeLabelKey, envExcludeLabelValues) - } else if !labelKeyExists && !labelValuesExist { - return nil, nil - } - labelValues := strings.Split(labelValue, ",") - labelExclusion, err := labels.NewRequirement(labelKey, selection.NotIn, labelValues) - if err != nil { - return nil, fmt.Errorf("could not create exclusion label: %s", err) - } - return labelExclusion, nil -} - -func labelRequirement() (*labels.Requirement, error) { - labelKey, labelKeyExists := os.LookupEnv(envRequireLabelKey) - labelValue, labelValuesExist := os.LookupEnv(envRequireLabelValues) - if labelKeyExists && !labelValuesExist { - return nil, fmt.Errorf("specified %s but not %s", envRequireLabelKey, envRequireLabelValues) - } else if !labelKeyExists && labelValuesExist { - return nil, fmt.Errorf("did not specify %s but did specify %s", envRequireLabelKey, envRequireLabelValues) - } else if !labelKeyExists && !labelValuesExist { - return nil, nil - } - labelValues := strings.Split(labelValue, ",") - labelRequirement, err := labels.NewRequirement(labelKey, selection.In, labelValues) - if err != nil { - return nil, fmt.Errorf("could not create requirement label: %s", err) - } - return labelRequirement, nil -} - -func loadOptions() (options options, err error) { - options.namespace = namespace() - options.gracePeriod, err = gracePeriod() - if err != nil { - return options, err - } - options.schedule = schedule() - options.runDuration, err = runDuration() - if err != nil { - return options, err - } - options.labelExclusion, err = labelExclusion() - if err != nil { - return options, err - } - options.labelRequirement, err = labelRequirement() - if err != nil { - return options, err - } - - // rules - options.rules, err = rules.LoadRules() - if err != nil { - return options, err - } - return options, err -} diff --git a/reaper/options_test.go b/reaper/options_test.go deleted file mode 100644 index 69b5bb5..0000000 --- a/reaper/options_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "io/ioutil" - "k8s.io/apimachinery/pkg/labels" -) - -func init() { - logrus.SetOutput(ioutil.Discard) -} - -func TestOptions(t *testing.T) { - t.Run("namespace", func(t *testing.T) { - t.Run("default", func(t *testing.T) { - os.Clearenv() - namespace := namespace() - assert.Equal(t, "", namespace) - }) - t.Run("valid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envNamespace, "test-namespace") - namespace := namespace() - assert.Equal(t, "test-namespace", namespace) - }) - }) - t.Run("grace period", func(t *testing.T) { - t.Run("default", func(t *testing.T) { - os.Clearenv() - gracePeriod, err := gracePeriod() - assert.NoError(t, err) - assert.Nil(t, gracePeriod) - }) - t.Run("valid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envGracePeriod, "1m53s999ms") - gracePeriod, err := gracePeriod() - assert.NoError(t, err) - assert.Equal(t, int64(113), *gracePeriod) - }) - t.Run("invalid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envGracePeriod, "invalid") - _, err := gracePeriod() - assert.Error(t, err) - }) - }) - t.Run("schedule", func(t *testing.T) { - t.Run("default", func(t *testing.T) { - os.Clearenv() - schedule := schedule() - assert.Equal(t, "@every 1m", schedule) - }) - }) - t.Run("run duration", func(t *testing.T) { - t.Run("default", func(t *testing.T) { - os.Clearenv() - duration, err := runDuration() - assert.NoError(t, err) - assert.Equal(t, 0*time.Second, duration) - }) - t.Run("invalid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envRunDuration, "not-a-duration") - _, err := runDuration() - assert.Error(t, err) - }) - t.Run("valid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envRunDuration, "1m58s") - duration, err := runDuration() - assert.NoError(t, err) - assert.Equal(t, 2*time.Minute-2*time.Second, duration) - }) - }) - t.Run("label exclusion", func(t *testing.T) { - t.Run("default", func(t *testing.T) { - os.Clearenv() - exclusion, err := labelExclusion() - assert.NoError(t, err) - assert.Nil(t, exclusion) - }) - t.Run("only key", func(t *testing.T) { - os.Clearenv() - os.Setenv(envExcludeLabelKey, "test-key") - _, err := labelExclusion() - assert.Error(t, err) - }) - t.Run("only values", func(t *testing.T) { - os.Clearenv() - os.Setenv(envExcludeLabelValues, "test-value1,test-value2") - _, err := labelExclusion() - assert.Error(t, err) - }) - t.Run("invalid key", func(t *testing.T) { - os.Clearenv() - os.Setenv(envExcludeLabelKey, "keys cannot have spaces") - os.Setenv(envExcludeLabelValues, "test-value1,test-value2") - _, err := labelExclusion() - assert.Error(t, err) - }) - t.Run("valid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envExcludeLabelKey, "test-key") - os.Setenv(envExcludeLabelValues, "test-value1,test-value2") - exclusion, err := labelExclusion() - assert.NoError(t, err) - assert.NotNil(t, exclusion) - assert.Equal(t, "test-key notin (test-value1,test-value2)", labels.NewSelector().Add(*exclusion).String()) - }) - }) - t.Run("label requirement", func(t *testing.T) { - t.Run("default", func(t *testing.T) { - os.Clearenv() - requirement, err := labelRequirement() - assert.NoError(t, err) - assert.Nil(t, requirement) - }) - t.Run("only key", func(t *testing.T) { - os.Clearenv() - os.Setenv(envRequireLabelKey, "test-key") - _, err := labelRequirement() - assert.Error(t, err) - }) - t.Run("only values", func(t *testing.T) { - os.Clearenv() - os.Setenv(envRequireLabelValues, "test-value1,test-value2") - _, err := labelRequirement() - assert.Error(t, err) - }) - t.Run("invalid key", func(t *testing.T) { - os.Clearenv() - os.Setenv(envRequireLabelKey, "keys cannot have spaces") - os.Setenv(envRequireLabelValues, "test-value1,test-value2") - _, err := labelRequirement() - assert.Error(t, err) - }) - t.Run("valid", func(t *testing.T) { - os.Clearenv() - os.Setenv(envRequireLabelKey, "test-key") - os.Setenv(envRequireLabelValues, "test-value1,test-value2") - requirement, err := labelRequirement() - assert.NoError(t, err) - assert.NotNil(t, requirement) - assert.Equal(t, "test-key in (test-value1,test-value2)", labels.NewSelector().Add(*requirement).String()) - }) - }) -} - -func TestOptionsLoad(t *testing.T) { - t.Run("invalid options", func(t *testing.T) { - os.Clearenv() - os.Setenv(envRunDuration, "invalid") - _, err := loadOptions() - assert.Error(t, err) - }) - t.Run("no rules", func(t *testing.T) { - os.Clearenv() - _, err := loadOptions() - assert.Error(t, err) - }) - t.Run("valid", func(t *testing.T) { - os.Clearenv() - // ensure at least one rule loads - os.Setenv("CHAOS_CHANCE", "1.0") - options, err := loadOptions() - assert.NoError(t, err) - assert.Equal(t, "@every 1m", options.schedule) - assert.Equal(t, 0*time.Second, options.runDuration) - assert.Nil(t, options.labelExclusion) - assert.Nil(t, options.labelRequirement) - }) -} diff --git a/reaper/reaper.go b/reaper/reaper.go deleted file mode 100644 index 2a14642..0000000 --- a/reaper/reaper.go +++ /dev/null @@ -1,117 +0,0 @@ -package main - -import ( - "time" - - "github.com/robfig/cron" - "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -type reaper struct { - clientSet *kubernetes.Clientset - options options -} - -func newReaper() reaper { - config, err := rest.InClusterConfig() - if err != nil { - logrus.WithError(err).Panic("error getting in cluster kubernetes config") - panic(err) - } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - logrus.WithError(err).Panic("unable to get client set for in cluster kubernetes config") - panic(err) - } - if clientSet == nil { - message := "kubernetes client set cannot be nil" - logrus.Panic(message) - panic(message) - } - options, err := loadOptions() - if err != nil { - logrus.WithError(err).Panic("error loading options") - panic(err) - } - return reaper{ - clientSet: clientSet, - options: options, - } -} - -func (reaper reaper) getPods() *v1.PodList { - coreClient := reaper.clientSet.CoreV1() - pods := coreClient.Pods(reaper.options.namespace) - listOptions := metav1.ListOptions{} - if reaper.options.labelExclusion != nil || reaper.options.labelRequirement != nil { - selector := labels.NewSelector() - if reaper.options.labelExclusion != nil { - selector = selector.Add(*reaper.options.labelExclusion) - } - if reaper.options.labelRequirement != nil { - selector = selector.Add(*reaper.options.labelRequirement) - } - listOptions.LabelSelector = selector.String() - } - podList, err := pods.List(listOptions) - if err != nil { - logrus.WithError(err).Panic("unable to get pods from the cluster") - panic(err) - } - return podList -} - -func (reaper reaper) reapPod(pod v1.Pod, reasons []string) { - logrus.WithFields(logrus.Fields{ - "pod": pod.Name, - "reasons": reasons, - }).Info("reaping pod") - deleteOptions := &metav1.DeleteOptions{ - GracePeriodSeconds: reaper.options.gracePeriod, - } - err := reaper.clientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, deleteOptions) - if err != nil { - // log the error, but continue on - logrus.WithFields(logrus.Fields{ - "pod": pod.Name, - }).WithError(err).Warn("unable to delete pod", err) - } -} - -func (reaper reaper) scytheCycle() { - logrus.Debug("starting reap cycle") - pods := reaper.getPods() - for _, pod := range pods.Items { - shouldReap, reasons := reaper.options.rules.ShouldReap(pod) - if shouldReap { - reaper.reapPod(pod, reasons) - } - } -} - -func (reaper reaper) harvest() { - runForever := reaper.options.runDuration == 0 - schedule := cron.New() - err := schedule.AddFunc(reaper.options.schedule, func() { - reaper.scytheCycle() - }) - - if err != nil { - logrus.WithError(err).Panic("unable to create cron schedule: " + reaper.options.schedule) - panic(err) - } - - schedule.Start() - - if runForever { - select {} // should only fail if no routine can make progress - } else { - time.Sleep(reaper.options.runDuration) - schedule.Stop() - } -} From 6bed4bff5d16ce9c28c0fa0471976502f292dd2a Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sun, 8 Mar 2020 14:46:28 -0400 Subject: [PATCH 03/15] Updated Dockerfiles --- .dockerignore | 4 ++++ Dockerfile | 8 ++++---- Dockerfile-minikube | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2c92871 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +vendor/ +_output/ +examples/ +*.md diff --git a/Dockerfile b/Dockerfile index 6bd285a..ef118f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,14 @@ # Build -FROM golang:1.9 AS build +FROM golang:1.13 AS build WORKDIR /go/src/github.com/target/pod-reaper ENV CGO_ENABLED=0 GOOS=linux RUN go get github.com/Masterminds/glide COPY glide.* ./ RUN glide install --strip-vendor -COPY reaper/*.go ./reaper/ -COPY rules/*.go ./rules/ +COPY cmd/ ./cmd/ +COPY internal/ ./internal/ RUN go test $(glide nv) -RUN go build -o pod-reaper -a -installsuffix go ./reaper +RUN go build -o pod-reaper -a -installsuffix go ./cmd/pod-reaper # Application FROM scratch diff --git a/Dockerfile-minikube b/Dockerfile-minikube index 20a175e..c9e6c76 100644 --- a/Dockerfile-minikube +++ b/Dockerfile-minikube @@ -1,5 +1,5 @@ # including this because default minikube drivers do not yet support multistage docker builds # this will only be used for minikube FROM scratch -COPY pod-reaper / +COPY _output/bin/pod-reaper / CMD ["/pod-reaper"] \ No newline at end of file From 5859fc2f224a3eac97edb36f58fdeba227dfef30 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sun, 8 Mar 2020 15:36:33 -0400 Subject: [PATCH 04/15] Simple e2e test --- Dockerfile | 2 +- glide.lock | 15 ++++++++++++--- glide.yaml | 11 ++++++----- test/e2e/e2e_test.go | 33 +++++++++++++++++++++++++++++++++ test/run-e2e-tests.sh | 25 +++++++++++++++++++++++++ test/run-unit-tests.sh | 19 +++++++++++++++++++ 6 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 test/e2e/e2e_test.go create mode 100755 test/run-e2e-tests.sh create mode 100755 test/run-unit-tests.sh diff --git a/Dockerfile b/Dockerfile index ef118f6..bf2c6b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY glide.* ./ RUN glide install --strip-vendor COPY cmd/ ./cmd/ COPY internal/ ./internal/ -RUN go test $(glide nv) +RUN go test $(glide nv | grep -v /test/) RUN go build -o pod-reaper -a -installsuffix go ./cmd/pod-reaper # Application diff --git a/glide.lock b/glide.lock index 412d7c6..7982cc6 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: b7c11caa604dd7eb1eba1de72321086b340f380d4ccfec817d3882d897d89115 -updated: 2020-03-07T16:43:00.6334477-05:00 +hash: 4ac30dc10a6c2d74c495ca3e076f69910e4e9051421aab5909dd7f6e10e40fb8 +updated: 2020-03-08T15:14:29.9321766-04:00 imports: - name: github.com/davecgh/go-spew version: 8991bc29aa16c548c550c7ff78260e27b9ab7c73 @@ -28,6 +28,8 @@ imports: - OpenAPIv2 - compiler - extensions +- name: github.com/imdario/mergo + version: 9316a62528ac99aaecb4e47eadd6dc8aa6533d58 - name: github.com/joonix/log version: f5f056244ba320717491aa9a073a2cb4fdc2ff30 - name: github.com/json-iterator/go @@ -42,6 +44,8 @@ imports: version: b41be1df696709bb6395fe435af20370037c0b4c - name: github.com/sirupsen/logrus version: 839c75faf7f98a33d445d181f3018b5c3409a45e +- name: github.com/spf13/pflag + version: 2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab - name: github.com/stretchr/testify version: 3ebf1ddaeb260c4b1ae502a01c7844fa8c1fa0e9 subpackages: @@ -176,7 +180,7 @@ imports: - name: k8s.io/client-go version: c68b62b1efa14564a47d67c07f013dc3553937b9 subpackages: - - api/core/v1 + - core/v1 - discovery - kubernetes - kubernetes/scheme @@ -227,13 +231,18 @@ imports: - plugin/pkg/client/auth/exec - rest - rest/watch + - tools/auth + - tools/clientcmd - tools/clientcmd/api + - tools/clientcmd/api/latest + - tools/clientcmd/api/v1 - tools/metrics - tools/reference - transport - util/cert - util/connrotation - util/flowcontrol + - util/homedir - util/keyutil - name: k8s.io/klog version: 2ca9ad30301bf30a8a6e0fa2110db6b8df699a91 diff --git a/glide.yaml b/glide.yaml index 1a8bc8e..c6a2a82 100644 --- a/glide.yaml +++ b/glide.yaml @@ -3,14 +3,15 @@ import: - package: k8s.io/apimachinery version: =v0.17.0 subpackages: - - /pkg/labels - - /pkg/selection + - pkg/labels + - pkg/selection - package: k8s.io/client-go version: =v0.17.0 subpackages: - - kubernetes - - core/v1 - - rest + - kubernetes + - core/v1 + - rest + - tools/clientcmd - package: github.com/stretchr/testify version: ^1.1.4 - package: github.com/robfig/cron diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 0000000..99ab7bb --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,33 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "os" + "testing" + + "github.com/target/pod-reaper/cmd/pod-reaper/app" +) + +func TestE2E(t *testing.T) { + // If we have reached here, it means cluster would have been already setup and the kubeconfig file should + // be in /tmp directory as admin.conf. + os.Setenv("RUN_DURATION", "1s") + os.Setenv("CHAOS_CHANCE", "0.5") + reaper := app.NewReaper("/tmp/admin.conf") + reaper.Harvest() +} diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh new file mode 100755 index 0000000..dcb5ce5 --- /dev/null +++ b/test/run-e2e-tests.sh @@ -0,0 +1,25 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +# This just run e2e tests. +PRJ_PREFIX="github.com/target/pod-reaper" + +kind create cluster +docker pull kubernetes/pause +kind load docker-image kubernetes/pause +kind get kubeconfig > /tmp/admin.conf +go test ${PRJ_PREFIX}/test/e2e/ -v +kind delete cluster \ No newline at end of file diff --git a/test/run-unit-tests.sh b/test/run-unit-tests.sh new file mode 100755 index 0000000..a6029e9 --- /dev/null +++ b/test/run-unit-tests.sh @@ -0,0 +1,19 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +# This just run unit-tests. Ignoring the current directory so as to avoid running e2e tests. +PRJ_PREFIX="github.com/target/pod-reaper" +go test $(go list ${PRJ_PREFIX}/... | grep -v ${PRJ_PREFIX}/vendor/| grep -v ${PRJ_PREFIX}/test/) \ No newline at end of file From 9a05f218c3b80e4747da5d27cab910fc0bb2b985 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sun, 8 Mar 2020 21:56:57 -0400 Subject: [PATCH 05/15] Add _output to .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e389f77..4241c70 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .idea/ vendor/ +_output/ pod-reaper.iml pod-reaper - From 3076c55bea55085b7c171951421ec264ef68e7f2 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sun, 8 Mar 2020 21:59:47 -0400 Subject: [PATCH 06/15] Moved kind create/destroy to Makefile --- Makefile | 7 ++++++- test/run-e2e-tests.sh | 8 +------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 6e95a61..d30f6b1 100644 --- a/Makefile +++ b/Makefile @@ -18,4 +18,9 @@ test-unit: ./test/run-unit-tests.sh test-e2e: - ./test/run-e2e-tests.sh \ No newline at end of file + kind create cluster + docker pull kubernetes/pause + kind load docker-image kubernetes/pause + kind get kubeconfig > /tmp/admin.conf + ./test/run-e2e-tests.sh + kind delete cluster \ No newline at end of file diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh index dcb5ce5..1c69d98 100755 --- a/test/run-e2e-tests.sh +++ b/test/run-e2e-tests.sh @@ -16,10 +16,4 @@ # This just run e2e tests. PRJ_PREFIX="github.com/target/pod-reaper" - -kind create cluster -docker pull kubernetes/pause -kind load docker-image kubernetes/pause -kind get kubeconfig > /tmp/admin.conf -go test ${PRJ_PREFIX}/test/e2e/ -v -kind delete cluster \ No newline at end of file +go test ${PRJ_PREFIX}/test/e2e/ -v \ No newline at end of file From c588a3f13aa8ba98bf2bf9fdb88468baf7085979 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Mon, 9 Mar 2020 17:32:41 -0400 Subject: [PATCH 07/15] Added missing /cmd directory --- .gitignore | 1 - cmd/pod-reaper/app/options.go | 138 ++++++++++++++++++++++ cmd/pod-reaper/app/options_test.go | 179 +++++++++++++++++++++++++++++ cmd/pod-reaper/app/reaper.go | 130 +++++++++++++++++++++ cmd/pod-reaper/main.go | 53 +++++++++ cmd/pod-reaper/main_test.go | 71 ++++++++++++ 6 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 cmd/pod-reaper/app/options.go create mode 100644 cmd/pod-reaper/app/options_test.go create mode 100644 cmd/pod-reaper/app/reaper.go create mode 100644 cmd/pod-reaper/main.go create mode 100644 cmd/pod-reaper/main_test.go diff --git a/.gitignore b/.gitignore index 4241c70..1750fa6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ vendor/ _output/ pod-reaper.iml -pod-reaper diff --git a/cmd/pod-reaper/app/options.go b/cmd/pod-reaper/app/options.go new file mode 100644 index 0000000..0004dea --- /dev/null +++ b/cmd/pod-reaper/app/options.go @@ -0,0 +1,138 @@ +package app + +import ( + "fmt" + "os" + "strings" + "time" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + + "github.com/target/pod-reaper/internal/pkg/rules" +) + +// environment variable names +const envNamespace = "NAMESPACE" +const envGracePeriod = "GRACE_PERIOD" +const envScheduleCron = "SCHEDULE" +const envRunDuration = "RUN_DURATION" +const envExcludeLabelKey = "EXCLUDE_LABEL_KEY" +const envExcludeLabelValues = "EXCLUDE_LABEL_VALUES" +const envRequireLabelKey = "REQUIRE_LABEL_KEY" +const envRequireLabelValues = "REQUIRE_LABEL_VALUES" + +type options struct { + namespace string + gracePeriod *int64 + schedule string + runDuration time.Duration + labelExclusion *labels.Requirement + labelRequirement *labels.Requirement + rules rules.Rules +} + +func namespace() string { + return os.Getenv(envNamespace) +} + +func gracePeriod() (*int64, error) { + envGraceDuration, exists := os.LookupEnv(envGracePeriod) + if !exists { + return nil, nil + } + duration, err := time.ParseDuration(envGraceDuration) + if err != nil { + return nil, fmt.Errorf("invalid %s: %s", envGracePeriod, err) + } + seconds := int64(duration.Seconds()) + return &seconds, nil +} + +func envDuration(key string, defValue string) (time.Duration, error) { + envDuration, exists := os.LookupEnv(key) + if !exists { + envDuration = defValue + } + duration, err := time.ParseDuration(envDuration) + if err != nil { + return duration, fmt.Errorf("invalid %s: %s", key, err) + } + return duration, nil +} + +func schedule() string { + schedule, exists := os.LookupEnv(envScheduleCron) + if !exists { + schedule = "@every 1m" + } + return schedule +} + +func runDuration() (time.Duration, error) { + return envDuration(envRunDuration, "0s") +} + +func labelExclusion() (*labels.Requirement, error) { + labelKey, labelKeyExists := os.LookupEnv(envExcludeLabelKey) + labelValue, labelValuesExist := os.LookupEnv(envExcludeLabelValues) + if labelKeyExists && !labelValuesExist { + return nil, fmt.Errorf("specified %s but not %s", envExcludeLabelKey, envExcludeLabelValues) + } else if !labelKeyExists && labelValuesExist { + return nil, fmt.Errorf("did not specify %s but did specify %s", envExcludeLabelKey, envExcludeLabelValues) + } else if !labelKeyExists && !labelValuesExist { + return nil, nil + } + labelValues := strings.Split(labelValue, ",") + labelExclusion, err := labels.NewRequirement(labelKey, selection.NotIn, labelValues) + if err != nil { + return nil, fmt.Errorf("could not create exclusion label: %s", err) + } + return labelExclusion, nil +} + +func labelRequirement() (*labels.Requirement, error) { + labelKey, labelKeyExists := os.LookupEnv(envRequireLabelKey) + labelValue, labelValuesExist := os.LookupEnv(envRequireLabelValues) + if labelKeyExists && !labelValuesExist { + return nil, fmt.Errorf("specified %s but not %s", envRequireLabelKey, envRequireLabelValues) + } else if !labelKeyExists && labelValuesExist { + return nil, fmt.Errorf("did not specify %s but did specify %s", envRequireLabelKey, envRequireLabelValues) + } else if !labelKeyExists && !labelValuesExist { + return nil, nil + } + labelValues := strings.Split(labelValue, ",") + labelRequirement, err := labels.NewRequirement(labelKey, selection.In, labelValues) + if err != nil { + return nil, fmt.Errorf("could not create requirement label: %s", err) + } + return labelRequirement, nil +} + +func loadOptions() (options options, err error) { + options.namespace = namespace() + options.gracePeriod, err = gracePeriod() + if err != nil { + return options, err + } + options.schedule = schedule() + options.runDuration, err = runDuration() + if err != nil { + return options, err + } + options.labelExclusion, err = labelExclusion() + if err != nil { + return options, err + } + options.labelRequirement, err = labelRequirement() + if err != nil { + return options, err + } + + // rules + options.rules, err = rules.LoadRules() + if err != nil { + return options, err + } + return options, err +} diff --git a/cmd/pod-reaper/app/options_test.go b/cmd/pod-reaper/app/options_test.go new file mode 100644 index 0000000..0a34424 --- /dev/null +++ b/cmd/pod-reaper/app/options_test.go @@ -0,0 +1,179 @@ +package app + +import ( + "os" + "testing" + "time" + + "io/ioutil" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/labels" +) + +func init() { + logrus.SetOutput(ioutil.Discard) +} + +func TestOptions(t *testing.T) { + t.Run("namespace", func(t *testing.T) { + t.Run("default", func(t *testing.T) { + os.Clearenv() + namespace := namespace() + assert.Equal(t, "", namespace) + }) + t.Run("valid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envNamespace, "test-namespace") + namespace := namespace() + assert.Equal(t, "test-namespace", namespace) + }) + }) + t.Run("grace period", func(t *testing.T) { + t.Run("default", func(t *testing.T) { + os.Clearenv() + gracePeriod, err := gracePeriod() + assert.NoError(t, err) + assert.Nil(t, gracePeriod) + }) + t.Run("valid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envGracePeriod, "1m53s999ms") + gracePeriod, err := gracePeriod() + assert.NoError(t, err) + assert.Equal(t, int64(113), *gracePeriod) + }) + t.Run("invalid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envGracePeriod, "invalid") + _, err := gracePeriod() + assert.Error(t, err) + }) + }) + t.Run("schedule", func(t *testing.T) { + t.Run("default", func(t *testing.T) { + os.Clearenv() + schedule := schedule() + assert.Equal(t, "@every 1m", schedule) + }) + }) + t.Run("run duration", func(t *testing.T) { + t.Run("default", func(t *testing.T) { + os.Clearenv() + duration, err := runDuration() + assert.NoError(t, err) + assert.Equal(t, 0*time.Second, duration) + }) + t.Run("invalid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envRunDuration, "not-a-duration") + _, err := runDuration() + assert.Error(t, err) + }) + t.Run("valid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envRunDuration, "1m58s") + duration, err := runDuration() + assert.NoError(t, err) + assert.Equal(t, 2*time.Minute-2*time.Second, duration) + }) + }) + t.Run("label exclusion", func(t *testing.T) { + t.Run("default", func(t *testing.T) { + os.Clearenv() + exclusion, err := labelExclusion() + assert.NoError(t, err) + assert.Nil(t, exclusion) + }) + t.Run("only key", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExcludeLabelKey, "test-key") + _, err := labelExclusion() + assert.Error(t, err) + }) + t.Run("only values", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExcludeLabelValues, "test-value1,test-value2") + _, err := labelExclusion() + assert.Error(t, err) + }) + t.Run("invalid key", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExcludeLabelKey, "keys cannot have spaces") + os.Setenv(envExcludeLabelValues, "test-value1,test-value2") + _, err := labelExclusion() + assert.Error(t, err) + }) + t.Run("valid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExcludeLabelKey, "test-key") + os.Setenv(envExcludeLabelValues, "test-value1,test-value2") + exclusion, err := labelExclusion() + assert.NoError(t, err) + assert.NotNil(t, exclusion) + assert.Equal(t, "test-key notin (test-value1,test-value2)", labels.NewSelector().Add(*exclusion).String()) + }) + }) + t.Run("label requirement", func(t *testing.T) { + t.Run("default", func(t *testing.T) { + os.Clearenv() + requirement, err := labelRequirement() + assert.NoError(t, err) + assert.Nil(t, requirement) + }) + t.Run("only key", func(t *testing.T) { + os.Clearenv() + os.Setenv(envRequireLabelKey, "test-key") + _, err := labelRequirement() + assert.Error(t, err) + }) + t.Run("only values", func(t *testing.T) { + os.Clearenv() + os.Setenv(envRequireLabelValues, "test-value1,test-value2") + _, err := labelRequirement() + assert.Error(t, err) + }) + t.Run("invalid key", func(t *testing.T) { + os.Clearenv() + os.Setenv(envRequireLabelKey, "keys cannot have spaces") + os.Setenv(envRequireLabelValues, "test-value1,test-value2") + _, err := labelRequirement() + assert.Error(t, err) + }) + t.Run("valid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envRequireLabelKey, "test-key") + os.Setenv(envRequireLabelValues, "test-value1,test-value2") + requirement, err := labelRequirement() + assert.NoError(t, err) + assert.NotNil(t, requirement) + assert.Equal(t, "test-key in (test-value1,test-value2)", labels.NewSelector().Add(*requirement).String()) + }) + }) +} + +func TestOptionsLoad(t *testing.T) { + t.Run("invalid options", func(t *testing.T) { + os.Clearenv() + os.Setenv(envRunDuration, "invalid") + _, err := loadOptions() + assert.Error(t, err) + }) + t.Run("no rules", func(t *testing.T) { + os.Clearenv() + _, err := loadOptions() + assert.Error(t, err) + }) + t.Run("valid", func(t *testing.T) { + os.Clearenv() + // ensure at least one rule loads + os.Setenv("CHAOS_CHANCE", "1.0") + options, err := loadOptions() + assert.NoError(t, err) + assert.Equal(t, "@every 1m", options.schedule) + assert.Equal(t, 0*time.Second, options.runDuration) + assert.Nil(t, options.labelExclusion) + assert.Nil(t, options.labelRequirement) + }) +} diff --git a/cmd/pod-reaper/app/reaper.go b/cmd/pod-reaper/app/reaper.go new file mode 100644 index 0000000..3f6d2c0 --- /dev/null +++ b/cmd/pod-reaper/app/reaper.go @@ -0,0 +1,130 @@ +package app + +import ( + "time" + + "github.com/robfig/cron" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +type Reaper struct { + clientSet *kubernetes.Clientset + options options +} + +func NewReaper(kubeconfig string) Reaper { + var config *rest.Config + var err error + + if len(kubeconfig) != 0 { + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + logrus.WithError(err).Panic("unable to build config from kubeconfig file") + panic(err.Error()) + } + } else { + config, err = rest.InClusterConfig() + if err != nil { + logrus.WithError(err).Panic("error getting in cluster kubernetes config") + panic(err) + } + } + + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + logrus.WithError(err).Panic("unable to get client set for in cluster kubernetes config") + panic(err) + } + if clientSet == nil { + message := "kubernetes client set cannot be nil" + logrus.Panic(message) + panic(message) + } + options, err := loadOptions() + if err != nil { + logrus.WithError(err).Panic("error loading options") + panic(err) + } + return Reaper{ + clientSet: clientSet, + options: options, + } +} + +func (reaper Reaper) getPods() *v1.PodList { + coreClient := reaper.clientSet.CoreV1() + pods := coreClient.Pods(reaper.options.namespace) + listOptions := metav1.ListOptions{} + if reaper.options.labelExclusion != nil || reaper.options.labelRequirement != nil { + selector := labels.NewSelector() + if reaper.options.labelExclusion != nil { + selector = selector.Add(*reaper.options.labelExclusion) + } + if reaper.options.labelRequirement != nil { + selector = selector.Add(*reaper.options.labelRequirement) + } + listOptions.LabelSelector = selector.String() + } + podList, err := pods.List(listOptions) + if err != nil { + logrus.WithError(err).Panic("unable to get pods from the cluster") + panic(err) + } + return podList +} + +func (reaper Reaper) reapPod(pod v1.Pod, reasons []string) { + logrus.WithFields(logrus.Fields{ + "pod": pod.Name, + "reasons": reasons, + }).Info("reaping pod") + deleteOptions := &metav1.DeleteOptions{ + GracePeriodSeconds: reaper.options.gracePeriod, + } + err := reaper.clientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, deleteOptions) + if err != nil { + // log the error, but continue on + logrus.WithFields(logrus.Fields{ + "pod": pod.Name, + }).WithError(err).Warn("unable to delete pod", err) + } +} + +func (reaper Reaper) scytheCycle() { + logrus.Debug("starting reap cycle") + pods := reaper.getPods() + for _, pod := range pods.Items { + shouldReap, reasons := reaper.options.rules.ShouldReap(pod) + if shouldReap { + reaper.reapPod(pod, reasons) + } + } +} + +func (reaper Reaper) Harvest() { + runForever := reaper.options.runDuration == 0 + schedule := cron.New() + err := schedule.AddFunc(reaper.options.schedule, func() { + reaper.scytheCycle() + }) + + if err != nil { + logrus.WithError(err).Panic("unable to create cron schedule: " + reaper.options.schedule) + panic(err) + } + + schedule.Start() + + if runForever { + select {} // should only fail if no routine can make progress + } else { + time.Sleep(reaper.options.runDuration) + schedule.Stop() + } +} diff --git a/cmd/pod-reaper/main.go b/cmd/pod-reaper/main.go new file mode 100644 index 0000000..c11cda7 --- /dev/null +++ b/cmd/pod-reaper/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "os" + + joonix "github.com/joonix/log" + "github.com/sirupsen/logrus" + "github.com/target/pod-reaper/cmd/pod-reaper/app" +) + +const envLogLevel = "LOG_LEVEL" +const envLogFormat = "LOG_FORMAT" +const fluentdFormat = "Fluentd" +const logrusFormat = "Logrus" +const defaultLogLevel = logrus.InfoLevel + +func main() { + logLevel := getLogLevel() + logrus.SetLevel(logLevel) + logFormat := getLogFormat() + logrus.SetFormatter(logFormat) + + reaper := app.NewReaper("") + reaper.Harvest() + logrus.Info("pod reaper is exiting") +} + +func getLogLevel() logrus.Level { + levelString, exists := os.LookupEnv(envLogLevel) + if !exists { + return defaultLogLevel + } + + level, err := logrus.ParseLevel(levelString) + if err != nil { + logrus.Errorf("error parsing %s: %v", envLogLevel, err) + return defaultLogLevel + } + + return level +} + +func getLogFormat() logrus.Formatter { + formatString, exists := os.LookupEnv(envLogFormat) + if !exists || formatString == logrusFormat { + return &logrus.JSONFormatter{} + } else if formatString == fluentdFormat { + return &joonix.FluentdFormatter{} + } else { + logrus.Errorf("unknown %s: %v", envLogFormat, formatString) + return &logrus.JSONFormatter{} + } +} diff --git a/cmd/pod-reaper/main_test.go b/cmd/pod-reaper/main_test.go new file mode 100644 index 0000000..cce9753 --- /dev/null +++ b/cmd/pod-reaper/main_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "os" + "reflect" + "testing" + + joonix "github.com/joonix/log" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +var levelTests = []struct { + str string + level logrus.Level +}{ + {"PANIC", logrus.PanicLevel}, + {"FATAL", logrus.FatalLevel}, + {"ERROR", logrus.ErrorLevel}, + {"WARNING", logrus.WarnLevel}, + {"DEBUG", logrus.DebugLevel}, + {"INFO", logrus.InfoLevel}, +} + +func TestGetLogLevel(t *testing.T) { + t.Run("default not set", func(t *testing.T) { + os.Clearenv() + level := getLogLevel() + assert.Equal(t, level, defaultLogLevel) + }) + t.Run("default invalid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envLogLevel, "foo") + level := getLogLevel() + assert.Equal(t, level, defaultLogLevel) + }) + for _, tt := range levelTests { + t.Run(tt.str, func(t *testing.T) { + os.Clearenv() + os.Setenv(envLogLevel, tt.str) + level := getLogLevel() + assert.Equal(t, level, tt.level) + }) + } +} + +func TestGetLogFormat(t *testing.T) { + t.Run("default not set", func(t *testing.T) { + os.Clearenv() + format := getLogFormat() + assert.Equal(t, reflect.TypeOf(format), reflect.TypeOf(&logrus.JSONFormatter{})) + }) + t.Run("default invalid", func(t *testing.T) { + os.Clearenv() + os.Setenv(envLogFormat, "foo") + format := getLogFormat() + assert.Equal(t, reflect.TypeOf(format), reflect.TypeOf(&logrus.JSONFormatter{})) + }) + t.Run("logrus", func(t *testing.T) { + os.Clearenv() + os.Setenv(envLogFormat, "Logrus") + format := getLogFormat() + assert.Equal(t, reflect.TypeOf(format), reflect.TypeOf(&logrus.JSONFormatter{})) + }) + t.Run("fluentd", func(t *testing.T) { + os.Clearenv() + os.Setenv(envLogFormat, "Fluentd") + format := getLogFormat() + assert.Equal(t, reflect.TypeOf(format), reflect.TypeOf(&joonix.FluentdFormatter{})) + }) +} From 5e037637da6770596dcedc791f30a7ee6093f4d7 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Wed, 11 Mar 2020 09:14:13 -0400 Subject: [PATCH 08/15] Move cluster create/delete to scripts --- Makefile | 9 ++++----- test/create-cluster.sh | 16 ++++++++++++++++ test/delete-cluster.sh | 9 +++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) create mode 100755 test/create-cluster.sh create mode 100755 test/delete-cluster.sh diff --git a/Makefile b/Makefile index d30f6b1..2b7b87a 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ VERSION?=$(shell git describe --tags) IMAGE:=pod-reaper:$(VERSION) +CLUSTER_NAME=e2e + all: build build: @@ -18,9 +20,6 @@ test-unit: ./test/run-unit-tests.sh test-e2e: - kind create cluster - docker pull kubernetes/pause - kind load docker-image kubernetes/pause - kind get kubeconfig > /tmp/admin.conf + ./test/create-cluster.sh $(CLUSTER_NAME) ./test/run-e2e-tests.sh - kind delete cluster \ No newline at end of file + ./test/delete-cluster.sh $(CLUSTER_NAME) \ No newline at end of file diff --git a/test/create-cluster.sh b/test/create-cluster.sh new file mode 100755 index 0000000..f8e0760 --- /dev/null +++ b/test/create-cluster.sh @@ -0,0 +1,16 @@ +#!/bin/bash +NAME=$1 +KUBECONFIG=/tmp/admin.conf + +kind get clusters | grep "${NAME}" > /dev/null +if [ $? -eq 1 ]; then + kind create cluster --name "${NAME}" +else + echo "Cluster \"${NAME}\" already exists." +fi + +docker pull kubernetes/pause +kind load docker-image --name "${NAME}" kubernetes/pause +kind get kubeconfig --name "${NAME}" > ${KUBECONFIG} + +echo "Output kubeconfig to ${KUBECONFIG}" \ No newline at end of file diff --git a/test/delete-cluster.sh b/test/delete-cluster.sh new file mode 100755 index 0000000..302798d --- /dev/null +++ b/test/delete-cluster.sh @@ -0,0 +1,9 @@ +#!/bin/bash +NAME=$1 + +kind get clusters | grep "${NAME}" > /dev/null +if [ $? -eq 0 ]; then + kind delete cluster --name "${NAME}" +else + echo "Cluster \"${NAME}\" does not exist." +fi \ No newline at end of file From f0bee002cedf100367b7de5c630fef00c84984cb Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Fri, 13 Mar 2020 09:12:24 -0400 Subject: [PATCH 09/15] Inject client into reaper --- Makefile | 1 + cmd/pod-reaper/app/reaper.go | 34 +++++------------- cmd/pod-reaper/main.go | 2 +- glide.lock | 39 ++++++++++++++++++++- internal/pkg/client/client.go | 65 +++++++++++++++++++++++++++++++++++ test/e2e/e2e_test.go | 8 ++++- 6 files changed, 120 insertions(+), 29 deletions(-) create mode 100644 internal/pkg/client/client.go diff --git a/Makefile b/Makefile index 2b7b87a..c660230 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .PHONY: test VERSION?=$(shell git describe --tags) +REPOSITORY=target IMAGE:=pod-reaper:$(VERSION) CLUSTER_NAME=e2e diff --git a/cmd/pod-reaper/app/reaper.go b/cmd/pod-reaper/app/reaper.go index 3f6d2c0..a50bf25 100644 --- a/cmd/pod-reaper/app/reaper.go +++ b/cmd/pod-reaper/app/reaper.go @@ -5,52 +5,34 @@ import ( "github.com/robfig/cron" "github.com/sirupsen/logrus" + "github.com/target/pod-reaper/internal/pkg/client" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" ) type Reaper struct { - clientSet *kubernetes.Clientset + clientSet kubernetes.Interface options options } -func NewReaper(kubeconfig string) Reaper { - var config *rest.Config - var err error - - if len(kubeconfig) != 0 { - config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - logrus.WithError(err).Panic("unable to build config from kubeconfig file") - panic(err.Error()) - } - } else { - config, err = rest.InClusterConfig() +func NewReaper(clientSet kubernetes.Interface) Reaper { + if clientSet == nil { + client, err := client.CreateClient("") if err != nil { - logrus.WithError(err).Panic("error getting in cluster kubernetes config") + logrus.WithError(err).Panic("unable to get client set for in cluster kubernetes config") panic(err) } + clientSet = client } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - logrus.WithError(err).Panic("unable to get client set for in cluster kubernetes config") - panic(err) - } - if clientSet == nil { - message := "kubernetes client set cannot be nil" - logrus.Panic(message) - panic(message) - } options, err := loadOptions() if err != nil { logrus.WithError(err).Panic("error loading options") panic(err) } + return Reaper{ clientSet: clientSet, options: options, diff --git a/cmd/pod-reaper/main.go b/cmd/pod-reaper/main.go index c11cda7..e1124be 100644 --- a/cmd/pod-reaper/main.go +++ b/cmd/pod-reaper/main.go @@ -20,7 +20,7 @@ func main() { logFormat := getLogFormat() logrus.SetFormatter(logFormat) - reaper := app.NewReaper("") + reaper := app.NewReaper(nil) reaper.Harvest() logrus.Info("pod reaper is exiting") } diff --git a/glide.lock b/glide.lock index 7982cc6..ebb03e0 100644 --- a/glide.lock +++ b/glide.lock @@ -1,10 +1,26 @@ hash: 4ac30dc10a6c2d74c495ca3e076f69910e4e9051421aab5909dd7f6e10e40fb8 -updated: 2020-03-08T15:14:29.9321766-04:00 +updated: 2020-03-12T17:18:30.7097025-04:00 imports: +- name: cloud.google.com/go + version: 8c41231e01b2085512d98153bcffb847ff9b4b9f + subpackages: + - compute/metadata +- name: github.com/Azure/go-autorest + version: "" + subpackages: + - autorest + - autorest/adal + - autorest/azure + - autorest/date + - autorest/mocks + - logger + - tracing - name: github.com/davecgh/go-spew version: 8991bc29aa16c548c550c7ff78260e27b9ab7c73 subpackages: - spew +- name: github.com/dgrijalva/jwt-go + version: 06ea1031745cb8b3dab3f6a236daf2b0aa468b7e - name: github.com/docker/distribution version: 2461543d988979529609e8cb6fca9ca190dc48da - name: github.com/gogo/protobuf @@ -28,6 +44,15 @@ imports: - OpenAPIv2 - compiler - extensions +- name: github.com/gophercloud/gophercloud + version: c2d73b246b48e239d3f03c455905e06fe26e33c3 + subpackages: + - openstack + - openstack/identity/v2/tenants + - openstack/identity/v2/tokens + - openstack/identity/v3/tokens + - openstack/utils + - pagination - name: github.com/imdario/mergo version: 9316a62528ac99aaecb4e47eadd6dc8aa6533d58 - name: github.com/joonix/log @@ -66,7 +91,10 @@ imports: - name: golang.org/x/oauth2 version: 0f29369cfe4552d0e4bcddc57cc75f4d7e672a33 subpackages: + - google - internal + - jws + - jwt - name: golang.org/x/sys version: fde4db37ae7ad8191b03d30d27f258b5291ae4e3 subpackages: @@ -87,9 +115,11 @@ imports: version: 54a98f90d1c46b7731eb8fb305d2a321c30ef610 subpackages: - internal + - internal/app_identity - internal/base - internal/datastore - internal/log + - internal/modules - internal/remote_api - internal/urlfetch - urlfetch @@ -228,9 +258,15 @@ imports: - pkg/apis/clientauthentication/v1alpha1 - pkg/apis/clientauthentication/v1beta1 - pkg/version + - plugin/pkg/client/auth + - plugin/pkg/client/auth/azure - plugin/pkg/client/auth/exec + - plugin/pkg/client/auth/gcp + - plugin/pkg/client/auth/oidc + - plugin/pkg/client/auth/openstack - rest - rest/watch + - third_party/forked/golang/template - tools/auth - tools/clientcmd - tools/clientcmd/api @@ -243,6 +279,7 @@ imports: - util/connrotation - util/flowcontrol - util/homedir + - util/jsonpath - util/keyutil - name: k8s.io/klog version: 2ca9ad30301bf30a8a6e0fa2110db6b8df699a91 diff --git a/internal/pkg/client/client.go b/internal/pkg/client/client.go new file mode 100644 index 0000000..15a14ac --- /dev/null +++ b/internal/pkg/client/client.go @@ -0,0 +1,65 @@ +/* +Copyright 2017 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + clientset "k8s.io/client-go/kubernetes" + // Ensure to load all auth plugins. + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func CreateClient(kubeconfig string) (clientset.Interface, error) { + var cfg *rest.Config + if len(kubeconfig) != 0 { + master, err := GetMasterFromKubeconfig(kubeconfig) + if err != nil { + return nil, fmt.Errorf("Failed to parse kubeconfig file: %v ", err) + } + + cfg, err = clientcmd.BuildConfigFromFlags(master, kubeconfig) + if err != nil { + return nil, fmt.Errorf("Unable to build config: %v", err) + } + + } else { + var err error + cfg, err = rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("Unable to build in cluster config: %v", err) + } + } + + return clientset.NewForConfig(cfg) +} + +func GetMasterFromKubeconfig(filename string) (string, error) { + config, err := clientcmd.LoadFromFile(filename) + if err != nil { + return "", err + } + + context, ok := config.Contexts[config.CurrentContext] + if !ok { + return "", fmt.Errorf("Failed to get master address from kubeconfig") + } + + if val, ok := config.Clusters[context.Cluster]; ok { + return val.Server, nil + } + return "", fmt.Errorf("Failed to get master address from kubeconfig") +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 99ab7bb..589ddd7 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -20,14 +20,20 @@ import ( "os" "testing" + "github.com/stretchr/testify/assert" "github.com/target/pod-reaper/cmd/pod-reaper/app" + "github.com/target/pod-reaper/internal/pkg/client" ) func TestE2E(t *testing.T) { // If we have reached here, it means cluster would have been already setup and the kubeconfig file should // be in /tmp directory as admin.conf. + client, err := client.CreateClient("/tmp/admin.conf") + assert.NoError(t, err) + + os.Clearenv() os.Setenv("RUN_DURATION", "1s") os.Setenv("CHAOS_CHANCE", "0.5") - reaper := app.NewReaper("/tmp/admin.conf") + reaper := app.NewReaper(client) reaper.Harvest() } From 8bc0230705be3ce41ed377923f1b7e39814ab211 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sat, 14 Mar 2020 16:34:20 -0400 Subject: [PATCH 10/15] Default name for cluster scripts --- test/create-cluster.sh | 2 +- test/delete-cluster.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/create-cluster.sh b/test/create-cluster.sh index f8e0760..5694c72 100755 --- a/test/create-cluster.sh +++ b/test/create-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -NAME=$1 +NAME=${1:-kind} KUBECONFIG=/tmp/admin.conf kind get clusters | grep "${NAME}" > /dev/null diff --git a/test/delete-cluster.sh b/test/delete-cluster.sh index 302798d..3591c76 100755 --- a/test/delete-cluster.sh +++ b/test/delete-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -NAME=$1 +NAME=${1:-kind} kind get clusters | grep "${NAME}" > /dev/null if [ $? -eq 0 ]; then From 0802e42560c4fa157e039c885f9b9efc1b99026d Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sun, 15 Mar 2020 11:57:04 -0400 Subject: [PATCH 11/15] Created chaos and unready e2e tests --- internal/pkg/client/client.go | 5 ++- internal/pkg/client/kubeutil.go | 73 +++++++++++++++++++++++++++++++++ test/create-cluster.sh | 9 +++- test/e2e/e2e_test.go | 54 ++++++++++++++++++++---- test/e2e/pause-pod.yml | 10 +++++ test/e2e/unready-pod.yml | 21 ++++++++++ test/run-e2e-tests.sh | 2 + 7 files changed, 163 insertions(+), 11 deletions(-) create mode 100644 internal/pkg/client/kubeutil.go create mode 100644 test/e2e/pause-pod.yml create mode 100644 test/e2e/unready-pod.yml diff --git a/internal/pkg/client/client.go b/internal/pkg/client/client.go index 15a14ac..eafdbcf 100644 --- a/internal/pkg/client/client.go +++ b/internal/pkg/client/client.go @@ -23,10 +23,11 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +// CreateClient creates a new Kubernets clientset with the given config or in cluster config func CreateClient(kubeconfig string) (clientset.Interface, error) { var cfg *rest.Config if len(kubeconfig) != 0 { - master, err := GetMasterFromKubeconfig(kubeconfig) + master, err := getMasterFromKubeconfig(kubeconfig) if err != nil { return nil, fmt.Errorf("Failed to parse kubeconfig file: %v ", err) } @@ -47,7 +48,7 @@ func CreateClient(kubeconfig string) (clientset.Interface, error) { return clientset.NewForConfig(cfg) } -func GetMasterFromKubeconfig(filename string) (string, error) { +func getMasterFromKubeconfig(filename string) (string, error) { config, err := clientcmd.LoadFromFile(filename) if err != nil { return "", err diff --git a/internal/pkg/client/kubeutil.go b/internal/pkg/client/kubeutil.go new file mode 100644 index 0000000..9b77a50 --- /dev/null +++ b/internal/pkg/client/kubeutil.go @@ -0,0 +1,73 @@ +package client + +import ( + "fmt" + "io/ioutil" + "time" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "sigs.k8s.io/yaml" +) + +// KubeUtil contains utility functions to interact with a Kubernetes cluster +type KubeUtil struct { + client clientset.Interface + timeout time.Duration +} + +const retryInterval = 500 * time.Millisecond +const defaultTimeout = 60 * time.Second + +// NewKubeUtil returns a new KubeUtil object that interacts with the given Kubernetes cluster +func NewKubeUtil(client clientset.Interface) KubeUtil { + return KubeUtil{ + client: client, + timeout: defaultTimeout, + } +} + +// ApplyPodManifest applies manifest file to specified namespace +func (k *KubeUtil) ApplyPodManifest(namespace string, path string) (*v1.Pod, error) { + manifest, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var pod *v1.Pod + err = yaml.Unmarshal(manifest, &pod) + if err != nil { + return nil, err + } + + return k.client.CoreV1().Pods(namespace).Create(pod) +} + +// WaitForPodPhase waits until a pod enters the specified phase or timeout is reached +func (k *KubeUtil) WaitForPodPhase(namespace string, name string, status v1.PodPhase) error { + return wait.PollImmediate(retryInterval, k.timeout, func() (bool, error) { + getOpts := metav1.GetOptions{} + pod, err := k.client.CoreV1().Pods(namespace).Get(name, getOpts) + if err != nil { + return false, fmt.Errorf("error getting pod name %s: %v", name, err) + } + return pod.Status.Phase == status, nil + }) +} + +// WaitForPodToDie waits until pod no longer exists or timeout is reached +func (k *KubeUtil) WaitForPodToDie(namespace string, name string) error { + return wait.PollImmediate(retryInterval, k.timeout, func() (bool, error) { + _, err := k.client.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) + if err == nil { + return false, nil + } + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + }) +} diff --git a/test/create-cluster.sh b/test/create-cluster.sh index 5694c72..b4d260f 100755 --- a/test/create-cluster.sh +++ b/test/create-cluster.sh @@ -2,6 +2,11 @@ NAME=${1:-kind} KUBECONFIG=/tmp/admin.conf +load_image() { + docker pull $1 + kind load docker-image --name "${NAME}" $1 +} + kind get clusters | grep "${NAME}" > /dev/null if [ $? -eq 1 ]; then kind create cluster --name "${NAME}" @@ -9,8 +14,8 @@ else echo "Cluster \"${NAME}\" already exists." fi -docker pull kubernetes/pause -kind load docker-image --name "${NAME}" kubernetes/pause +load_image kubernetes/pause +load_image alpine kind get kubeconfig --name "${NAME}" > ${KUBECONFIG} echo "Output kubeconfig to ${KUBECONFIG}" \ No newline at end of file diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 589ddd7..8328438 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -23,17 +23,57 @@ import ( "github.com/stretchr/testify/assert" "github.com/target/pod-reaper/cmd/pod-reaper/app" "github.com/target/pod-reaper/internal/pkg/client" + + v1 "k8s.io/api/core/v1" ) -func TestE2E(t *testing.T) { - // If we have reached here, it means cluster would have been already setup and the kubeconfig file should - // be in /tmp directory as admin.conf. - client, err := client.CreateClient("/tmp/admin.conf") - assert.NoError(t, err) +const namespace string = "default" +const kubeConfig string = "/tmp/admin.conf" + +func TestChaosRule(t *testing.T) { + clientset, err := client.CreateClient(kubeConfig) + assert.NoError(t, err, "error creating kubernetes client") + kubeUtil := client.NewKubeUtil(clientset) + + // Create pod and wait for it to be running + pod, err := kubeUtil.ApplyPodManifest(namespace, "./pause-pod.yml") + assert.NoError(t, err, "error applying pod manifest") + kubeUtil.WaitForPodPhase(namespace, pod.Name, v1.PodRunning) + + // Run reaper + os.Clearenv() + os.Setenv("NAMESPACE", namespace) + os.Setenv("RUN_DURATION", "1s") + os.Setenv("SCHEDULE", "@every 1s") + os.Setenv("CHAOS_CHANCE", "1") + reaper := app.NewReaper(clientset) + reaper.Harvest() + + // Wait for pod to die so other tests aren't affected + err = kubeUtil.WaitForPodToDie(namespace, pod.Name) + assert.NoError(t, err, "timed out waiting for pod to die") +} + +func TestUnreadyRule(t *testing.T) { + clientset, err := client.CreateClient(kubeConfig) + assert.NoError(t, err, "error creating kubernetes client") + kubeUtil := client.NewKubeUtil(clientset) + + // Create pod and wait for it to be running + pod, err := kubeUtil.ApplyPodManifest(namespace, "./unready-pod.yml") + assert.NoError(t, err, "error applying pod manifest") + kubeUtil.WaitForPodPhase(namespace, pod.Name, v1.PodRunning) + // Run reaper os.Clearenv() + os.Setenv("NAMESPACE", namespace) os.Setenv("RUN_DURATION", "1s") - os.Setenv("CHAOS_CHANCE", "0.5") - reaper := app.NewReaper(client) + os.Setenv("SCHEDULE", "@every 1s") + os.Setenv("MAX_UNREADY", "1s") + reaper := app.NewReaper(clientset) reaper.Harvest() + + // Wait for pod to die so other tests aren't affected + err = kubeUtil.WaitForPodToDie(namespace, pod.Name) + assert.NoError(t, err, "timed out waiting for pod to die") } diff --git a/test/e2e/pause-pod.yml b/test/e2e/pause-pod.yml new file mode 100644 index 0000000..782c2f5 --- /dev/null +++ b/test/e2e/pause-pod.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pause +spec: + containers: + - name: pause + image: kubernetes/pause + imagePullPolicy: Never + restartPolicy: Never diff --git a/test/e2e/unready-pod.yml b/test/e2e/unready-pod.yml new file mode 100644 index 0000000..e60a21b --- /dev/null +++ b/test/e2e/unready-pod.yml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: unready +spec: + containers: + - name: unready + image: alpine + imagePullPolicy: Never + command: + - tail + - -f + - /dev/null + readinessProbe: + exec: + command: + - exit 1 + failureThreshold: 1 + periodSeconds: 1 + successThreshold: 1 + restartPolicy: Never diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh index 1c69d98..3718492 100755 --- a/test/run-e2e-tests.sh +++ b/test/run-e2e-tests.sh @@ -16,4 +16,6 @@ # This just run e2e tests. PRJ_PREFIX="github.com/target/pod-reaper" + +go clean -testcache go test ${PRJ_PREFIX}/test/e2e/ -v \ No newline at end of file From 39b6a562b3eb5d37c68165c76c9030cbb4c27dce Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Sun, 15 Mar 2020 15:32:18 -0400 Subject: [PATCH 12/15] Use pause image for unready test --- internal/pkg/client/kubeutil.go | 2 +- test/create-cluster.sh | 1 - test/e2e/e2e_test.go | 45 ++++++++++++++++++++------------- test/e2e/unready-pod.yml | 15 +++++------ 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/internal/pkg/client/kubeutil.go b/internal/pkg/client/kubeutil.go index 9b77a50..15b451b 100644 --- a/internal/pkg/client/kubeutil.go +++ b/internal/pkg/client/kubeutil.go @@ -20,7 +20,7 @@ type KubeUtil struct { } const retryInterval = 500 * time.Millisecond -const defaultTimeout = 60 * time.Second +const defaultTimeout = 30 * time.Second // NewKubeUtil returns a new KubeUtil object that interacts with the given Kubernetes cluster func NewKubeUtil(client clientset.Interface) KubeUtil { diff --git a/test/create-cluster.sh b/test/create-cluster.sh index b4d260f..ab3edef 100755 --- a/test/create-cluster.sh +++ b/test/create-cluster.sh @@ -15,7 +15,6 @@ else fi load_image kubernetes/pause -load_image alpine kind get kubeconfig --name "${NAME}" > ${KUBECONFIG} echo "Output kubeconfig to ${KUBECONFIG}" \ No newline at end of file diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 8328438..a7e0f0c 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,29 +1,13 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package e2e import ( "os" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/target/pod-reaper/cmd/pod-reaper/app" "github.com/target/pod-reaper/internal/pkg/client" - v1 "k8s.io/api/core/v1" ) @@ -54,6 +38,33 @@ func TestChaosRule(t *testing.T) { assert.NoError(t, err, "timed out waiting for pod to die") } +func TestDurationRule(t *testing.T) { + clientset, err := client.CreateClient(kubeConfig) + assert.NoError(t, err, "error creating kubernetes client") + kubeUtil := client.NewKubeUtil(clientset) + + // Create pod and wait for it to be running + pod, err := kubeUtil.ApplyPodManifest(namespace, "./pause-pod.yml") + assert.NoError(t, err, "error applying pod manifest") + kubeUtil.WaitForPodPhase(namespace, pod.Name, v1.PodRunning) + + // Make sure pod has been running at least 1s + time.Sleep(1 * time.Second) + + // Run reaper + os.Clearenv() + os.Setenv("NAMESPACE", namespace) + os.Setenv("RUN_DURATION", "1s") + os.Setenv("SCHEDULE", "@every 1s") + os.Setenv("MAX_DURATION", "1s") + reaper := app.NewReaper(clientset) + reaper.Harvest() + + // Wait for pod to die so other tests aren't affected + err = kubeUtil.WaitForPodToDie(namespace, pod.Name) + assert.NoError(t, err, "timed out waiting for pod to die") +} + func TestUnreadyRule(t *testing.T) { clientset, err := client.CreateClient(kubeConfig) assert.NoError(t, err, "error creating kubernetes client") diff --git a/test/e2e/unready-pod.yml b/test/e2e/unready-pod.yml index e60a21b..bf49ed8 100644 --- a/test/e2e/unready-pod.yml +++ b/test/e2e/unready-pod.yml @@ -5,17 +5,14 @@ metadata: spec: containers: - name: unready - image: alpine + image: kubernetes/pause imagePullPolicy: Never - command: - - tail - - -f - - /dev/null readinessProbe: - exec: - command: - - exit 1 + httpGet: + path: /ready + port: 80 + initialDelaySeconds: 0 + periodSeconds: 1 failureThreshold: 1 periodSeconds: 1 - successThreshold: 1 restartPolicy: Never From 514d3225b03d381451c02e11643b309a85f9e199 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Mon, 16 Mar 2020 09:01:28 -0400 Subject: [PATCH 13/15] Inject client from main --- cmd/pod-reaper/app/reaper.go | 11 +++-------- cmd/pod-reaper/main.go | 12 +++++++++--- internal/pkg/client/client.go | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/pod-reaper/app/reaper.go b/cmd/pod-reaper/app/reaper.go index a50bf25..ac2e66c 100644 --- a/cmd/pod-reaper/app/reaper.go +++ b/cmd/pod-reaper/app/reaper.go @@ -5,7 +5,6 @@ import ( "github.com/robfig/cron" "github.com/sirupsen/logrus" - "github.com/target/pod-reaper/internal/pkg/client" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -19,14 +18,10 @@ type Reaper struct { func NewReaper(clientSet kubernetes.Interface) Reaper { if clientSet == nil { - client, err := client.CreateClient("") - if err != nil { - logrus.WithError(err).Panic("unable to get client set for in cluster kubernetes config") - panic(err) - } - clientSet = client + message := "kubernetes client set cannot be nil" + logrus.Panic(message) + panic(message) } - options, err := loadOptions() if err != nil { logrus.WithError(err).Panic("error loading options") diff --git a/cmd/pod-reaper/main.go b/cmd/pod-reaper/main.go index e1124be..335d790 100644 --- a/cmd/pod-reaper/main.go +++ b/cmd/pod-reaper/main.go @@ -6,6 +6,7 @@ import ( joonix "github.com/joonix/log" "github.com/sirupsen/logrus" "github.com/target/pod-reaper/cmd/pod-reaper/app" + "github.com/target/pod-reaper/internal/pkg/client" ) const envLogLevel = "LOG_LEVEL" @@ -15,12 +16,17 @@ const logrusFormat = "Logrus" const defaultLogLevel = logrus.InfoLevel func main() { - logLevel := getLogLevel() - logrus.SetLevel(logLevel) logFormat := getLogFormat() logrus.SetFormatter(logFormat) + logLevel := getLogLevel() + logrus.SetLevel(logLevel) - reaper := app.NewReaper(nil) + clientset, err := client.CreateClient("") + if err != nil { + logrus.WithError(err).Panic("cannot create client") + panic(err) + } + reaper := app.NewReaper(clientset) reaper.Harvest() logrus.Info("pod reaper is exiting") } diff --git a/internal/pkg/client/client.go b/internal/pkg/client/client.go index eafdbcf..ec79c73 100644 --- a/internal/pkg/client/client.go +++ b/internal/pkg/client/client.go @@ -23,7 +23,7 @@ import ( "k8s.io/client-go/tools/clientcmd" ) -// CreateClient creates a new Kubernets clientset with the given config or in cluster config +// CreateClient creates a new Kubernets clientset with the given config or in-cluster config func CreateClient(kubeconfig string) (clientset.Interface, error) { var cfg *rest.Config if len(kubeconfig) != 0 { From 69529afad7195c884d4af5302999e8e5ac32f2e2 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Tue, 19 Jan 2021 14:02:44 -0500 Subject: [PATCH 14/15] Add WaitForServiceAccountExists to ApplyPodManifest --- internal/pkg/client/kubeutil.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/internal/pkg/client/kubeutil.go b/internal/pkg/client/kubeutil.go index 15b451b..b4b2233 100644 --- a/internal/pkg/client/kubeutil.go +++ b/internal/pkg/client/kubeutil.go @@ -43,6 +43,15 @@ func (k *KubeUtil) ApplyPodManifest(namespace string, path string) (*v1.Pod, err return nil, err } + serviceAccountName := pod.Spec.ServiceAccountName + if serviceAccountName == "" { + serviceAccountName = "default" + } + err = k.WaitForServiceAccountExists(namespace, serviceAccountName) + if err != nil { + return nil, err + } + return k.client.CoreV1().Pods(namespace).Create(pod) } @@ -71,3 +80,14 @@ func (k *KubeUtil) WaitForPodToDie(namespace string, name string) error { return false, err }) } + +// WaitForServiceAccountExists waits until the ServiceAccount exists or timeout is reached +func (k *KubeUtil) WaitForServiceAccountExists(namespace string, name string) error { + return wait.PollImmediate(retryInterval, k.timeout, func() (bool, error) { + serviceAccounts := k.client.CoreV1().ServiceAccounts(namespace) + if _, err := serviceAccounts.Get(name, metav1.GetOptions{}); err == nil { + return true, nil + } + return false, nil + }) +} From f6493731015901720d6fb0ea24680df991c29cf2 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Thu, 21 Jan 2021 15:49:25 -0500 Subject: [PATCH 15/15] Fix Dockerfile after merge --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index dcb18ed..8dc53cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,9 @@ FROM golang:1.14 AS build WORKDIR /go/src/github.com/target/pod-reaper ENV CGO_ENABLED=0 GOOS=linux COPY ./ ./ -RUN go test ./rules/... && \ - go test ./reaper/... && \ - go build -o pod-reaper -a -installsuffix go ./reaper +RUN go test ./cmd/... && \ + go test ./internal/... && \ + go build -o pod-reaper -a -installsuffix go ./cmd/pod-reaper # Application FROM scratch