Merge pull request #17 from iflix-letsplay/update_from_upstream

Merge v0.10.2 from upstream

.gitignore

@@ -1,3 +1,4 @@
+kube-aws
 /artifacts/template.json
 /bin
 /e2e/assets
@@ -16,3 +17,4 @@ _book
 *.mobi
 *.pdf
 node_modules
+kube-aws

README.md

@@ -31,6 +31,7 @@ Generate `cluster.yaml`:
 $ mkdir my-cluster
 $ cd my-cluster
 $ kube-aws init --cluster-name=my-cluster \
+--s3-uri=s3://examplebucket/mydir \
 --external-dns-name=<my-cluster-endpoint> \
 --region=us-west-1 \
 --availability-zone=us-west-1c \
@@ -51,16 +52,16 @@ $ kube-aws render stack
 Validate configuration:
 
 ```
-$ kube-aws validate --s3-uri s3://<your-bucket>/<optional-prefix>
+$ kube-aws validate
 ```
 
 Launch:
 
 ```
-$ kube-aws up --s3-uri s3://<your-bucket>/<optional-prefix>
+$ kube-aws up 
 
 # Or export your cloudformation stack and dependent assets into the `exported/` directory
-$ kube-aws up --s3-uri s3://<your-bucket>/<optional-prefix> --export
+$ kube-aws up --export
 
 # Access the cluster
 $ KUBECONFIG=kubeconfig kubectl get nodes --show-labels
@@ -71,13 +72,13 @@ Update:
 ```
 $ $EDITOR cluster.yaml
 # Update all the cfn stacks including the one for control-plane and the ones for worker node pools
-$ kube-aws update --s3-uri s3://<your-bucket>/<optional-prefix>
+$ kube-aws update 
 ```
 
 Destroy:
 
 ```
-# Destroy all the cfn stacks including the one for control-plane and the ones for worker node pools
+# Destroy all the cfn stacks including the one for control-plane and the ones for worker node pools. Use `--force` for skip confirmation. 
 $ kube-aws destroy
 ```
 

ROADMAP.md

@@ -78,16 +78,25 @@ Please file an issue to make suggestions on this roadmap!
 
 ## v0.9.10
 
-  * kubeadm support to simplify k8s components configuration (#654)
+  * Kubernetes 1.9.x
+  * Security+Usability improvements
+    * [kiam](https://github.com/uswitch/kiam/) integration (#1055)
+    * [authenticator](https://github.com/heptio/authenticator) integration (#1153)
+    * Support for pregenerating IAM roles used by kube2iam/kiam (#1145, #1150)
+  * Operatability improvements
+    * [More manageable Calico + Flannel](https://github.com/kubernetes-incubator/kube-aws/pull/675#issuecomment-303669142) (@redbaron) (#909)
+    * Graduate from relying on CloudFormation nested stacks (#1112)
+    * Ease certificate rotation (#1146)
+
+## v0.9.11
+
+  * Kubernetes 1.10
+  * (After easy H/A controller support) kubeadm support to simplify k8s components configuration (#654)
     * Reduces the amount of code required in kube-aws
     * To better follow upstream improvements on how k8s components are deployed
-  * [More manageable Calico + Flannel](https://github.com/kubernetes-incubator/kube-aws/pull/675#issuecomment-303669142) (@redbaron)
-  * istio integration
+  * (After scalability/reliability/upgradability cleared) istio integration
     * Probably after k8s supported injecting init containers from PodPreset
       * [Upstream issue](https://github.com/kubernetes/kubernetes/issues/43874)
-
-## v0.9.11
-
   * Migrate from coreos-cloudinit to ignition for node bootstrapping (@redbaron)
 
 ## v0.9.12
@@ -97,4 +106,4 @@ Please file an issue to make suggestions on this roadmap!
 
 ## v0.9.x
 
-  * YAML CloudFormation templates
+  * YAML CloudFormation templates?

build

@@ -3,10 +3,13 @@ set -euo pipefail
 
 COMMIT=$(git rev-parse HEAD)
 TAG=$(git describe --exact-match --abbrev=0 --tags "${COMMIT}" 2> /dev/null || true)
+BRANCH=$(git branch | grep \* | cut -d ' ' -f2 || true)
 OUTPUT_PATH=${OUTPUT_PATH:-"bin/kube-aws"}
+VERSION=""
 
 if [ -z "$TAG" ]; then
-	VERSION=$COMMIT
+        [[ -n "$BRANCH" ]] && VERSION="${BRANCH}/"
+	VERSION="${VERSION}${COMMIT:0:8}"
 else
 	VERSION=$TAG
 fi
@@ -18,9 +21,23 @@ fi
 
 echo Building kube-aws ${VERSION}
 
-go generate ./core/controlplane/config
-go generate ./core/nodepool/config
-go generate ./core/root/config
+# generate controlplane templates
+pushd core/controlplane/config
+go run ../../../codegen/templates_gen.go CloudConfigController=cloud-config-controller CloudConfigWorker=cloud-config-worker CloudConfigEtcd=cloud-config-etcd DefaultClusterConfig=cluster.yaml KubeConfigTemplate=kubeconfig.tmpl StackTemplateTemplate=stack-template.json
+gofmt -w templates.go
+go run ../../../codegen/files_gen.go Etcdadm=../../../etcdadm/etcdadm
+gofmt -w files.go
+popd
+
+pushd core/nodepool/config
+go run ../../../codegen/templates_gen.go StackTemplateTemplate=stack-template.json
+gofmt -w templates.go
+popd
+
+pushd core/root/config
+go run ../../../codegen/templates_gen.go StackTemplateTemplate=stack-template.json
+gofmt -w templates.go
+popd
 
 if [[ ! "${BUILD_GOOS:-}" == "" ]];then
   export GOOS=$BUILD_GOOS

cfnstack/ec2.go

@@ -0,0 +1,9 @@
+package cfnstack
+
+import (
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+type EC2Interrogator interface {
+	DescribeSubnets(input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error)
+}

cmd/calculator.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/kubernetes-incubator/kube-aws/core/root"
 	"github.com/spf13/cobra"
-	"strings"
 )
 
 //TODO this is a first step to calculate the stack cost
@@ -21,23 +22,17 @@ var (
 
 	calculatorOpts = struct {
 		awsDebug bool
-		s3URI    string
 	}{}
 )
 
 func init() {
 	RootCmd.AddCommand(cmdCalculator)
 	cmdCalculator.Flags().BoolVar(&calculatorOpts.awsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
-	cmdCalculator.Flags().StringVar(&calculatorOpts.s3URI, "s3-uri", "", "When your template is bigger than the cloudformation limit of 51200 bytes, upload the template to the specified location in S3. S3 location expressed as s3://<bucket>/path/to/dir")
 }
 
-func runCmdCalculator(cmd *cobra.Command, args []string) error {
-
-	if err := validateRequired(flag{"--s3-uri", calculatorOpts.s3URI}); err != nil {
-		return err
-	}
+func runCmdCalculator(_ *cobra.Command, _ []string) error {
 
-	opts := root.NewOptions(calculatorOpts.s3URI, false, false)
+	opts := root.NewOptions(false, false)
 
 	cluster, err := root.ClusterFromFile(configPath, opts, calculatorOpts.awsDebug)
 	if err != nil {

cmd/destroy.go

@@ -1,7 +1,10 @@
 package cmd
 
 import (
+	"bufio"
 	"fmt"
+	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -22,9 +25,15 @@ var (
 func init() {
 	RootCmd.AddCommand(cmdDestroy)
 	cmdDestroy.Flags().BoolVar(&destroyOpts.AwsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
+	cmdDestroy.Flags().BoolVar(&destroyOpts.Force, "force", false, "Don't ask for confirmation")
 }
 
-func runCmdDestroy(cmd *cobra.Command, args []string) error {
+func runCmdDestroy(_ *cobra.Command, _ []string) error {
+	if !destroyOpts.Force && !destroyConfirmation() {
+		fmt.Printf("Operation Cancelled")
+		return nil
+	}
+
 	c, err := root.ClusterDestroyerFromFile(configPath, destroyOpts)
 	if err != nil {
 		return fmt.Errorf("Error parsing config: %v", err)
@@ -37,3 +46,12 @@ func runCmdDestroy(cmd *cobra.Command, args []string) error {
 	fmt.Println("CloudFormation stack is being destroyed. This will take several minutes")
 	return nil
 }
+
+func destroyConfirmation() bool {
+	reader := bufio.NewReader(os.Stdin)
+	fmt.Print("This operation will destroy the cluster. Are you sure? [y,n]: ")
+	text, _ := reader.ReadString('\n')
+	text = strings.TrimSuffix(strings.ToLower(text), "\n")
+
+	return text == "y" || text == "yes"
+}

cmd/init.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/kubernetes-incubator/kube-aws/core/controlplane/config"
+	"github.com/kubernetes-incubator/kube-aws/coreos/amiregistry"
 	"github.com/kubernetes-incubator/kube-aws/filegen"
 	"github.com/spf13/cobra"
 )
@@ -21,22 +22,28 @@ var (
 	initOpts = config.InitialConfig{}
 )
 
+const (
+	defaultReleaseChannel = "stable"
+)
+
 func init() {
 	RootCmd.AddCommand(cmdInit)
+	cmdInit.Flags().StringVar(&initOpts.S3URI, "s3-uri", "", "The URI of the S3 bucket")
 	cmdInit.Flags().StringVar(&initOpts.ClusterName, "cluster-name", "", "The name of this cluster. This will be the name of the cloudformation stack")
 	cmdInit.Flags().StringVar(&initOpts.ExternalDNSName, "external-dns-name", "", "The hostname that will route to the api server")
 	cmdInit.Flags().StringVar(&initOpts.HostedZoneID, "hosted-zone-id", "", "The hosted zone in which a Route53 record set for a k8s API endpoint is created")
 	cmdInit.Flags().StringVar(&initOpts.Region.Name, "region", "", "The AWS region to deploy to")
 	cmdInit.Flags().StringVar(&initOpts.AvailabilityZone, "availability-zone", "", "The AWS availability-zone to deploy to")
 	cmdInit.Flags().StringVar(&initOpts.KeyName, "key-name", "", "The AWS key-pair for ssh access to nodes")
 	cmdInit.Flags().StringVar(&initOpts.KMSKeyARN, "kms-key-arn", "", "The ARN of the AWS KMS key for encrypting TLS assets")
-	cmdInit.Flags().StringVar(&initOpts.AmiId, "ami-id", "", "The AMI ID of CoreOS")
+	cmdInit.Flags().StringVar(&initOpts.AmiId, "ami-id", "", "The AMI ID of CoreOS. Last CoreOS Stable Channel selected by default if empty")
 	cmdInit.Flags().BoolVar(&initOpts.NoRecordSet, "no-record-set", false, "Instruct kube-aws to not manage Route53 record sets for your K8S API endpoints")
 }
 
-func runCmdInit(cmd *cobra.Command, args []string) error {
+func runCmdInit(_ *cobra.Command, _ []string) error {
 	// Validate flags.
 	if err := validateRequired(
+		flag{"--s3-uri", initOpts.S3URI},
 		flag{"--cluster-name", initOpts.ClusterName},
 		flag{"--external-dns-name", initOpts.ExternalDNSName},
 		flag{"--region", initOpts.Region.Name},
@@ -45,6 +52,14 @@ func runCmdInit(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	if initOpts.AmiId == "" {
+		amiID, err := amiregistry.GetAMI(initOpts.Region.Name, defaultReleaseChannel)
+		initOpts.AmiId = amiID
+		if err != nil {
+			return fmt.Errorf("Cannot retrieve CoreOS AMI for region %s, channel %s", initOpts.Region.Name, defaultReleaseChannel)
+		}
+	}
+
 	if !initOpts.NoRecordSet && initOpts.HostedZoneID == "" {
 		return errors.New("Missing required flags: either --hosted-zone-id or --no-record-set is required")
 	}

cmd/render.go

@@ -46,8 +46,9 @@ func init() {
 	cmdRenderCredentials.Flags().BoolVar(&renderCredentialsOpts.GenerateCA, "generate-ca", false, "if generating credentials, generate root CA key and cert. NOT RECOMMENDED FOR PRODUCTION USE- use '-ca-key-path' and '-ca-cert-path' options to provide your own certificate authority assets")
 	cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.CaKeyPath, "ca-key-path", "./credentials/ca-key.pem", "path to pem-encoded CA RSA key")
 	cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.CaCertPath, "ca-cert-path", "./credentials/ca.pem", "path to pem-encoded CA x509 certificate")
+	cmdRenderCredentials.Flags().BoolVar(&renderCredentialsOpts.KIAM, "kiam", true, "generate TLS assets for kiam")
 }
-func runCmdRender(cmd *cobra.Command, args []string) error {
+func runCmdRender(_ *cobra.Command, args []string) error {
 	fmt.Println("WARNING: 'kube-aws render' is deprecated. See 'kube-aws render --help' for usage")
 	if len(args) != 0 {
 		return fmt.Errorf("render takes no arguments\n")
@@ -66,7 +67,7 @@ func runCmdRender(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
-func runCmdRenderStack(cmd *cobra.Command, args []string) error {
+func runCmdRenderStack(_ *cobra.Command, _ []string) error {
 	// Read the config from file.
 	cluster, err := root.StackAssetsRendererFromFile(configPath)
 	if err != nil {
@@ -90,7 +91,7 @@ Next steps:
 	return nil
 }
 
-func runCmdRenderCredentials(cmd *cobra.Command, args []string) error {
+func runCmdRenderCredentials(_ *cobra.Command, _ []string) error {
 	cluster, err := root.CredentialsRendererFromFile(configPath)
 	if err != nil {
 		return fmt.Errorf("failed to read cluster config: %v", err)

cmd/status.go

@@ -21,7 +21,7 @@ func init() {
 	RootCmd.AddCommand(cmdStatus)
 }
 
-func runCmdStatus(cmd *cobra.Command, args []string) error {
+func runCmdStatus(_ *cobra.Command, _ []string) error {
 	describer, err := root.ClusterDescriberFromFile(configPath)
 	if err != nil {
 		return fmt.Errorf("Failed to read cluster config: %v", err)

cmd/up.go

@@ -18,7 +18,6 @@ var (
 
 	upOpts = struct {
 		awsDebug, export, prettyPrint, skipWait bool
-		s3URI                                   string
 	}{}
 )
 
@@ -27,19 +26,11 @@ func init() {
 	cmdUp.Flags().BoolVar(&upOpts.export, "export", false, "Don't create cluster, instead export cloudformation stack file")
 	cmdUp.Flags().BoolVar(&upOpts.prettyPrint, "pretty-print", false, "Pretty print the resulting CloudFormation")
 	cmdUp.Flags().BoolVar(&upOpts.awsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
-	cmdUp.Flags().StringVar(&upOpts.s3URI, "s3-uri", "", "When your template is bigger than the cloudformation limit of 51200 bytes, upload the template to the specified location in S3. S3 location expressed as s3://<bucket>/path/to/dir")
 	cmdUp.Flags().BoolVar(&upOpts.skipWait, "skip-wait", false, "Don't wait for the cluster components be ready")
 }
 
-func runCmdUp(cmd *cobra.Command, args []string) error {
-	// s3URI is required in order to render stack templates because the URI is parsed, combined and then included in the stack templates as
-	// (1) URLs to actual worker/controller cloud-configs in S3 and
-	// (2) URLs to nested stack templates referenced from the root stack template
-	if err := validateRequired(flag{"--s3-uri", upOpts.s3URI}); err != nil {
-		return err
-	}
-
-	opts := root.NewOptions(upOpts.s3URI, upOpts.prettyPrint, upOpts.skipWait)
+func runCmdUp(_ *cobra.Command, _ []string) error {
+	opts := root.NewOptions(upOpts.prettyPrint, upOpts.skipWait)
 
 	cluster, err := root.ClusterFromFile(configPath, opts, upOpts.awsDebug)
 	if err != nil {

cmd/update.go

@@ -18,24 +18,18 @@ var (
 
 	updateOpts = struct {
 		awsDebug, prettyPrint, skipWait bool
-		s3URI                           string
 	}{}
 )
 
 func init() {
 	RootCmd.AddCommand(cmdUpdate)
 	cmdUpdate.Flags().BoolVar(&updateOpts.awsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
 	cmdUpdate.Flags().BoolVar(&updateOpts.prettyPrint, "pretty-print", false, "Pretty print the resulting CloudFormation")
-	cmdUpdate.Flags().StringVar(&updateOpts.s3URI, "s3-uri", "", "When your template is bigger than the cloudformation limit of 51200 bytes, upload the template to the specified location in S3. S3 location expressed as s3://<bucket>/path/to/dir")
 	cmdUpdate.Flags().BoolVar(&updateOpts.skipWait, "skip-wait", false, "Don't wait the resources finish")
 }
 
-func runCmdUpdate(cmd *cobra.Command, args []string) error {
-	if err := validateRequired(flag{"--s3-uri", updateOpts.s3URI}); err != nil {
-		return err
-	}
-
-	opts := root.NewOptions(updateOpts.s3URI, updateOpts.prettyPrint, updateOpts.skipWait)
+func runCmdUpdate(_ *cobra.Command, _ []string) error {
+	opts := root.NewOptions(updateOpts.prettyPrint, updateOpts.skipWait)
 
 	cluster, err := root.ClusterFromFile(configPath, opts, updateOpts.awsDebug)
 	if err != nil {