From a77b6ed17a82bca2017882cf5a2dd01d2aee7a4e Mon Sep 17 00:00:00 2001 From: Michael Dresser Date: Thu, 6 May 2021 12:44:27 -0400 Subject: [PATCH 1/2] Use port forwarding instead of API server proxying Multiple users have reported problems with kubectl cost when running in a more security conscious environment that implements network security policies of various kinds. These policies often prevent the K8s API server from communicating with the Kubecost service. This commit introduces new behavior that will cause requests, by default, to go through a temporarily forwarded port on localhost that goes to a Kubecost pod. This entirely circumvents the issue of API server proxying, but is more complex. The old behavior is now under the flag --use-proxy. Tested locally against a GKE cluster. --- README.md | 19 ++++- go.mod | 2 + go.sum | 1 + pkg/cmd/common.go | 5 ++ pkg/cmd/controller.go | 32 ++++++-- pkg/cmd/deployment.go | 31 ++++++-- pkg/cmd/label.go | 32 ++++++-- pkg/cmd/namespace.go | 32 ++++++-- pkg/cmd/pod.go | 32 ++++++-- pkg/query/aggapi.go | 31 ++++++++ pkg/query/allocation.go | 32 ++++++++ pkg/query/portforward.go | 164 +++++++++++++++++++++++++++++++++++++++ 12 files changed, 382 insertions(+), 31 deletions(-) create mode 100644 pkg/query/portforward.go diff --git a/README.md b/README.md index 2ec93d1..1de7234 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ The following flags modify the behavior of the subcommands: --service-name string The name of the kubecost cost analyzer service. Change if you're running a non-standard deployment, like the staging helm chart. (default "kubecost-cost-analyzer") -n, --namespace string Limit results to only one namespace. Defaults to all namespaces. -N, --kubecost-namespace string The namespace that kubecost is deployed in. Requests to the API will be directed to this namespace. (default "kubecost") + --use-proxy Instead of temporarily port-forwarding, proxy a request to Kubecost through the Kubernetes API server. ``` @@ -177,8 +178,24 @@ The following flags modify the behavior of the subcommands: ## Implementation Quirks -In order to provide a seamless experience for standard Kubernetes configurations, `kubectl-cost` talks to the Kubernetes API server based on your Kubeconfig and uses the API server to proxy a request to the Kubecost API. If you get an error like `failed to proxy get kubecost`, there is something going wrong with this behavior. +In order to provide a seamless experience for standard Kubernetes configurations, `kubectl-cost` temporariliy forwards a port on your system to a Kubecost pod and uses that port to proxy a request. The port will only be bound to `localhost` and will only be open for the duration of the API request. +If you don't want a port to be temporarily forwarded, there is legacy behavior exposed with the flag `--use-proxy` that will instead use the Kubernetes API server to proxy a request to Kubecost. This behavior has its own pitfalls, especially with security policies that would prevent the API server from communicating with services. If you'd like to test this behavior, to make sure it will work with your cluster: + +``` sh +kubectl proxy --port 8080 + +``` + +``` sh +curl -G 'http://localhost:8080/api/v1/namespaces/kubecost/services/kubecost-cost-analyzer:tcp-model/proxy/getConfigs' +``` + +> If you are running an old version of Kubecost, you may have to replace `tcp-model` with `model` + +If that `curl` succeeds, `--use-proxy` should work for you. + +Otherwise: - There may be an underlying problem with your Kubecost install, try `kubectl port-forward`ing the `kubecost-cost-analyzer` service, port 9090, and querying [one of our APIs](https://github.com/kubecost/docs/blob/master/apis.md). - Your problem could be a security configuration that is preventing the API server communicating with certain namespaces or proxying requests in general. - If you're still having problems, hit us up on Slack (see below) or open an issue on this repo. diff --git a/go.mod b/go.mod index dd30f69..ab77580 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect + k8s.io/api v0.20.2 // indirect + k8s.io/apimachinery v0.20.2 // indirect k8s.io/cli-runtime v0.20.2 k8s.io/client-go v0.20.2 k8s.io/klog v1.0.0 diff --git a/go.sum b/go.sum index 5b3e082..37d25f4 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= diff --git a/pkg/cmd/common.go b/pkg/cmd/common.go index 700786e..0a75acf 100644 --- a/pkg/cmd/common.go +++ b/pkg/cmd/common.go @@ -11,6 +11,10 @@ import ( // CostOptions holds common options for querying and displaying // data from the kubecost API type CostOptions struct { + // If set, will proxy a request through the K8s API server + // instead of port forwarding. + useProxy bool + window string isHistorical bool @@ -48,6 +52,7 @@ func addCostOptionsFlags(cmd *cobra.Command, options *CostOptions) { cmd.Flags().BoolVar(&options.showEfficiency, "show-efficiency", true, "show efficiency of cost alongside CPU and memory cost") cmd.Flags().BoolVarP(&options.showAll, "show-all-resources", "A", false, "Equivalent to --show-cpu --show-memory --show-gpu --show-pv --show-network --show-efficiency.") cmd.Flags().StringVar(&options.serviceName, "service-name", "kubecost-cost-analyzer", "The name of the kubecost cost analyzer service. Change if you're running a non-standard deployment, like the staging helm chart.") + cmd.Flags().BoolVar(&options.useProxy, "use-proxy", false, "Instead of temporarily port-forwarding, proxy a request to Kubecost through the Kubernetes API server.") } // addKubeOptionsFlags sets up the cobra command with the flags from diff --git a/pkg/cmd/controller.go b/pkg/cmd/controller.go index 72cfdcc..fc0c86f 100644 --- a/pkg/cmd/controller.go +++ b/pkg/cmd/controller.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" + "github.com/kubecost/cost-model/pkg/kubecost" "github.com/kubecost/kubectl-cost/pkg/query" ) @@ -59,9 +60,19 @@ func runCostController(ko *KubeOptions, no *CostOptionsController) error { } if !no.isHistorical { - aggs, err := query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "controller", "", context.Background()) - if err != nil { - return fmt.Errorf("failed to query agg cost model: %s", err) + var aggs map[string]query.Aggregation + var err error + + if no.useProxy { + aggs, err = query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "controller", "", context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } + } else { + aggs, err = query.QueryAggCostModelFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "controller", "", context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } } // don't show unallocated controller data @@ -78,9 +89,18 @@ func runCostController(ko *KubeOptions, no *CostOptionsController) error { currencyCode, ) } else { - allocations, err := query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "controller", context.Background()) - if err != nil { - return fmt.Errorf("failed to query allocation API: %s", err) + var allocations []map[string]kubecost.Allocation + var err error + if no.useProxy { + allocations, err = query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "controller", context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } + } else { + allocations, err = query.QueryAllocationFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "controller", context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } } // Use allocations[0] because the query accumulates to a single result diff --git a/pkg/cmd/deployment.go b/pkg/cmd/deployment.go index 2416dc9..119db6b 100644 --- a/pkg/cmd/deployment.go +++ b/pkg/cmd/deployment.go @@ -62,9 +62,19 @@ func runCostDeployment(ko *KubeOptions, no *CostOptionsDeployment) error { } if !no.isHistorical { - aggs, err := query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "deployment", "", context.Background()) - if err != nil { - return fmt.Errorf("failed to query agg cost model: %s", err) + var aggs map[string]query.Aggregation + var err error + + if no.useProxy { + aggs, err = query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "deployment", "", context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } + } else { + aggs, err = query.QueryAggCostModelFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "deployment", "", context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } } // don't show unallocated deployment data @@ -81,9 +91,18 @@ func runCostDeployment(ko *KubeOptions, no *CostOptionsDeployment) error { currencyCode, ) } else { - allocations, err := query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "deployment", context.Background()) - if err != nil { - return fmt.Errorf("failed to query allocation API: %s", err) + var allocations []map[string]kubecost.Allocation + var err error + if no.useProxy { + allocations, err = query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "deployment", context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } + } else { + allocations, err = query.QueryAllocationFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "deployment", context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } } // Use allocations[0] because the query accumulates to a single result diff --git a/pkg/cmd/label.go b/pkg/cmd/label.go index ddb83b6..28dab25 100644 --- a/pkg/cmd/label.go +++ b/pkg/cmd/label.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" + "github.com/kubecost/cost-model/pkg/kubecost" "github.com/kubecost/kubectl-cost/pkg/query" ) @@ -66,9 +67,19 @@ func runCostLabel(ko *KubeOptions, no *CostOptionsLabel) error { } if !no.isHistorical { - aggs, err := query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "label", no.queryLabel, context.Background()) - if err != nil { - return fmt.Errorf("failed to query agg cost model: %s", err) + var aggs map[string]query.Aggregation + var err error + + if no.useProxy { + aggs, err = query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "label", no.queryLabel, context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } + } else { + aggs, err = query.QueryAggCostModelFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "label", no.queryLabel, context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } } // don't show unallocated controller data @@ -83,9 +94,18 @@ func runCostLabel(ko *KubeOptions, no *CostOptionsLabel) error { currencyCode, ) } else { - allocations, err := query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, fmt.Sprintf("label:%s", no.queryLabel), context.Background()) - if err != nil { - return fmt.Errorf("failed to query allocation API: %s", err) + var allocations []map[string]kubecost.Allocation + var err error + if no.useProxy { + allocations, err = query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, fmt.Sprintf("label:%s", no.queryLabel), context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } + } else { + allocations, err = query.QueryAllocationFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, fmt.Sprintf("label:%s", no.queryLabel), context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } } // Use allocations[0] because the query accumulates to a single result diff --git a/pkg/cmd/namespace.go b/pkg/cmd/namespace.go index 7bbbe67..e09d236 100644 --- a/pkg/cmd/namespace.go +++ b/pkg/cmd/namespace.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" + "github.com/kubecost/cost-model/pkg/kubecost" "github.com/kubecost/kubectl-cost/pkg/query" ) @@ -56,9 +57,19 @@ func runCostNamespace(ko *KubeOptions, no *CostOptionsNamespace) error { } if !no.isHistorical { - aggs, err := query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "namespace", "", context.Background()) - if err != nil { - return fmt.Errorf("failed to query agg cost model: %s", err) + var aggs map[string]query.Aggregation + var err error + + if no.useProxy { + aggs, err = query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "namespace", "", context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } + } else { + aggs, err = query.QueryAggCostModelFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "namespace", "", context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } } writeAggregationRateTable( @@ -70,9 +81,18 @@ func runCostNamespace(ko *KubeOptions, no *CostOptionsNamespace) error { currencyCode, ) } else { - allocations, err := query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "namespace", context.Background()) - if err != nil { - return fmt.Errorf("failed to query allocation API: %s", err) + var allocations []map[string]kubecost.Allocation + var err error + if no.useProxy { + allocations, err = query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "namespace", context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } + } else { + allocations, err = query.QueryAllocationFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "namespace", context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } } // Use allocations[0] because the query accumulates to a single result diff --git a/pkg/cmd/pod.go b/pkg/cmd/pod.go index 659e1b8..0a6fa6a 100644 --- a/pkg/cmd/pod.go +++ b/pkg/cmd/pod.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" + "github.com/kubecost/cost-model/pkg/kubecost" "github.com/kubecost/kubectl-cost/pkg/query" ) @@ -59,9 +60,19 @@ func runCostPod(ko *KubeOptions, no *CostOptionsPod) error { } if !no.isHistorical { - aggs, err := query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "pod", "", context.Background()) - if err != nil { - return fmt.Errorf("failed to query agg cost model: %s", err) + var aggs map[string]query.Aggregation + var err error + + if no.useProxy { + aggs, err = query.QueryAggCostModel(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "pod", "", context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } + } else { + aggs, err = query.QueryAggCostModelFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "pod", "", context.Background()) + if err != nil { + return fmt.Errorf("failed to query agg cost model: %s", err) + } } // don't show unallocated controller data @@ -78,9 +89,18 @@ func runCostPod(ko *KubeOptions, no *CostOptionsPod) error { currencyCode, ) } else { - allocations, err := query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "pod", context.Background()) - if err != nil { - return fmt.Errorf("failed to query allocation API: %s", err) + var allocations []map[string]kubecost.Allocation + var err error + if no.useProxy { + allocations, err = query.QueryAllocation(ko.clientset, *ko.configFlags.Namespace, no.serviceName, no.window, "pod", context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } + } else { + allocations, err = query.QueryAllocationFwd(ko.restConfig, *ko.configFlags.Namespace, no.serviceName, no.window, "pod", context.Background()) + if err != nil { + return fmt.Errorf("failed to query allocation API: %s", err) + } } // Use allocations[0] because the query accumulates to a single result diff --git a/pkg/query/aggapi.go b/pkg/query/aggapi.go index d5b8d89..98b58ec 100644 --- a/pkg/query/aggapi.go +++ b/pkg/query/aggapi.go @@ -6,6 +6,7 @@ import ( "fmt" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" // "github.com/kubecost/cost-model/pkg/costmodel" "github.com/kubecost/cost-model/pkg/kubecost" @@ -18,6 +19,8 @@ type aggCostModelResponse struct { Data map[string]Aggregation `json:"data"` } +// QueryAggCostModel queries /model/aggregatedCostModel by proxying a request to Kubecost +// through the Kubernetes API server. func QueryAggCostModel(clientset *kubernetes.Clientset, kubecostNamespace, serviceName, window, aggregate, aggregationSubfield string, ctx context.Context) (map[string]Aggregation, error) { params := map[string]string{ "window": window, @@ -45,6 +48,34 @@ func QueryAggCostModel(clientset *kubernetes.Clientset, kubecostNamespace, servi return ar.Data, nil } +// QueryAggCostModelFwd queries /model/aggregatedCostModel by temporarily port-forwarding +// to a Kubecost pod and executing a request against the forwarded port. +func QueryAggCostModelFwd(restConfig *rest.Config, kubecostNamespace, serviceName, window, aggregate, aggregationSubfield string, ctx context.Context) (map[string]Aggregation, error) { + params := map[string]string{ + "window": window, + "aggregation": aggregate, + "rate": "monthly", + "etl": "true", + } + + if aggregationSubfield != "" { + params["aggregationSubfield"] = aggregationSubfield + } + + data, err := portForwardedQueryService(restConfig, kubecostNamespace, serviceName, "model/aggregatedCostModel", params, ctx) + if err != nil { + return nil, fmt.Errorf("failed to port forward query: %s", err) + } + + var ar aggCostModelResponse + err = json.Unmarshal(data, &ar) + if err != nil { + return ar.Data, fmt.Errorf("failed to unmarshal allocation response: %s", err) + } + + return ar.Data, nil +} + // Hardcoded instead of imported because of dependency problems introduced when // github.com/kubecost/cost-model/pkg/costmodel is imported. The breakage involves // Azure's go-autorest, the azure-sdk-for-go, and k8s client-go. Basically, cost-model diff --git a/pkg/query/allocation.go b/pkg/query/allocation.go index c8d7d4c..c8f1c3b 100644 --- a/pkg/query/allocation.go +++ b/pkg/query/allocation.go @@ -7,6 +7,7 @@ import ( "strings" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "github.com/kubecost/cost-model/pkg/kubecost" ) @@ -72,6 +73,8 @@ type allocationResponse struct { Data []map[string]kubecost.Allocation `json:"data"` } +// QueryAllocation queries /model/allocation by proxying a request to Kubecost +// through the Kubernetes API server. func QueryAllocation(clientset *kubernetes.Clientset, kubecostNamespace, serviceName, window, aggregate string, ctx context.Context) ([]map[string]kubecost.Allocation, error) { params := map[string]string{ @@ -100,3 +103,32 @@ func QueryAllocation(clientset *kubernetes.Clientset, kubecostNamespace, service return ar.Data, nil } + +// QueryAllocationFwd queries /model/allocation by temporarily port-forwarding to +// a Kubecost pod and executing a request against the forwarded port. +func QueryAllocationFwd(restConfig *rest.Config, kubecostNamespace, serviceName, window, aggregate string, ctx context.Context) ([]map[string]kubecost.Allocation, error) { + params := map[string]string{ + // if we set this to false, output would be + // per-day (we could use it in a more + // complicated way to build in-terminal charts) + "accumulate": "true", + "window": window, + } + + if aggregate != "" { + params["aggregate"] = aggregate + } + + data, err := portForwardedQueryService(restConfig, kubecostNamespace, serviceName, "model/allocation", params, ctx) + if err != nil { + return nil, fmt.Errorf("failed to port forward query: %s", err) + } + + var ar allocationResponse + err = json.Unmarshal(data, &ar) + if err != nil { + return ar.Data, fmt.Errorf("failed to unmarshal allocation response: %s", err) + } + + return ar.Data, nil +} diff --git a/pkg/query/portforward.go b/pkg/query/portforward.go new file mode 100644 index 0000000..f8e2df3 --- /dev/null +++ b/pkg/query/portforward.go @@ -0,0 +1,164 @@ +package query + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io/ioutil" + "net/http" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" +) + +// reference: https://stackoverflow.com/questions/41545123/how-to-get-pods-under-the-service-with-client-go-the-client-library-of-kubernete +func getServicePods(restConfig *rest.Config, namespace, serviceName string, ctx context.Context) (*corev1.PodList, error) { + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("failed to make clientset: %s", err) + } + + svc, err := clientset.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get service %s in namespace %s: %s", serviceName, namespace, err) + } + + labelSet := labels.Set(svc.Spec.Selector) + labelSelector := labelSet.AsSelector().String() + + pods, err := clientset.CoreV1(). + Pods(namespace). + List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + return nil, fmt.Errorf("failed to get pods in namespace %s for label selector %s: %s", namespace, labelSelector, err) + } + + return pods, nil +} + +// portForwardedQueryService finds the pods associated with the given namespace and service, +// port forwards to them, and executes a GET request to the endpoint with the specified params. +// It then stops the port forward. +func portForwardedQueryService(restConfig *rest.Config, namespace, serviceName, endpoint string, params map[string]string, ctx context.Context) ([]byte, error) { + // First: find a pod to port forward to + pods, err := getServicePods(restConfig, namespace, serviceName, ctx) + if err != nil { + return nil, fmt.Errorf("failed to get service pods: %s", err) + } + if len(pods.Items) == 0 { + return nil, fmt.Errorf("no pods for service %s in namespace %s", serviceName, namespace) + } + + podToForward := pods.Items[0] + + // Second: build the port forwarding config + // https://stackoverflow.com/questions/59027739/upgrading-connection-error-in-port-forwarding-via-client-go + clientset, err := kubernetes.NewForConfig(restConfig) + reqURL := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Namespace(namespace). + Name(podToForward.Name). + SubResource("portforward").URL() + + var berr, bout bytes.Buffer + buffErr := bufio.NewWriter(&berr) + buffOut := bufio.NewWriter(&bout) + + readyCh := make(chan struct{}) + stopCh := make(chan struct{}, 1) + + targetPort := 9090 + + transport, upgrader, err := spdy.RoundTripperFor(restConfig) + if err != nil { + return nil, fmt.Errorf("failed to create round tripper for rest config: %s", err) + } + + dialer := spdy.NewDialer( + upgrader, + &http.Client{Transport: transport}, + http.MethodPost, + reqURL, + ) + + fw, err := portforward.New( + dialer, + []string{fmt.Sprintf("%d:%d", 0, targetPort)}, + stopCh, + readyCh, + buffOut, + buffErr, + ) + if err != nil { + return nil, fmt.Errorf("failed to create portfoward: %s", err) + } + + // Third: port forward + go func() { + err = fw.ForwardPorts() + if err != nil { + panic(err) + } + }() + + // Cleanup once we're done + defer close(stopCh) + + // Fourth: wait until the port forward is ready, or we hit a timeout. + select { + case <-readyCh: + break + case <-time.After(1 * time.Minute): + return nil, fmt.Errorf("timed out (1 min) trying to port forward") + } + + // Confirm that we've port forwarded and allows us to discover the local forwarded port. + // Because we specified port 0, we will have used a random, previously unused port. + ports, err := fw.GetPorts() + if err != nil { + return nil, fmt.Errorf("failed to get list of forwarded ports: %s", err) + } + if len(ports) == 0 { + return nil, fmt.Errorf("unexpected error: no ports forwarded") + } + + // Fifth: make the request to the forwarded port + // TODO: url path join properly + req, err := http.NewRequestWithContext( + ctx, + "GET", + fmt.Sprintf("http://localhost:%d/%s", ports[0].Local, endpoint), + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create base query request: %s", err) + } + q := req.URL.Query() + for key, val := range params { + q.Add(key, val) + } + req.URL.RawQuery = q.Encode() + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to GET %s: %s", endpoint, err) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read %s response body: %s", endpoint, err) + } + if resp.StatusCode != 200 { + return nil, fmt.Errorf("received non-200 status code %d and data: %s", resp.StatusCode, body) + } + + return body, nil +} From 5f5850a27afe3ab8e76a2d953a19d08873a9dab3 Mon Sep 17 00:00:00 2001 From: Michael Dresser Date: Thu, 6 May 2021 13:49:03 -0400 Subject: [PATCH 2/2] README typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1de7234..d707236 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ The following flags modify the behavior of the subcommands: ## Implementation Quirks -In order to provide a seamless experience for standard Kubernetes configurations, `kubectl-cost` temporariliy forwards a port on your system to a Kubecost pod and uses that port to proxy a request. The port will only be bound to `localhost` and will only be open for the duration of the API request. +In order to provide a seamless experience for standard Kubernetes configurations, `kubectl-cost` temporarily forwards a port on your system to a Kubecost pod and uses that port to proxy a request. The port will only be bound to `localhost` and will only be open for the duration of the API request. If you don't want a port to be temporarily forwarded, there is legacy behavior exposed with the flag `--use-proxy` that will instead use the Kubernetes API server to proxy a request to Kubecost. This behavior has its own pitfalls, especially with security policies that would prevent the API server from communicating with services. If you'd like to test this behavior, to make sure it will work with your cluster: