diff --git a/build/build.sh b/build/build.sh index e06135531d..0432db1597 100755 --- a/build/build.sh +++ b/build/build.sh @@ -61,7 +61,7 @@ fi # Since github.com/google/cadvisor/cmd is a submodule, we must build from inside that directory pushd cmd > /dev/null - go build ${GO_FLAGS} -ldflags "${ldflags}" -o "${output_file}" "${repo_path}/cmd" + go mod tidy && go build ${GO_FLAGS} -ldflags "${ldflags}" -o "${output_file}" "${repo_path}/cmd" popd > /dev/null exit 0 diff --git a/cmd/go.mod b/cmd/go.mod index 29c2007424..4a0c0e7983 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -93,7 +93,7 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/karrick/godirwalk v1.17.0 // indirect github.com/klauspost/compress v1.15.11 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -135,4 +135,5 @@ require ( google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/cri-api v0.29.3 // indirect ) diff --git a/cmd/go.sum b/cmd/go.sum index 551bac3ad7..9feb82904e 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -351,7 +351,6 @@ github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK9 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -389,7 +388,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= @@ -790,6 +788,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/cri-api v0.29.3 h1:ppKSui+hhTJW774Mou6x+/ealmzt2jmTM0vsEQVWrjI= +k8s.io/cri-api v0.29.3/go.mod h1:3X7EnhsNaQnCweGhQCJwKNHlH7wHEYuKQ19bRvXMoJY= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= diff --git a/container/containerd/client.go b/container/containerd/client.go index ff5625170a..18a7493a33 100644 --- a/container/containerd/client.go +++ b/container/containerd/client.go @@ -16,9 +16,12 @@ package containerd import ( "context" + "encoding/json" "errors" "fmt" "net" + "path" + "strings" "sync" "time" @@ -27,10 +30,12 @@ import ( versionapi "github.com/containerd/containerd/api/services/version/v1" tasktypes "github.com/containerd/containerd/api/types/task" "github.com/containerd/errdefs" + "github.com/google/cadvisor/container/containerd/config" "google.golang.org/grpc" "google.golang.org/grpc/backoff" "google.golang.org/grpc/credentials/insecure" emptypb "google.golang.org/protobuf/types/known/emptypb" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/google/cadvisor/container/containerd/containers" "github.com/google/cadvisor/container/containerd/pkg/dialer" @@ -40,18 +45,25 @@ type client struct { containerService containersapi.ContainersClient taskService tasksapi.TasksClient versionService versionapi.VersionClient + runtimeService runtimeapi.RuntimeServiceClient } type ContainerdClient interface { LoadContainer(ctx context.Context, id string) (*containers.Container, error) TaskPid(ctx context.Context, id string) (uint32, error) Version(ctx context.Context) (string, error) + RootfsDir(ctx context.Context) (string, error) } var ( ErrTaskIsInUnknownState = errors.New("containerd task is in unknown state") // used when process reported in containerd task is in Unknown State ) +var ( + statePlugin = "io.containerd.grpc.v1.cri" + taskRuntimePlugin = "io.containerd.runtime.v2.task" +) + var once sync.Once var ctrdClient ContainerdClient = nil @@ -104,6 +116,7 @@ func Client(address, namespace string) (ContainerdClient, error) { containerService: containersapi.NewContainersClient(conn), taskService: tasksapi.NewTasksClient(conn), versionService: versionapi.NewVersionClient(conn), + runtimeService: runtimeapi.NewRuntimeServiceClient(conn), } }) return ctrdClient, retErr @@ -140,6 +153,19 @@ func (c *client) Version(ctx context.Context) (string, error) { return response.Version, nil } +func (c *client) RootfsDir(ctx context.Context) (string, error) { + r, err := c.runtimeService.Status(ctx, &runtimeapi.StatusRequest{Verbose: true}) + if err != nil { + return "", err + } + configStr := r.Info["config"] + config := config.Config{} + if err := json.Unmarshal([]byte(configStr), &config); err != nil { + return "", err + } + return path.Join(strings.TrimSuffix(config.StateDir, statePlugin), taskRuntimePlugin), nil +} + func containerFromProto(containerpb *containersapi.Container) *containers.Container { var runtime containers.RuntimeInfo // TODO: is nil check required for containerpb diff --git a/container/containerd/client_test.go b/container/containerd/client_test.go index eaaab0e44e..c7fd2f8c37 100644 --- a/container/containerd/client_test.go +++ b/container/containerd/client_test.go @@ -45,6 +45,10 @@ func (c *containerdClientMock) TaskPid(ctx context.Context, id string) (uint32, return 2389, nil } +func (c *containerdClientMock) RootfsDir(ctx context.Context) (string, error) { + return "/run/containerd/io.containerd.runtime.v2.task", nil +} + func mockcontainerdClient(cntrs map[string]*containers.Container, returnErr error) ContainerdClient { return &containerdClientMock{ cntrs: cntrs, diff --git a/container/containerd/config/config.go b/container/containerd/config/config.go new file mode 100644 index 0000000000..6b25283a55 --- /dev/null +++ b/container/containerd/config/config.go @@ -0,0 +1,436 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +const ( +// defaultImagePullProgressTimeoutDuration is the default value of imagePullProgressTimeout. +// +// NOTE: +// +// This ImagePullProgressTimeout feature is ported from kubelet/dockershim's +// --image-pull-progress-deadline. The original value is 1m0. Unlike docker +// daemon, the containerd doesn't have global concurrent download limitation +// before migrating to Transfer Service. If kubelet runs with concurrent +// image pull, the node will run under IO pressure. The ImagePull process +// could be impacted by self, if the target image is large one with a +// lot of layers. And also both container's writable layers and image's storage +// share one disk. The ImagePull process commits blob to content store +// with fsync, which might bring the unrelated files' dirty pages into +// disk in one transaction [1]. The 1m0 value isn't good enough. Based +// on #9347 case and kubernetes community's usage [2], the default value +// is updated to 5m0. If end-user still runs into unexpected cancel, +// they need to config it based on their environment. +// +// [1]: Fast commits for ext4 - https://lwn.net/Articles/842385/ +// [2]: https://github.com/kubernetes/kubernetes/blob/1635c380b26a1d8cc25d36e9feace9797f4bae3c/cluster/gce/util.sh#L882 +// defaultImagePullProgressTimeoutDuration = 5 * time.Minute +) + +type SandboxControllerMode string + +const ( + // ModePodSandbox means use Controller implementation from sbserver podsandbox package. + // We take this one as a default mode. + ModePodSandbox SandboxControllerMode = "podsandbox" + // ModeShim means use whatever Controller implementation provided by shim. + ModeShim SandboxControllerMode = "shim" + // DefaultSandboxImage is the default image to use for sandboxes when empty or + // for default configurations. + DefaultSandboxImage = "registry.k8s.io/pause:3.9" +) + +// Runtime struct to contain the type(ID), engine, and root variables for a default runtime +// and a runtime for untrusted workload. +type Runtime struct { + // Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux + Type string `toml:"runtime_type" json:"runtimeType"` + // Path is an optional field that can be used to overwrite path to a shim runtime binary. + // When specified, containerd will ignore runtime name field when resolving shim location. + // Path must be abs. + Path string `toml:"runtime_path" json:"runtimePath"` + // PodAnnotations is a list of pod annotations passed to both pod sandbox as well as + // container OCI annotations. + PodAnnotations []string `toml:"pod_annotations" json:"PodAnnotations"` + // ContainerAnnotations is a list of container annotations passed through to the OCI config of the containers. + // Container annotations in CRI are usually generated by other Kubernetes node components (i.e., not users). + // Currently, only device plugins populate the annotations. + ContainerAnnotations []string `toml:"container_annotations" json:"ContainerAnnotations"` + // Options are config options for the runtime. + Options map[string]interface{} `toml:"options" json:"options"` + // PrivilegedWithoutHostDevices overloads the default behaviour for adding host devices to the + // runtime spec when the container is privileged. Defaults to false. + PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"` + // PrivilegedWithoutHostDevicesAllDevicesAllowed overloads the default behaviour device allowlisting when + // to the runtime spec when the container when PrivilegedWithoutHostDevices is already enabled. Requires + // PrivilegedWithoutHostDevices to be enabled. Defaults to false. + PrivilegedWithoutHostDevicesAllDevicesAllowed bool `toml:"privileged_without_host_devices_all_devices_allowed" json:"privileged_without_host_devices_all_devices_allowed"` + // BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from. + BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"` + // NetworkPluginConfDir is a directory containing the CNI network information for the runtime class. + NetworkPluginConfDir string `toml:"cni_conf_dir" json:"cniConfDir"` + // NetworkPluginMaxConfNum is the max number of plugin config files that will + // be loaded from the cni config directory by go-cni. Set the value to 0 to + // load all config files (no arbitrary limit). The legacy default value is 1. + NetworkPluginMaxConfNum int `toml:"cni_max_conf_num" json:"cniMaxConfNum"` + // Snapshotter setting snapshotter at runtime level instead of making it as a global configuration. + // An example use case is to use devmapper or other snapshotters in Kata containers for performance and security + // while using default snapshotters for operational simplicity. + // See https://github.com/containerd/containerd/issues/6657 for details. + Snapshotter string `toml:"snapshotter" json:"snapshotter"` + // Sandboxer defines which sandbox runtime to use when scheduling pods + // This features requires the new CRI server implementation (enabled by default in 2.0) + // shim - means use whatever Controller implementation provided by shim (e.g. use RemoteController). + // podsandbox - means use Controller implementation from sbserver podsandbox package. + Sandboxer string `toml:"sandboxer" json:"sandboxer"` +} + +// ContainerdConfig contains toml config related to containerd +type ContainerdConfig struct { + // DefaultRuntimeName is the default runtime name to use from the runtimes table. + DefaultRuntimeName string `toml:"default_runtime_name" json:"defaultRuntimeName"` + + // Runtimes is a map from CRI RuntimeHandler strings, which specify types of runtime + // configurations, to the matching configurations. + Runtimes map[string]Runtime `toml:"runtimes" json:"runtimes"` + + // IgnoreBlockIONotEnabledErrors is a boolean flag to ignore + // blockio related errors when blockio support has not been + // enabled. + IgnoreBlockIONotEnabledErrors bool `toml:"ignore_blockio_not_enabled_errors" json:"ignoreBlockIONotEnabledErrors"` + + // IgnoreRdtNotEnabledErrors is a boolean flag to ignore RDT related errors + // when RDT support has not been enabled. + IgnoreRdtNotEnabledErrors bool `toml:"ignore_rdt_not_enabled_errors" json:"ignoreRdtNotEnabledErrors"` +} + +// CniConfig contains toml config related to cni +type CniConfig struct { + // NetworkPluginBinDir is the directory in which the binaries for the plugin is kept. + NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"` + // NetworkPluginConfDir is the directory in which the admin places a CNI conf. + NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"` + // NetworkPluginMaxConfNum is the max number of plugin config files that will + // be loaded from the cni config directory by go-cni. Set the value to 0 to + // load all config files (no arbitrary limit). The legacy default value is 1. + NetworkPluginMaxConfNum int `toml:"max_conf_num" json:"maxConfNum"` + // NetworkPluginSetupSerially is a boolean flag to specify whether containerd sets up networks serially + // if there are multiple CNI plugin config files existing and NetworkPluginMaxConfNum is larger than 1. + // + // NOTE: On the Linux platform, containerd provides loopback network + // configuration by default. There are at least two network plugins. + // The default value of NetworkPluginSetupSerially is false which means + // the loopback and eth0 are handled in parallel mode. Since the loopback + // device is created as the net namespace is created, it's safe to run + // in parallel mode as the default setting. + NetworkPluginSetupSerially bool `toml:"setup_serially" json:"setupSerially"` + // NetworkPluginConfTemplate is the file path of golang template used to generate cni config. + // When it is set, containerd will get cidr(s) from kubelet to replace {{.PodCIDR}}, + // {{.PodCIDRRanges}} or {{.Routes}} in the template, and write the config into + // NetworkPluginConfDir. + // Ideally the cni config should be placed by system admin or cni daemon like calico, + // weaveworks etc. However, this is useful for the cases when there is no cni daemonset to place cni config. + // This allowed for very simple generic networking using the Kubernetes built in node pod CIDR IPAM, avoiding the + // need to fetch the node object through some external process (which has scalability, auth, complexity issues). + // It is currently heavily used in kubernetes-containerd CI testing + // NetworkPluginConfTemplate was once deprecated in containerd v1.7.0, + // but its deprecation was cancelled in v1.7.3. + NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"` + // IPPreference specifies the strategy to use when selecting the main IP address for a pod. + // + // Options include: + // * ipv4, "" - (default) select the first ipv4 address + // * ipv6 - select the first ipv6 address + // * cni - use the order returned by the CNI plugins, returning the first IP address from the results + IPPreference string `toml:"ip_pref" json:"ipPref"` +} + +// Mirror contains the config related to the registry mirror +type Mirror struct { + // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints + // one by one until a working one is found. The endpoint must be a valid url + // with host specified. + // The scheme, host and path from the endpoint URL will be used. + Endpoints []string `toml:"endpoint" json:"endpoint"` +} + +// AuthConfig contains the config related to authentication to a specific registry +type AuthConfig struct { + // Username is the username to login the registry. + Username string `toml:"username" json:"username"` + // Password is the password to login the registry. + Password string `toml:"password" json:"password"` + // Auth is a base64 encoded string from the concatenation of the username, + // a colon, and the password. + Auth string `toml:"auth" json:"auth"` + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `toml:"identitytoken" json:"identitytoken"` +} + +// Registry is registry settings configured +type Registry struct { + // ConfigPath is a path to the root directory containing registry-specific + // configurations. + // If ConfigPath is set, the rest of the registry specific options are ignored. + ConfigPath string `toml:"config_path" json:"configPath"` + // Mirrors are namespace to mirror mapping for all namespaces. + // This option will not be used when ConfigPath is provided. + // DEPRECATED: Use ConfigPath instead. Remove in containerd 2.0. + Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"` + // Configs are configs for each registry. + // The key is the domain name or IP of the registry. + // DEPRECATED: Use ConfigPath instead. + Configs map[string]RegistryConfig `toml:"configs" json:"configs"` + // Auths are registry endpoint to auth config mapping. The registry endpoint must + // be a valid url with host specified. + // DEPRECATED: Use ConfigPath instead. Remove in containerd 2.0, supported in 1.x releases. + Auths map[string]AuthConfig `toml:"auths" json:"auths"` + // Headers adds additional HTTP headers that get sent to all registries + Headers map[string][]string `toml:"headers" json:"headers"` +} + +// RegistryConfig contains configuration used to communicate with the registry. +type RegistryConfig struct { + // Auth contains information to authenticate to the registry. + Auth *AuthConfig `toml:"auth" json:"auth"` +} + +// ImageDecryption contains configuration to handling decryption of encrypted container images. +type ImageDecryption struct { + // KeyModel specifies the trust model of where keys should reside. + // + // Details of field usage can be found in: + // https://github.com/containerd/containerd/tree/main/docs/cri/config.md + // + // Details of key models can be found in: + // https://github.com/containerd/containerd/tree/main/docs/cri/decryption.md + KeyModel string `toml:"key_model" json:"keyModel"` +} + +// ImagePlatform represents the platform to use for an image including the +// snapshotter to use. If snapshotter is not provided, the platform default +// can be assumed. When platform is not provided, the default platform can +// be assumed +type ImagePlatform struct { + Platform string `toml:"platform" json:"platform"` + // Snapshotter setting snapshotter at runtime level instead of making it as a global configuration. + // An example use case is to use devmapper or other snapshotters in Kata containers for performance and security + // while using default snapshotters for operational simplicity. + // See https://github.com/containerd/containerd/issues/6657 for details. + Snapshotter string `toml:"snapshotter" json:"snapshotter"` +} + +type ImageConfig struct { + // Snapshotter is the snapshotter used by containerd. + Snapshotter string `toml:"snapshotter" json:"snapshotter"` + + // DisableSnapshotAnnotations disables to pass additional annotations (image + // related information) to snapshotters. These annotations are required by + // stargz snapshotter (https://github.com/containerd/stargz-snapshotter). + DisableSnapshotAnnotations bool `toml:"disable_snapshot_annotations" json:"disableSnapshotAnnotations"` + + // DiscardUnpackedLayers is a boolean flag to specify whether to allow GC to + // remove layers from the content store after successfully unpacking these + // layers to the snapshotter. + DiscardUnpackedLayers bool `toml:"discard_unpacked_layers" json:"discardUnpackedLayers"` + + // PinnedImages are images which the CRI plugin uses and should not be + // removed by the CRI client. The images have a key which can be used + // by other plugins to lookup the current image name. + // Image names should be full names including domain and tag + // Examples: + // "sandbox": "k8s.gcr.io/pause:3.9" + // "base": "docker.io/library/ubuntu:latest" + // Migrated from: + // (PluginConfig).SandboxImage string `toml:"sandbox_image" json:"sandboxImage"` + PinnedImages map[string]string `toml:"pinned_images" json:"pinned_images"` + + // RuntimePlatforms is map between the runtime and the image platform to + // use for that runtime. When resolving an image for a runtime, this + // mapping will be used to select the image for the platform and the + // snapshotter for unpacking. + RuntimePlatforms map[string]ImagePlatform `toml:"runtime_platforms" json:"runtimePlatforms"` + + // Registry contains config related to the registry + Registry Registry `toml:"registry" json:"registry"` + + // ImageDecryption contains config related to handling decryption of encrypted container images + ImageDecryption `toml:"image_decryption" json:"imageDecryption"` + + // MaxConcurrentDownloads restricts the number of concurrent downloads for each image. + // TODO: Migrate to transfer service + MaxConcurrentDownloads int `toml:"max_concurrent_downloads" json:"maxConcurrentDownloads"` + + // ImagePullProgressTimeout is the maximum duration that there is no + // image data read from image registry in the open connection. It will + // be reset whatever a new byte has been read. If timeout, the image + // pulling will be cancelled. A zero value means there is no timeout. + // + // The string is in the golang duration format, see: + // https://golang.org/pkg/time/#ParseDuration + ImagePullProgressTimeout string `toml:"image_pull_progress_timeout" json:"imagePullProgressTimeout"` + + // ImagePullWithSyncFs is an experimental setting. It's to force sync + // filesystem during unpacking to ensure that data integrity. + // TODO: Migrate to transfer service + ImagePullWithSyncFs bool `toml:"image_pull_with_sync_fs" json:"imagePullWithSyncFs"` + + // StatsCollectPeriod is the period (in seconds) of snapshots stats collection. + StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod"` +} + +// RuntimeConfig contains toml config related to CRI plugin, +// it is a subset of Config. +type RuntimeConfig struct { + // ContainerdConfig contains config related to containerd + ContainerdConfig `toml:"containerd" json:"containerd"` + // CniConfig contains config related to cni + CniConfig `toml:"cni" json:"cni"` + // EnableSelinux indicates to enable the selinux support. + EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux"` + // SelinuxCategoryRange allows the upper bound on the category range to be set. + // If not specified or set to 0, defaults to 1024 from the selinux package. + SelinuxCategoryRange int `toml:"selinux_category_range" json:"selinuxCategoryRange"` + // MaxContainerLogLineSize is the maximum log line size in bytes for a container. + // Log line longer than the limit will be split into multiple lines. Non-positive + // value means no limit. + MaxContainerLogLineSize int `toml:"max_container_log_line_size" json:"maxContainerLogSize"` + // DisableCgroup indicates to disable the cgroup support. + // This is useful when the containerd does not have permission to access cgroup. + DisableCgroup bool `toml:"disable_cgroup" json:"disableCgroup"` + // DisableApparmor indicates to disable the apparmor support. + // This is useful when the containerd does not have permission to access Apparmor. + DisableApparmor bool `toml:"disable_apparmor" json:"disableApparmor"` + // RestrictOOMScoreAdj indicates to limit the lower bound of OOMScoreAdj to the containerd's + // current OOMScoreADj. + // This is useful when the containerd does not have permission to decrease OOMScoreAdj. + RestrictOOMScoreAdj bool `toml:"restrict_oom_score_adj" json:"restrictOOMScoreAdj"` + // DisableProcMount disables Kubernetes ProcMount support. This MUST be set to `true` + // when using containerd with Kubernetes <=1.11. + DisableProcMount bool `toml:"disable_proc_mount" json:"disableProcMount"` + // UnsetSeccompProfile is the profile containerd/cri will use If the provided seccomp profile is + // unset (`""`) for a container (default is `unconfined`) + UnsetSeccompProfile string `toml:"unset_seccomp_profile" json:"unsetSeccompProfile"` + // TolerateMissingHugetlbController if set to false will error out on create/update + // container requests with huge page limits if the cgroup controller for hugepages is not present. + // This helps with supporting Kubernetes <=1.18 out of the box. (default is `true`) + TolerateMissingHugetlbController bool `toml:"tolerate_missing_hugetlb_controller" json:"tolerateMissingHugetlbController"` + // DisableHugetlbController indicates to silently disable the hugetlb controller, even when it is + // present in /sys/fs/cgroup/cgroup.controllers. + // This helps with running rootless mode + cgroup v2 + systemd but without hugetlb delegation. + DisableHugetlbController bool `toml:"disable_hugetlb_controller" json:"disableHugetlbController"` + // DeviceOwnershipFromSecurityContext changes the default behavior of setting container devices uid/gid + // from CRI's SecurityContext (RunAsUser/RunAsGroup) instead of taking host's uid/gid. Defaults to false. + DeviceOwnershipFromSecurityContext bool `toml:"device_ownership_from_security_context" json:"device_ownership_from_security_context"` + // IgnoreImageDefinedVolumes ignores volumes defined by the image. Useful for better resource + // isolation, security and early detection of issues in the mount configuration when using + // ReadOnlyRootFilesystem since containers won't silently mount a temporary volume. + IgnoreImageDefinedVolumes bool `toml:"ignore_image_defined_volumes" json:"ignoreImageDefinedVolumes"` + // NetNSMountsUnderStateDir places all mounts for network namespaces under StateDir/netns instead + // of being placed under the hardcoded directory /var/run/netns. Changing this setting requires + // that all containers are deleted. + NetNSMountsUnderStateDir bool `toml:"netns_mounts_under_state_dir" json:"netnsMountsUnderStateDir"` + // EnableUnprivilegedPorts configures net.ipv4.ip_unprivileged_port_start=0 + // for all containers which are not using host network + // and if it is not overwritten by PodSandboxConfig + // Note that currently default is set to disabled but target change it in future, see: + // https://github.com/kubernetes/kubernetes/issues/102612 + EnableUnprivilegedPorts bool `toml:"enable_unprivileged_ports" json:"enableUnprivilegedPorts"` + // EnableUnprivilegedICMP configures net.ipv4.ping_group_range="0 2147483647" + // for all containers which are not using host network, are not running in user namespace + // and if it is not overwritten by PodSandboxConfig + // Note that currently default is set to disabled but target change it in future together with EnableUnprivilegedPorts + EnableUnprivilegedICMP bool `toml:"enable_unprivileged_icmp" json:"enableUnprivilegedICMP"` + // EnableCDI indicates to enable injection of the Container Device Interface Specifications + // into the OCI config + // For more details about CDI and the syntax of CDI Spec files please refer to + // https://tags.cncf.io/container-device-interface. + EnableCDI bool `toml:"enable_cdi" json:"enableCDI"` + // CDISpecDirs is the list of directories to scan for Container Device Interface Specifications + // For more details about CDI configuration please refer to + // https://tags.cncf.io/container-device-interface#containerd-configuration + CDISpecDirs []string `toml:"cdi_spec_dirs" json:"cdiSpecDirs"` + + // DrainExecSyncIOTimeout is the maximum duration to wait for ExecSync + // API' IO EOF event after exec init process exits. A zero value means + // there is no timeout. + // + // The string is in the golang duration format, see: + // https://golang.org/pkg/time/#ParseDuration + // + // For example, the value can be '5h', '2h30m', '10s'. + DrainExecSyncIOTimeout string `toml:"drain_exec_sync_io_timeout" json:"drainExecSyncIOTimeout"` + + // IgnoreDeprecationWarnings is the list of the deprecation IDs (such as "io.containerd.deprecation/pull-schema-1-image") + // that should be ignored for checking "ContainerdHasNoDeprecationWarnings" condition. + IgnoreDeprecationWarnings []string `toml:"ignore_deprecation_warnings" json:"ignoreDeprecationWarnings"` +} + +// X509KeyPairStreaming contains the x509 configuration for streaming +type X509KeyPairStreaming struct { + // TLSCertFile is the path to a certificate file + TLSCertFile string `toml:"tls_cert_file" json:"tlsCertFile"` + // TLSKeyFile is the path to a private key file + TLSKeyFile string `toml:"tls_key_file" json:"tlsKeyFile"` +} + +// Config contains all configurations for CRI runtime plugin. +type Config struct { + // RuntimeConfig is the config for CRI runtime. + RuntimeConfig + // ContainerdRootDir is the root directory path for containerd. + ContainerdRootDir string `json:"containerdRootDir"` + // ContainerdEndpoint is the containerd endpoint path. + ContainerdEndpoint string `json:"containerdEndpoint"` + // RootDir is the root directory path for managing cri plugin files + // (metadata checkpoint etc.) + RootDir string `json:"rootDir"` + // StateDir is the root directory path for managing volatile pod/container data + StateDir string `json:"stateDir"` +} + +// ServerConfig contains all the configuration for the CRI API server. +type ServerConfig struct { + // DisableTCPService disables serving CRI on the TCP server. + DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"` + // StreamServerAddress is the ip address streaming server is listening on. + StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"` + // StreamServerPort is the port streaming server is listening on. + StreamServerPort string `toml:"stream_server_port" json:"streamServerPort"` + // StreamIdleTimeout is the maximum time a streaming connection + // can be idle before the connection is automatically closed. + // The string is in the golang duration format, see: + // https://golang.org/pkg/time/#ParseDuration + StreamIdleTimeout string `toml:"stream_idle_timeout" json:"streamIdleTimeout"` + // EnableTLSStreaming indicates to enable the TLS streaming support. + EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"` + // X509KeyPairStreaming is a x509 key pair used for TLS streaming + X509KeyPairStreaming `toml:"x509_key_pair_streaming" json:"x509KeyPairStreaming"` +} diff --git a/container/containerd/factory.go b/container/containerd/factory.go index d44abc4eac..7e243d2d46 100644 --- a/container/containerd/factory.go +++ b/container/containerd/factory.go @@ -64,6 +64,13 @@ func (f *containerdFactory) NewContainerHandler(name string, metadataEnvAllowLis return } + ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout) + defer cancel() + rootfsDir, err := client.RootfsDir(ctx) + if err != nil { + klog.Warningf("unable to get containerd rootfs dir, err: %v, continue register", err) + } + containerdMetadataEnvAllowList := strings.Split(*containerdEnvMetadataWhiteList, ",") // prefer using the unified metadataEnvAllowList @@ -80,6 +87,7 @@ func (f *containerdFactory) NewContainerHandler(name string, metadataEnvAllowLis inHostNamespace, containerdMetadataEnvAllowList, f.includedMetrics, + rootfsDir, ) } diff --git a/container/containerd/handler.go b/container/containerd/handler.go index 7746341cd9..3f61eedf12 100644 --- a/container/containerd/handler.go +++ b/container/containerd/handler.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "fmt" + "path" "strings" "time" @@ -38,8 +39,10 @@ type containerdContainerHandler struct { machineInfoFactory info.MachineInfoFactory // Absolute path to the cgroup hierarchies of this container. // (e.g.: "cpu" -> "/sys/fs/cgroup/cpu/test") - cgroupPaths map[string]string - fsInfo fs.FsInfo + cgroupPaths map[string]string + fsInfo fs.FsInfo + fsHandler common.FsHandler + rootfsStorageDir string // Metadata associated with the container. reference info.ContainerReference envs map[string]string @@ -64,6 +67,7 @@ func newContainerdContainerHandler( inHostNamespace bool, metadataEnvAllowList []string, includedMetrics container.MetricSet, + rootfsDir string, ) (container.ContainerHandler, error) { // Create the cgroup paths. cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems, name) @@ -146,6 +150,10 @@ func newContainerdContainerHandler( } // Add the name and bare ID as aliases of the container. handler.image = cntr.Image + if handler.includedMetrics.Has(container.DiskUsageMetrics) { + handler.rootfsStorageDir = path.Join(rootfsDir, *ArgContainerdNamespace, id, "rootfs") + handler.fsHandler = common.NewFsHandler(common.DefaultPeriod, handler.rootfsStorageDir, "", fsInfo) + } for _, exposedEnv := range metadataEnvAllowList { if exposedEnv == "" { @@ -171,9 +179,7 @@ func (h *containerdContainerHandler) ContainerReference() (info.ContainerReferen } func (h *containerdContainerHandler) GetSpec() (info.ContainerSpec, error) { - // TODO: Since we dont collect disk usage stats for containerd, we set hasFilesystem - // to false. Revisit when we support disk usage stats for containerd - hasFilesystem := false + hasFilesystem := h.includedMetrics.Has(container.DiskUsageMetrics) hasNet := h.includedMetrics.Has(container.NetworkUsageMetrics) spec, err := common.GetSpec(h.cgroupPaths, h.machineInfoFactory, hasNet, hasFilesystem) spec.Labels = h.labels @@ -192,6 +198,34 @@ func (h *containerdContainerHandler) getFsStats(stats *info.ContainerStats) erro if h.includedMetrics.Has(container.DiskIOMetrics) { common.AssignDeviceNamesToDiskStats((*common.MachineInfoNamer)(mi), &stats.DiskIo) } + + if h.includedMetrics.Has(container.DiskUsageMetrics) { + deviceInfo, err := h.fsInfo.GetDirFsDevice(h.rootfsStorageDir) + if err != nil { + return fmt.Errorf("unable to determine device info for dir: %v: %v", h.rootfsStorageDir, err) + } + for _, fileSystem := range mi.Filesystems { + if fileSystem.Device == deviceInfo.Device { + usage := h.fsHandler.Usage() + fsStat := info.FsStats{ + Device: deviceInfo.Device, + Type: fileSystem.Type, + Limit: fileSystem.Capacity, + BaseUsage: usage.BaseUsageBytes, + Usage: usage.TotalUsageBytes, + Inodes: usage.InodeUsage, + } + fileSystems, err := h.fsInfo.GetGlobalFsInfo() + if err != nil { + return fmt.Errorf("unable to obtain diskstats for filesystem %s: %v", fsStat.Device, err) + } + fs.AddDiskStats(fileSystems, &fileSystem, &fsStat) + stats.Filesystem = append(stats.Filesystem, fsStat) + break + } + } + + } return nil } @@ -239,9 +273,15 @@ func (h *containerdContainerHandler) Type() container.ContainerType { } func (h *containerdContainerHandler) Start() { + if h.fsHandler != nil { + h.fsHandler.Start() + } } func (h *containerdContainerHandler) Cleanup() { + if h.fsHandler != nil { + h.fsHandler.Stop() + } } func (h *containerdContainerHandler) GetContainerIPAddress() string { diff --git a/container/containerd/handler_test.go b/container/containerd/handler_test.go index aeb86ee9b5..eac406cfa1 100644 --- a/container/containerd/handler_test.go +++ b/container/containerd/handler_test.go @@ -16,6 +16,7 @@ package containerd import ( + "context" "testing" "github.com/containerd/typeurl/v2" @@ -121,7 +122,8 @@ func TestHandler(t *testing.T) { map[string]string{"TEST_REGION": "FRA", "TEST_ZONE": "A"}, }, } { - handler, err := newContainerdContainerHandler(ts.client, ts.name, ts.machineInfoFactory, ts.fsInfo, ts.cgroupSubsystems, ts.inHostNamespace, ts.metadataEnvAllowList, ts.includedMetrics) + rootfsDir, _ := ts.client.RootfsDir(context.TODO()) + handler, err := newContainerdContainerHandler(ts.client, ts.name, ts.machineInfoFactory, ts.fsInfo, ts.cgroupSubsystems, ts.inHostNamespace, ts.metadataEnvAllowList, ts.includedMetrics, rootfsDir) if ts.hasErr { as.NotNil(err) if ts.errContains != "" { diff --git a/container/docker/fs.go b/container/docker/fs.go index 79384d0e40..89420264c3 100644 --- a/container/docker/fs.go +++ b/container/docker/fs.go @@ -64,13 +64,13 @@ func FsStats( return nil } - for _, fs := range mi.Filesystems { - if fs.Device == device { + for _, fileSystem := range mi.Filesystems { + if fileSystem.Device == device { usage := fsHandler.Usage() fsStat := info.FsStats{ Device: device, - Type: fs.Type, - Limit: fs.Capacity, + Type: fileSystem.Type, + Limit: fileSystem.Capacity, BaseUsage: usage.BaseUsageBytes, Usage: usage.TotalUsageBytes, Inodes: usage.InodeUsage, @@ -79,7 +79,7 @@ func FsStats( if err != nil { return fmt.Errorf("unable to obtain diskstats for filesystem %s: %v", fsStat.Device, err) } - addDiskStats(fileSystems, &fs, &fsStat) + fs.AddDiskStats(fileSystems, &fileSystem, &fsStat) stats.Filesystem = append(stats.Filesystem, fsStat) break } @@ -89,30 +89,6 @@ func FsStats( return nil } -func addDiskStats(fileSystems []fs.Fs, fsInfo *info.FsInfo, fsStats *info.FsStats) { - if fsInfo == nil { - return - } - - for _, fileSys := range fileSystems { - if fsInfo.DeviceMajor == fileSys.DiskStats.Major && - fsInfo.DeviceMinor == fileSys.DiskStats.Minor { - fsStats.ReadsCompleted = fileSys.DiskStats.ReadsCompleted - fsStats.ReadsMerged = fileSys.DiskStats.ReadsMerged - fsStats.SectorsRead = fileSys.DiskStats.SectorsRead - fsStats.ReadTime = fileSys.DiskStats.ReadTime - fsStats.WritesCompleted = fileSys.DiskStats.WritesCompleted - fsStats.WritesMerged = fileSys.DiskStats.WritesMerged - fsStats.SectorsWritten = fileSys.DiskStats.SectorsWritten - fsStats.WriteTime = fileSys.DiskStats.WriteTime - fsStats.IoInProgress = fileSys.DiskStats.IoInProgress - fsStats.IoTime = fileSys.DiskStats.IoTime - fsStats.WeightedIoTime = fileSys.DiskStats.WeightedIoTime - break - } - } -} - // FsHandler is a composite FsHandler implementation the incorporates // the common fs handler, a devicemapper ThinPoolWatcher, and a zfsWatcher type FsHandler struct { diff --git a/container/docker/handler_test.go b/container/docker/handler_test.go index e65acc9f6f..706fa0e568 100644 --- a/container/docker/handler_test.go +++ b/container/docker/handler_test.go @@ -23,9 +23,6 @@ import ( "github.com/docker/docker/api/types/container" "github.com/stretchr/testify/assert" - - "github.com/google/cadvisor/fs" - info "github.com/google/cadvisor/info/v1" ) func TestStorageDirDetectionWithOldVersions(t *testing.T) { @@ -145,76 +142,3 @@ func TestDockerEnvWhitelist(t *testing.T) { as.Equal(rawEnvsMatchWithEmptyWhitelist, emptyExpected) } - -func TestAddDiskStatsCheck(t *testing.T) { - var readsCompleted, readsMerged, sectorsRead, readTime, writesCompleted, writesMerged, sectorsWritten, - writeTime, ioInProgress, ioTime, weightedIoTime uint64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 - - fileSystem := fs.Fs{ - DiskStats: fs.DiskStats{ - ReadsCompleted: readsCompleted, - ReadsMerged: readsMerged, - SectorsRead: sectorsRead, - ReadTime: readTime, - WritesCompleted: writesCompleted, - WritesMerged: writesMerged, - SectorsWritten: sectorsWritten, - WriteTime: writeTime, - IoInProgress: ioInProgress, - IoTime: ioTime, - WeightedIoTime: weightedIoTime, - }, - } - - fileSystems := []fs.Fs{fileSystem} - - var fsStats info.FsStats - addDiskStats(fileSystems, nil, &fsStats) -} - -func TestAddDiskStats(t *testing.T) { - // Arrange - as := assert.New(t) - var readsCompleted, readsMerged, sectorsRead, readTime, writesCompleted, writesMerged, sectorsWritten, - writeTime, ioInProgress, ioTime, weightedIoTime uint64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 - var fsStats info.FsStats - - fsInfo := info.FsInfo{ - DeviceMajor: 4, - DeviceMinor: 64, - } - - fileSystem := fs.Fs{ - DiskStats: fs.DiskStats{ - ReadsCompleted: readsCompleted, - ReadsMerged: readsMerged, - SectorsRead: sectorsRead, - ReadTime: readTime, - WritesCompleted: writesCompleted, - WritesMerged: writesMerged, - SectorsWritten: sectorsWritten, - WriteTime: writeTime, - IoInProgress: ioInProgress, - IoTime: ioTime, - WeightedIoTime: weightedIoTime, - }, - } - - fileSystems := []fs.Fs{fileSystem} - - // Act - addDiskStats(fileSystems, &fsInfo, &fsStats) - - // Assert - as.Equal(readsCompleted, fileSystem.DiskStats.ReadsCompleted, "ReadsCompleted metric should be %d but was %d", readsCompleted, fileSystem.DiskStats.ReadsCompleted) - as.Equal(readsMerged, fileSystem.DiskStats.ReadsMerged, "ReadsMerged metric should be %d but was %d", readsMerged, fileSystem.DiskStats.ReadsMerged) - as.Equal(sectorsRead, fileSystem.DiskStats.SectorsRead, "SectorsRead metric should be %d but was %d", sectorsRead, fileSystem.DiskStats.SectorsRead) - as.Equal(readTime, fileSystem.DiskStats.ReadTime, "ReadTime metric should be %d but was %d", readTime, fileSystem.DiskStats.ReadTime) - as.Equal(writesCompleted, fileSystem.DiskStats.WritesCompleted, "WritesCompleted metric should be %d but was %d", writesCompleted, fileSystem.DiskStats.WritesCompleted) - as.Equal(writesMerged, fileSystem.DiskStats.WritesMerged, "WritesMerged metric should be %d but was %d", writesMerged, fileSystem.DiskStats.WritesMerged) - as.Equal(sectorsWritten, fileSystem.DiskStats.SectorsWritten, "SectorsWritten metric should be %d but was %d", sectorsWritten, fileSystem.DiskStats.SectorsWritten) - as.Equal(writeTime, fileSystem.DiskStats.WriteTime, "WriteTime metric should be %d but was %d", writeTime, fileSystem.DiskStats.WriteTime) - as.Equal(ioInProgress, fileSystem.DiskStats.IoInProgress, "IoInProgress metric should be %d but was %d", ioInProgress, fileSystem.DiskStats.IoInProgress) - as.Equal(ioTime, fileSystem.DiskStats.IoTime, "IoTime metric should be %d but was %d", ioTime, fileSystem.DiskStats.IoTime) - as.Equal(weightedIoTime, fileSystem.DiskStats.WeightedIoTime, "WeightedIoTime metric should be %d but was %d", weightedIoTime, fileSystem.DiskStats.WeightedIoTime) -} diff --git a/fs/fs.go b/fs/fs.go index 4c4de51931..5a47504bf8 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -22,6 +22,7 @@ import ( "bufio" "context" "fmt" + info "github.com/google/cadvisor/info/v1" "os" "os/exec" "path" @@ -880,3 +881,27 @@ func getBtrfsMajorMinorIds(mount *mount.Info) (int, int, error) { } return 0, 0, fmt.Errorf("%s is not a block device", mount.Source) } + +func AddDiskStats(fileSystems []Fs, fsInfo *info.FsInfo, fsStats *info.FsStats) { + if fsInfo == nil { + return + } + + for _, fileSys := range fileSystems { + if fsInfo.DeviceMajor == fileSys.DiskStats.Major && + fsInfo.DeviceMinor == fileSys.DiskStats.Minor { + fsStats.ReadsCompleted = fileSys.DiskStats.ReadsCompleted + fsStats.ReadsMerged = fileSys.DiskStats.ReadsMerged + fsStats.SectorsRead = fileSys.DiskStats.SectorsRead + fsStats.ReadTime = fileSys.DiskStats.ReadTime + fsStats.WritesCompleted = fileSys.DiskStats.WritesCompleted + fsStats.WritesMerged = fileSys.DiskStats.WritesMerged + fsStats.SectorsWritten = fileSys.DiskStats.SectorsWritten + fsStats.WriteTime = fileSys.DiskStats.WriteTime + fsStats.IoInProgress = fileSys.DiskStats.IoInProgress + fsStats.IoTime = fileSys.DiskStats.IoTime + fsStats.WeightedIoTime = fileSys.DiskStats.WeightedIoTime + break + } + } +} diff --git a/fs/fs_test.go b/fs/fs_test.go index 24e878c0a9..25e61a40f6 100644 --- a/fs/fs_test.go +++ b/fs/fs_test.go @@ -16,6 +16,7 @@ package fs import ( "errors" + "github.com/google/cadvisor/info/v1" "os" "reflect" "testing" @@ -700,3 +701,76 @@ func TestProcessMounts(t *testing.T) { } } } + +func TestAddDiskStatsCheck(t *testing.T) { + var readsCompleted, readsMerged, sectorsRead, readTime, writesCompleted, writesMerged, sectorsWritten, + writeTime, ioInProgress, ioTime, weightedIoTime uint64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + + fileSystem := Fs{ + DiskStats: DiskStats{ + ReadsCompleted: readsCompleted, + ReadsMerged: readsMerged, + SectorsRead: sectorsRead, + ReadTime: readTime, + WritesCompleted: writesCompleted, + WritesMerged: writesMerged, + SectorsWritten: sectorsWritten, + WriteTime: writeTime, + IoInProgress: ioInProgress, + IoTime: ioTime, + WeightedIoTime: weightedIoTime, + }, + } + + fileSystems := []Fs{fileSystem} + + var fsStats v1.FsStats + AddDiskStats(fileSystems, nil, &fsStats) +} + +func TestAddDiskStats(t *testing.T) { + // Arrange + as := assert.New(t) + var readsCompleted, readsMerged, sectorsRead, readTime, writesCompleted, writesMerged, sectorsWritten, + writeTime, ioInProgress, ioTime, weightedIoTime uint64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + var fsStats v1.FsStats + + fsInfo := v1.FsInfo{ + DeviceMajor: 4, + DeviceMinor: 64, + } + + fileSystem := Fs{ + DiskStats: DiskStats{ + ReadsCompleted: readsCompleted, + ReadsMerged: readsMerged, + SectorsRead: sectorsRead, + ReadTime: readTime, + WritesCompleted: writesCompleted, + WritesMerged: writesMerged, + SectorsWritten: sectorsWritten, + WriteTime: writeTime, + IoInProgress: ioInProgress, + IoTime: ioTime, + WeightedIoTime: weightedIoTime, + }, + } + + fileSystems := []Fs{fileSystem} + + // Act + AddDiskStats(fileSystems, &fsInfo, &fsStats) + + // Assert + as.Equal(readsCompleted, fileSystem.DiskStats.ReadsCompleted, "ReadsCompleted metric should be %d but was %d", readsCompleted, fileSystem.DiskStats.ReadsCompleted) + as.Equal(readsMerged, fileSystem.DiskStats.ReadsMerged, "ReadsMerged metric should be %d but was %d", readsMerged, fileSystem.DiskStats.ReadsMerged) + as.Equal(sectorsRead, fileSystem.DiskStats.SectorsRead, "SectorsRead metric should be %d but was %d", sectorsRead, fileSystem.DiskStats.SectorsRead) + as.Equal(readTime, fileSystem.DiskStats.ReadTime, "ReadTime metric should be %d but was %d", readTime, fileSystem.DiskStats.ReadTime) + as.Equal(writesCompleted, fileSystem.DiskStats.WritesCompleted, "WritesCompleted metric should be %d but was %d", writesCompleted, fileSystem.DiskStats.WritesCompleted) + as.Equal(writesMerged, fileSystem.DiskStats.WritesMerged, "WritesMerged metric should be %d but was %d", writesMerged, fileSystem.DiskStats.WritesMerged) + as.Equal(sectorsWritten, fileSystem.DiskStats.SectorsWritten, "SectorsWritten metric should be %d but was %d", sectorsWritten, fileSystem.DiskStats.SectorsWritten) + as.Equal(writeTime, fileSystem.DiskStats.WriteTime, "WriteTime metric should be %d but was %d", writeTime, fileSystem.DiskStats.WriteTime) + as.Equal(ioInProgress, fileSystem.DiskStats.IoInProgress, "IoInProgress metric should be %d but was %d", ioInProgress, fileSystem.DiskStats.IoInProgress) + as.Equal(ioTime, fileSystem.DiskStats.IoTime, "IoTime metric should be %d but was %d", ioTime, fileSystem.DiskStats.IoTime) + as.Equal(weightedIoTime, fileSystem.DiskStats.WeightedIoTime, "WeightedIoTime metric should be %d but was %d", weightedIoTime, fileSystem.DiskStats.WeightedIoTime) +} diff --git a/go.mod b/go.mod index 9e4467bd20..473a31a96b 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( golang.org/x/sys v0.21.0 google.golang.org/grpc v1.64.1 google.golang.org/protobuf v1.34.1 + k8s.io/cri-api v0.29.3 k8s.io/klog/v2 v2.100.1 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 ) diff --git a/go.sum b/go.sum index 2a4659a3e4..e715622b0d 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,9 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -303,6 +304,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -635,8 +638,9 @@ google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -656,6 +660,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/cri-api v0.29.3 h1:ppKSui+hhTJW774Mou6x+/ealmzt2jmTM0vsEQVWrjI= +k8s.io/cri-api v0.29.3/go.mod h1:3X7EnhsNaQnCweGhQCJwKNHlH7wHEYuKQ19bRvXMoJY= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=