fix fs_limit, fs_inode_total, fs_inode_free in containerd handler

Author: qiutongs

container/containerd/client.go

@@ -22,8 +22,10 @@ import (
 	"time"
 
 	containersapi "github.com/containerd/containerd/api/services/containers/v1"
+	snapshotapi "github.com/containerd/containerd/api/services/snapshots/v1"
 	tasksapi "github.com/containerd/containerd/api/services/tasks/v1"
 	versionapi "github.com/containerd/containerd/api/services/version/v1"
+	types "github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/pkg/dialer"
@@ -37,13 +39,15 @@ type client struct {
 	containerService containersapi.ContainersClient
 	taskService      tasksapi.TasksClient
 	versionService   versionapi.VersionClient
+	snapshotService  snapshotapi.SnapshotsClient
 	criService       criapi.RuntimeServiceClient
 }
 
 type ContainerdClient interface {
 	LoadContainer(ctx context.Context, id string) (*containers.Container, error)
 	TaskPid(ctx context.Context, id string) (uint32, error)
 	Version(ctx context.Context) (string, error)
+	SnapshotMounts(ctx context.Context, snapshotter, key string) ([]*types.Mount, error)
 	ContainerStatus(ctx context.Context, id string) (*criapi.ContainerStatus, error)
 	ContainerStats(ctx context.Context, id string) (*criapi.ContainerStats, error)
 }
@@ -96,6 +100,7 @@ func Client(address, namespace string) (ContainerdClient, error) {
 			containerService: containersapi.NewContainersClient(conn),
 			taskService:      tasksapi.NewTasksClient(conn),
 			versionService:   versionapi.NewVersionClient(conn),
+			snapshotService:  snapshotapi.NewSnapshotsClient(conn),
 			criService:       criapi.NewRuntimeServiceClient(conn),
 		}
 	})
@@ -130,6 +135,17 @@ func (c *client) Version(ctx context.Context) (string, error) {
 	return response.Version, nil
 }
 
+func (c *client) SnapshotMounts(ctx context.Context, snapshotter, key string) ([]*types.Mount, error) {
+	response, err := c.snapshotService.Mounts(ctx, &snapshotapi.MountsRequest{
+		Snapshotter: snapshotter,
+		Key:         key,
+	})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	return response.Mounts, nil
+}
+
 func (c *client) ContainerStatus(ctx context.Context, id string) (*criapi.ContainerStatus, error) {
 	response, err := c.criService.ContainerStatus(ctx, &criapi.ContainerStatusRequest{
 		ContainerId: id,

container/containerd/client_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 
+	types "github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/containers"
 	criapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
@@ -56,6 +57,10 @@ func (c *containerdClientMock) ContainerStats(ctx context.Context, id string) (*
 	return c.stats, nil
 }
 
+func (c *containerdClientMock) SnapshotMounts(ctx context.Context, snapshotter, key string) ([]*types.Mount, error) {
+	return nil, nil
+}
+
 func mockcontainerdClient(cntrs map[string]*containers.Container, status *criapi.ContainerStatus, stats *criapi.ContainerStats, returnErr error) ContainerdClient {
 	return &containerdClientMock{
 		cntrs:     cntrs,

container/containerd/handler.go

@@ -42,9 +42,14 @@ type containerdContainerHandler struct {
 	machineInfoFactory info.MachineInfoFactory
 	// Absolute path to the cgroup hierarchies of this container.
 	// (e.g.: "cpu" -> "/sys/fs/cgroup/cpu/test")
-	cgroupPaths map[string]string
-	fsInfo      fs.FsInfo
-	fsHandler   common.FsHandler
+	cgroupPaths   map[string]string
+	fsInfo        fs.FsInfo
+	fsHandler     common.FsHandler
+	fsLimit       uint64
+	fsTotalInodes uint64
+	fsType        string
+	device        string
+
 	// Metadata associated with the container.
 	reference info.ContainerReference
 	envs      map[string]string
@@ -165,6 +170,52 @@ func newContainerdContainerHandler(
 	handler.image = cntr.Image
 
 	if includedMetrics.Has(container.DiskUsageMetrics) && cntr.Labels["io.cri-containerd.kind"] != "sandbox" {
+		mounts, err := client.SnapshotMounts(ctx, cntr.Snapshotter, cntr.SnapshotKey)
+		if err != nil {
+			return nil, err
+		}
+
+		// Default to top directory if the specific upperdir snapshot is not found
+		snapshotDir := "/var/lib/containerd"
+		// Example: upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/5001/fs
+		if len(mounts) > 0 {
+			for _, option := range mounts[0].Options {
+				if strings.Index(option, "upperdir=") == 0 {
+					snapshotDir = option[len("upperdir="):]
+					break
+				}
+			}
+		}
+		deviceInfo, err := fsInfo.GetDirFsDevice(snapshotDir)
+		if err != nil {
+			return nil, err
+		}
+
+		mi, err := machineInfoFactory.GetMachineInfo()
+		if err != nil {
+			return nil, err
+		}
+
+		var (
+			fsLimit       uint64
+			fsType        string
+			fsTotalInodes uint64
+		)
+		// Containerd does not impose any filesystem limits for containers. So use capacity as limit.
+		for _, fs := range mi.Filesystems {
+			if fs.Device == deviceInfo.Device {
+				fsLimit = fs.Capacity
+				fsType = fs.Type
+				fsTotalInodes = fs.Inodes
+				break
+			}
+		}
+
+		handler.fsLimit = fsLimit
+		handler.fsType = fsType
+		handler.fsTotalInodes = fsTotalInodes
+		handler.device = deviceInfo.Device
+
 		handler.fsHandler = common.NewFsHandler(common.DefaultPeriod, &fsUsageProvider{
 			ctx:         ctx,
 			client:      client,
@@ -232,12 +283,15 @@ func (h *containerdContainerHandler) getFsStats(stats *info.ContainerStats) erro
 		return nil
 	}
 
-	// Device 、fsType and fsLimits and other information are not supported yet
 	fsStat := info.FsStats{}
 	usage := h.fsHandler.Usage()
 	fsStat.BaseUsage = usage.BaseUsageBytes
 	fsStat.Usage = usage.TotalUsageBytes
-	fsStat.Inodes = usage.InodeUsage
+	fsStat.InodesFree = h.fsTotalInodes - usage.InodeUsage
+	fsStat.Inodes = h.fsTotalInodes
+	fsStat.Device = h.device
+	fsStat.Limit = h.fsLimit
+	fsStat.Type = h.fsType
 
 	stats.Filesystem = append(stats.Filesystem, fsStat)