Skip to content

Commit 98b6a0b

Browse files
machine424machine424
committed
Add Prometheus metrics to expose emptyDir used bytes and size limit
per pod/volume. Signed-off-by: machine424 <ayoubmrini424@gmail.com>
1 parent bbd83d8 commit 98b6a0b

File tree

4 files changed

+303
-0
lines changed

4 files changed

+303
-0
lines changed

‎pkg/kubelet/kubelet.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1497,6 +1497,7 @@ func (kl *Kubelet) initializeModules() error {
14971497
metrics.Register(
14981498
collectors.NewVolumeStatsCollector(kl),
14991499
collectors.NewLogMetricsCollector(kl.StatsProvider.ListPodStats),
1500+
collectors.NewEmptyDirMetricsCollector(kl),
15001501
)
15011502
metrics.SetNodeName(kl.nodeName)
15021503
servermetrics.Register()

‎pkg/kubelet/metrics/collectors/emptydir_metrics.go

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package collectors
18+
19+
import (
20+
"context"
21+
22+
"k8s.io/api/core/v1"
23+
"k8s.io/component-base/metrics"
24+
"k8s.io/klog/v2"
25+
26+
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
27+
serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
28+
)
29+
30+
var (
31+
emptyDirUsedBytesDesc = metrics.NewDesc(
32+
metrics.BuildFQName(
33+
"",
34+
kubeletmetrics.KubeletSubsystem,
35+
kubeletmetrics.EmptyDirUsedBytesKey,
36+
),
37+
"Bytes used by the emptyDir volume.",
38+
[]string{
39+
"volume_name",
40+
"namespace",
41+
"pod",
42+
},
43+
nil,
44+
metrics.ALPHA,
45+
"",
46+
)
47+
emptyDirSizeLimitBytesDesc = metrics.NewDesc(
48+
metrics.BuildFQName(
49+
"",
50+
kubeletmetrics.KubeletSubsystem,
51+
kubeletmetrics.EmptyDirSizeLimitBytesKey,
52+
),
53+
"Size limit of the emptyDir volume in bytes, if set.",
54+
[]string{
55+
"volume_name",
56+
"namespace",
57+
"pod",
58+
},
59+
nil,
60+
metrics.ALPHA,
61+
"",
62+
)
63+
)
64+
65+
type emptyDirMetricsCollector struct {
66+
metrics.BaseStableCollector
67+
68+
statsProvider serverstats.Provider
69+
}
70+
71+
// Check if emptyDirMetricsCollector implements necessary interface
72+
var _ metrics.StableCollector = &emptyDirMetricsCollector{}
73+
74+
// NewEmptyDirMetricsCollector implements the metrics.StableCollector interface and
75+
// exposes metrics about pod's emptyDir.
76+
func NewEmptyDirMetricsCollector(statsProvider serverstats.Provider) metrics.StableCollector {
77+
return &emptyDirMetricsCollector{statsProvider: statsProvider}
78+
}
79+
80+
// DescribeWithStability implements the metrics.StableCollector interface.
81+
func (c *emptyDirMetricsCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
82+
ch <- emptyDirUsedBytesDesc
83+
ch <- emptyDirSizeLimitBytesDesc
84+
}
85+
86+
// CollectWithStability implements the metrics.StableCollector interface.
87+
func (c *emptyDirMetricsCollector) CollectWithStability(ch chan<- metrics.Metric) {
88+
podStats, err := c.statsProvider.ListPodStats(context.Background())
89+
if err != nil {
90+
klog.ErrorS(err, "Failed to get pod stats")
91+
return
92+
}
93+
94+
for _, podStat := range podStats {
95+
podName := podStat.PodRef.Name
96+
podNamespace := podStat.PodRef.Namespace
97+
98+
if podStat.VolumeStats == nil {
99+
klog.V(5).InfoS("Pod has no volume stats", "pod", podName, "namespace", podNamespace)
100+
continue
101+
}
102+
103+
pod, found := c.statsProvider.GetPodByName(podNamespace, podName)
104+
if !found {
105+
klog.V(5).InfoS("Couldn't get pod", "pod", podName, "namespace", podNamespace)
106+
continue
107+
}
108+
109+
podVolumes := make(map[string]v1.Volume, len(pod.Spec.Volumes))
110+
for _, volume := range pod.Spec.Volumes {
111+
podVolumes[volume.Name] = volume
112+
}
113+
114+
for _, volumeStat := range podStat.VolumeStats {
115+
if volume, found := podVolumes[volumeStat.Name]; found {
116+
if volume.EmptyDir != nil && volume.EmptyDir.Medium == v1.StorageMediumDefault {
117+
if volumeStat.UsedBytes != nil {
118+
ch <- metrics.NewLazyConstMetric(
119+
emptyDirUsedBytesDesc,
120+
metrics.GaugeValue,
121+
float64(*volumeStat.UsedBytes),
122+
volumeStat.Name,
123+
podNamespace,
124+
podName,
125+
)
126+
}
127+
if volume.EmptyDir.SizeLimit != nil {
128+
ch <- metrics.NewLazyConstMetric(
129+
emptyDirSizeLimitBytesDesc,
130+
metrics.GaugeValue,
131+
volume.EmptyDir.SizeLimit.AsApproximateFloat64(),
132+
volumeStat.Name,
133+
podNamespace,
134+
podName,
135+
)
136+
}
137+
}
138+
}
139+
140+
}
141+
}
142+
}

‎pkg/kubelet/metrics/collectors/emptydir_metrics_test.go

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package collectors
18+
19+
import (
20+
"context"
21+
"strings"
22+
"testing"
23+
24+
"k8s.io/apimachinery/pkg/api/resource"
25+
26+
v1 "k8s.io/api/core/v1"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/component-base/metrics/testutil"
29+
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
30+
statstest "k8s.io/kubernetes/pkg/kubelet/server/stats/testing"
31+
)
32+
33+
func TestEmptyDirCollector(t *testing.T) {
34+
35+
testNamespace := "test-namespace"
36+
existingPodNameWithStats := "foo"
37+
podNameWithoutStats := "bar"
38+
39+
podStats := []statsapi.PodStats{
40+
{
41+
PodRef: statsapi.PodReference{
42+
Name: existingPodNameWithStats,
43+
Namespace: testNamespace,
44+
UID: "UID_foo",
45+
},
46+
StartTime: metav1.Now(),
47+
VolumeStats: []statsapi.VolumeStats{
48+
{
49+
Name: "foo-emptydir-1",
50+
FsStats: statsapi.FsStats{
51+
UsedBytes: newUint64Pointer(2101248),
52+
},
53+
},
54+
{
55+
Name: "foo-emptydir-2",
56+
FsStats: statsapi.FsStats{
57+
UsedBytes: newUint64Pointer(6488064),
58+
},
59+
},
60+
{
61+
Name: "foo-memory-emptydir",
62+
FsStats: statsapi.FsStats{
63+
UsedBytes: newUint64Pointer(25362432),
64+
},
65+
},
66+
{
67+
Name: "foo-configmap",
68+
FsStats: statsapi.FsStats{
69+
UsedBytes: newUint64Pointer(4096),
70+
},
71+
},
72+
},
73+
},
74+
}
75+
76+
existingPod := &v1.Pod{
77+
ObjectMeta: metav1.ObjectMeta{
78+
Name: existingPodNameWithStats,
79+
Namespace: testNamespace,
80+
},
81+
Spec: v1.PodSpec{
82+
Volumes: []v1.Volume{
83+
{
84+
Name: "foo-emptydir-1",
85+
VolumeSource: v1.VolumeSource{
86+
EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: resource.NewQuantity(3000100, resource.BinarySI)},
87+
},
88+
},
89+
{
90+
Name: "foo-emptydir-2",
91+
VolumeSource: v1.VolumeSource{
92+
EmptyDir: &v1.EmptyDirVolumeSource{},
93+
},
94+
},
95+
{
96+
Name: "foo-memory-emptydir",
97+
VolumeSource: v1.VolumeSource{
98+
EmptyDir: &v1.EmptyDirVolumeSource{Medium: v1.StorageMediumMemory},
99+
},
100+
},
101+
{
102+
Name: "foo-configmap",
103+
VolumeSource: v1.VolumeSource{
104+
ConfigMap: &v1.ConfigMapVolumeSource{},
105+
},
106+
},
107+
},
108+
},
109+
}
110+
111+
podWithoutStats := &v1.Pod{
112+
ObjectMeta: metav1.ObjectMeta{
113+
Name: podNameWithoutStats,
114+
Namespace: testNamespace,
115+
},
116+
Spec: v1.PodSpec{
117+
Volumes: []v1.Volume{
118+
{
119+
Name: "bar-emptydir",
120+
VolumeSource: v1.VolumeSource{
121+
EmptyDir: &v1.EmptyDirVolumeSource{},
122+
},
123+
},
124+
},
125+
},
126+
}
127+
128+
mockStatsProvider := statstest.NewMockProvider(t)
129+
130+
mockStatsProvider.EXPECT().ListPodStats(context.Background()).Return(podStats, nil).Maybe()
131+
mockStatsProvider.EXPECT().
132+
GetPodByName(testNamespace, existingPodNameWithStats).
133+
Return(existingPod, true).
134+
Maybe()
135+
mockStatsProvider.EXPECT().
136+
GetPodByName(testNamespace, podNameWithoutStats).
137+
Return(podWithoutStats, true).
138+
Maybe()
139+
140+
err := testutil.CustomCollectAndCompare(
141+
&emptyDirMetricsCollector{statsProvider: mockStatsProvider},
142+
strings.NewReader(`
143+
# HELP kubelet_pod_emptydir_volume_size_limit_bytes [ALPHA] Size limit of the emptyDir volume in bytes, if set.
144+
# TYPE kubelet_pod_emptydir_volume_size_limit_bytes gauge
145+
kubelet_pod_emptydir_volume_size_limit_bytes{namespace="test-namespace",pod="foo",volume_name="foo-emptydir-1"} 3.0001e+06
146+
# HELP kubelet_pod_emptydir_volume_used_bytes [ALPHA] Bytes used by the emptyDir volume.
147+
# TYPE kubelet_pod_emptydir_volume_used_bytes gauge
148+
kubelet_pod_emptydir_volume_used_bytes{namespace="test-namespace",pod="foo",volume_name="foo-emptydir-1"} 2.101248e+06
149+
kubelet_pod_emptydir_volume_used_bytes{namespace="test-namespace",pod="foo",volume_name="foo-emptydir-2"} 6.488064e+06
150+
`),
151+
"kubelet_pod_emptydir_volume_size_limit_bytes",
152+
"kubelet_pod_emptydir_volume_used_bytes",
153+
)
154+
if err != nil {
155+
t.Fatal(err)
156+
}
157+
158+
}

‎pkg/kubelet/metrics/metrics.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ const (
5858
PreemptionsKey = "preemptions"
5959
VolumeStatsCapacityBytesKey = "volume_stats_capacity_bytes"
6060
VolumeStatsAvailableBytesKey = "volume_stats_available_bytes"
61+
EmptyDirUsedBytesKey = "pod_emptydir_volume_used_bytes"
62+
EmptyDirSizeLimitBytesKey = "pod_emptydir_volume_size_limit_bytes"
6163
VolumeStatsUsedBytesKey = "volume_stats_used_bytes"
6264
VolumeStatsInodesKey = "volume_stats_inodes"
6365
VolumeStatsInodesFreeKey = "volume_stats_inodes_free"

0 commit comments

Comments
 (0)