Skip to content

Commit 455b460

Browse files
machine424machine424
committed
Add kubelet_pod_emptydir_volume_used_bytes Prometheus metric to expose emptyDir used bytes per Pod
1 parent 03ba7ef commit 455b460

File tree

4 files changed

+277
-0
lines changed

4 files changed

+277
-0
lines changed

‎pkg/kubelet/kubelet.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1458,6 +1458,7 @@ func (kl *Kubelet) initializeModules() error {
14581458
metrics.Register(
14591459
collectors.NewVolumeStatsCollector(kl),
14601460
collectors.NewLogMetricsCollector(kl.StatsProvider.ListPodStats),
1461+
collectors.NewEmptyDirMetricsCollector(kl),
14611462
)
14621463
metrics.SetNodeName(kl.nodeName)
14631464
servermetrics.Register()

‎pkg/kubelet/metrics/collectors/emptydir_metrics.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package collectors
18+
19+
import (
20+
"context"
21+
22+
"k8s.io/api/core/v1"
23+
"k8s.io/component-base/metrics"
24+
"k8s.io/klog/v2"
25+
26+
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
27+
serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
28+
)
29+
30+
var (
31+
emptyDirUsedBytesDesc = metrics.NewDesc(
32+
metrics.BuildFQName(
33+
"",
34+
kubeletmetrics.KubeletSubsystem,
35+
kubeletmetrics.EmptyDirUsedBytesKey,
36+
),
37+
"Bytes used by the emptyDir volume.",
38+
[]string{
39+
"volume_name",
40+
"namespace",
41+
"pod",
42+
},
43+
nil,
44+
metrics.ALPHA,
45+
"",
46+
)
47+
)
48+
49+
type emptyDirMetricsCollector struct {
50+
metrics.BaseStableCollector
51+
52+
statsProvider serverstats.Provider
53+
}
54+
55+
// Check if emptyDirMetricsCollector implements necessary interface
56+
var _ metrics.StableCollector = &emptyDirMetricsCollector{}
57+
58+
// NewEmptyDirMetricsCollector implements the metrics.StableCollector interface and
59+
// exposes metrics about pod's emptyDir.
60+
func NewEmptyDirMetricsCollector(statsProvider serverstats.Provider) metrics.StableCollector {
61+
return &emptyDirMetricsCollector{statsProvider: statsProvider}
62+
}
63+
64+
// DescribeWithStability implements the metrics.StableCollector interface.
65+
func (c *emptyDirMetricsCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
66+
ch <- emptyDirUsedBytesDesc
67+
}
68+
69+
// CollectWithStability implements the metrics.StableCollector interface.
70+
func (c *emptyDirMetricsCollector) CollectWithStability(ch chan<- metrics.Metric) {
71+
podStats, err := c.statsProvider.ListPodStats(context.Background())
72+
if err != nil {
73+
klog.ErrorS(err, "Failed to get pod stats")
74+
return
75+
}
76+
77+
for _, podStat := range podStats {
78+
podName := podStat.PodRef.Name
79+
podNamespace := podStat.PodRef.Namespace
80+
81+
if podStat.VolumeStats == nil {
82+
// TODO: another level?
83+
klog.V(5).InfoS("Pod has no volume stats", "pod", podName, "namespace", podNamespace)
84+
continue
85+
}
86+
87+
pod, found := c.statsProvider.GetPodByName(podNamespace, podName)
88+
if !found {
89+
// TODO: another level?
90+
klog.V(5).InfoS("Couldn't get pod", "pod", podName, "namespace", podNamespace)
91+
continue
92+
}
93+
94+
podVolumes := make(map[string]v1.Volume, len(pod.Spec.Volumes))
95+
for _, volume := range pod.Spec.Volumes {
96+
podVolumes[volume.Name] = volume
97+
}
98+
99+
for _, volumeStat := range podStat.VolumeStats {
100+
if volume, found := podVolumes[volumeStat.Name]; found {
101+
if volume.EmptyDir != nil && volume.EmptyDir.Medium == v1.StorageMediumDefault {
102+
// TODO: Expose even if not present, with NaN? Add debug log?
103+
if volumeStat.UsedBytes != nil {
104+
ch <- metrics.NewLazyConstMetric(
105+
emptyDirUsedBytesDesc,
106+
metrics.GaugeValue,
107+
float64(*volumeStat.UsedBytes),
108+
volumeStat.Name,
109+
podNamespace,
110+
podName,
111+
)
112+
}
113+
}
114+
}
115+
116+
}
117+
}
118+
}

‎pkg/kubelet/metrics/collectors/emptydir_metrics_test.go

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package collectors
18+
19+
import (
20+
"context"
21+
"strings"
22+
"testing"
23+
24+
"github.com/golang/mock/gomock"
25+
26+
v1 "k8s.io/api/core/v1"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/component-base/metrics/testutil"
29+
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
30+
statstest "k8s.io/kubernetes/pkg/kubelet/server/stats/testing"
31+
)
32+
33+
func TestEmptyDirCollector(t *testing.T) {
34+
35+
testNamespace := "test-namespace"
36+
existingPodNameWithStats := "foo"
37+
podNameWithoutStats := "bar"
38+
39+
podStats := []statsapi.PodStats{
40+
{
41+
PodRef: statsapi.PodReference{
42+
Name: existingPodNameWithStats,
43+
Namespace: testNamespace,
44+
UID: "UID_foo",
45+
},
46+
StartTime: metav1.Now(),
47+
VolumeStats: []statsapi.VolumeStats{
48+
{
49+
Name: "foo-emptydir-1",
50+
FsStats: statsapi.FsStats{
51+
UsedBytes: newUint64Pointer(2101248),
52+
},
53+
},
54+
{
55+
Name: "foo-emptydir-2",
56+
FsStats: statsapi.FsStats{
57+
UsedBytes: newUint64Pointer(6488064),
58+
},
59+
},
60+
{
61+
Name: "foo-memory-emptydir",
62+
FsStats: statsapi.FsStats{
63+
UsedBytes: newUint64Pointer(25362432),
64+
},
65+
},
66+
{
67+
Name: "foo-configmap",
68+
FsStats: statsapi.FsStats{
69+
UsedBytes: newUint64Pointer(4096),
70+
},
71+
},
72+
},
73+
},
74+
}
75+
76+
existingPod := &v1.Pod{
77+
ObjectMeta: metav1.ObjectMeta{
78+
Name: existingPodNameWithStats,
79+
Namespace: testNamespace,
80+
},
81+
Spec: v1.PodSpec{
82+
Volumes: []v1.Volume{
83+
{
84+
Name: "foo-emptydir-1",
85+
VolumeSource: v1.VolumeSource{
86+
EmptyDir: &v1.EmptyDirVolumeSource{},
87+
},
88+
},
89+
{
90+
Name: "foo-emptydir-2",
91+
VolumeSource: v1.VolumeSource{
92+
EmptyDir: &v1.EmptyDirVolumeSource{},
93+
},
94+
},
95+
{
96+
Name: "foo-memory-emptydir",
97+
VolumeSource: v1.VolumeSource{
98+
EmptyDir: &v1.EmptyDirVolumeSource{Medium: v1.StorageMediumMemory},
99+
},
100+
},
101+
{
102+
Name: "foo-configmap",
103+
VolumeSource: v1.VolumeSource{
104+
ConfigMap: &v1.ConfigMapVolumeSource{},
105+
},
106+
},
107+
},
108+
},
109+
}
110+
111+
podWithoutStats := &v1.Pod{
112+
ObjectMeta: metav1.ObjectMeta{
113+
Name: podNameWithoutStats,
114+
Namespace: testNamespace,
115+
},
116+
Spec: v1.PodSpec{
117+
Volumes: []v1.Volume{
118+
{
119+
Name: "bar-emptydir",
120+
VolumeSource: v1.VolumeSource{
121+
EmptyDir: &v1.EmptyDirVolumeSource{},
122+
},
123+
},
124+
},
125+
},
126+
}
127+
128+
ctx := context.Background()
129+
130+
mockCtrl := gomock.NewController(t)
131+
mockStatsProvider := statstest.NewMockProvider(mockCtrl)
132+
133+
mockStatsProvider.EXPECT().ListPodStats(ctx).Return(podStats, nil).AnyTimes()
134+
mockStatsProvider.EXPECT().
135+
GetPodByName(testNamespace, existingPodNameWithStats).
136+
Return(existingPod, true).
137+
AnyTimes()
138+
mockStatsProvider.EXPECT().
139+
GetPodByName(testNamespace, podNameWithoutStats).
140+
Return(podWithoutStats, true).
141+
AnyTimes()
142+
143+
err := testutil.CustomCollectAndCompare(
144+
&emptyDirMetricsCollector{statsProvider: mockStatsProvider},
145+
strings.NewReader(`
146+
# HELP kubelet_pod_emptydir_volume_used_bytes [ALPHA] Bytes used by the emptyDir volume.
147+
# TYPE kubelet_pod_emptydir_volume_used_bytes gauge
148+
kubelet_pod_emptydir_volume_used_bytes{namespace="test-namespace",pod="foo",volume_name="foo-emptydir-1"} 2.101248e+06
149+
kubelet_pod_emptydir_volume_used_bytes{namespace="test-namespace",pod="foo",volume_name="foo-emptydir-2"} 6.488064e+06
150+
`),
151+
"kubelet_pod_emptydir_volume_used_bytes",
152+
)
153+
if err != nil {
154+
t.Fatal(err)
155+
}
156+
157+
}

‎pkg/kubelet/metrics/metrics.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ const (
5656
PreemptionsKey = "preemptions"
5757
VolumeStatsCapacityBytesKey = "volume_stats_capacity_bytes"
5858
VolumeStatsAvailableBytesKey = "volume_stats_available_bytes"
59+
EmptyDirUsedBytesKey = "pod_emptydir_volume_used_bytes"
5960
VolumeStatsUsedBytesKey = "volume_stats_used_bytes"
6061
VolumeStatsInodesKey = "volume_stats_inodes"
6162
VolumeStatsInodesFreeKey = "volume_stats_inodes_free"

0 commit comments

Comments
 (0)