Skip to content

Commit 70ef79b

Browse files
johnstcnjohnstcn
committed
improve tests to allow testing cpu used
1 parent fa0c4c6 commit 70ef79b

File tree

3 files changed

+238
-204
lines changed

3 files changed

+238
-204
lines changed

‎cli/clistat/cgroup.go

Lines changed: 132 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,45 @@ import (
44
"bufio"
55
"bytes"
66
"strconv"
7+
"strings"
78
"time"
89

910
"github.com/spf13/afero"
1011
"golang.org/x/xerrors"
1112
"tailscale.com/types/ptr"
1213
)
1314

15+
// Paths for CGroupV1.
16+
// Ref: https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
1417
const (
15-
cgroupV1CPUAcctUsage = "/sys/fs/cgroup/cpu/cpuacct.usage"
16-
cgroupV1CPUAcctUsageAlt = "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage"
17-
cgroupV1CFSQuotaUs = "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us"
18+
// CPU usage of all tasks in cgroup in nanoseconds.
19+
cgroupV1CPUAcctUsage = "/sys/fs/cgroup/cpu/cpuacct.usage"
20+
// Alternate path
21+
cgroupV1CPUAcctUsageAlt = "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage"
22+
// CFS quota and period for cgroup in MICROseconds
23+
cgroupV1CFSQuotaUs = "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us"
24+
cgroupV1CFSPeriodUs = "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us"
25+
// Maximum memory usable by cgroup in bytes
1826
cgroupV1MemoryMaxUsageBytes = "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
19-
cgroupV1MemoryUsageBytes = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
20-
cgroupV1MemoryStat = "/sys/fs/cgroup/memory/memory.stat"
21-
cgroupV2CPUMax = "/sys/fs/cgroup/cpu.max"
22-
cgroupV2CPUStat = "/sys/fs/cgroup/cpu.stat"
27+
// Current memory usage of cgroup in bytes
28+
cgroupV1MemoryUsageBytes = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
29+
// Other memory stats - we are interested in total_inactive_file
30+
cgroupV1MemoryStat = "/sys/fs/cgroup/memory/memory.stat"
31+
)
32+
33+
// Paths for CGroupV2.
34+
// Ref: https://docs.kernel.org/admin-guide/cgroup-v2.html
35+
const (
36+
// Contains quota and period in microseconds separated by a space.
37+
cgroupV2CPUMax = "/sys/fs/cgroup/cpu.max"
38+
// Contains current CPU usage under usage_usec
39+
cgroupV2CPUStat = "/sys/fs/cgroup/cpu.stat"
40+
// Contains current cgroup memory usage in bytes.
41+
cgroupV2MemoryUsageBytes = "/sys/fs/cgroup/memory.current"
42+
// Contains max cgroup memory usage in bytes.
43+
cgroupV2MemoryMaxBytes = "/sys/fs/cgroup/memory.max"
44+
// Other memory stats - we are interested in total_inactive_file
45+
cgroupV2MemoryStat = "/sys/fs/cgroup/memory.stat"
2346
)
2447

2548
// ContainerCPU returns the CPU usage of the container cgroup.
@@ -34,7 +57,7 @@ func (s *Statter) ContainerCPU() (*Result, error) {
3457
if err != nil {
3558
return nil, xerrors.Errorf("get cgroup CPU usage: %w", err)
3659
}
37-
<-time.After(s.sampleInterval)
60+
s.wait(s.sampleInterval)
3861

3962
// total is unlikely to change. Use the first value.
4063
used2, _, err := s.cgroupCPU()
@@ -44,7 +67,7 @@ func (s *Statter) ContainerCPU() (*Result, error) {
4467

4568
r := &Result{
4669
Unit: "cores",
47-
Used: (used2 - used1).Seconds() * s.sampleInterval.Seconds(),
70+
Used: (used2 - used1).Seconds(),
4871
Total: ptr.To(total.Seconds()), // close enough to the truth
4972
}
5073
return r, nil
@@ -68,72 +91,31 @@ func (s *Statter) isCGroupV2() bool {
6891
func (s *Statter) cGroupV2CPU() (used, total time.Duration, err error) {
6992
total, err = s.cGroupv2CPUTotal()
7093
if err != nil {
71-
return 0, 0, xerrors.Errorf("get cgroup v2 cpu total: %w", err)
94+
return 0, 0, xerrors.Errorf("get cpu total: %w", err)
7295
}
7396

7497
used, err = s.cGroupv2CPUUsed()
7598
if err != nil {
76-
return 0, 0, xerrors.Errorf("get cgroup v2 cpu used: %w", err)
99+
return 0, 0, xerrors.Errorf("get cpu used: %w", err)
77100
}
78101

79102
return used, total, nil
80103
}
81104

82105
func (s *Statter) cGroupv2CPUUsed() (used time.Duration, err error) {
83-
var data []byte
84-
data, err = afero.ReadFile(s.fs, cgroupV2CPUStat)
106+
iused, err := readInt64Prefix(s.fs, cgroupV2CPUStat, "usage_usec")
85107
if err != nil {
86-
return 0, xerrors.Errorf("read %s: %w", cgroupV2CPUStat, err)
87-
}
88-
89-
bs := bufio.NewScanner(bytes.NewReader(data))
90-
for bs.Scan() {
91-
line := bs.Bytes()
92-
if !bytes.HasPrefix(line, []byte("usage_usec ")) {
93-
continue
94-
}
95-
96-
parts := bytes.Split(line, []byte(" "))
97-
if len(parts) != 2 {
98-
return 0, xerrors.Errorf("unexpected line in %s: %s", cgroupV2CPUStat, line)
99-
}
100-
101-
iused, err := strconv.Atoi(string(parts[1]))
102-
if err != nil {
103-
return 0, xerrors.Errorf("parse %s: %w", err, cgroupV2CPUStat)
104-
}
105-
106-
return time.Duration(iused) * time.Microsecond, nil
108+
return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err)
107109
}
108-
109-
return 0, xerrors.Errorf("did not find expected usage_usec in %s", cgroupV2CPUStat)
110+
return time.Duration(iused) * time.Microsecond, nil
110111
}
111112

112113
func (s *Statter) cGroupv2CPUTotal() (total time.Duration, err error) {
113-
var data []byte
114114
var quotaUs int64
115-
data, err = afero.ReadFile(s.fs, cgroupV2CPUMax)
115+
quotaUs, err = readInt64SepIdx(s.fs, cgroupV2CPUMax, " ", 0)
116116
if err != nil {
117-
return 0, xerrors.Errorf("read %s: %w", cgroupV2CPUMax, err)
118-
}
119-
120-
lines := bytes.Split(data, []byte("\n"))
121-
if len(lines) < 1 {
122-
return 0, xerrors.Errorf("unexpected empty %s", cgroupV2CPUMax)
123-
}
124-
125-
parts := bytes.Split(lines[0], []byte(" "))
126-
if len(parts) != 2 {
127-
return 0, xerrors.Errorf("unexpected line in %s: %s", cgroupV2CPUMax, lines[0])
128-
}
129-
130-
if bytes.Equal(parts[0], []byte("max")) {
117+
// Fall back to number of cores
131118
quotaUs = int64(s.nproc) * time.Second.Microseconds()
132-
} else {
133-
quotaUs, err = strconv.ParseInt(string(parts[0]), 10, 64)
134-
if err != nil {
135-
return 0, xerrors.Errorf("parse %s: %w", cgroupV2CPUMax, err)
136-
}
137119
}
138120

139121
return time.Duration(quotaUs) * time.Microsecond, nil
@@ -142,66 +124,49 @@ func (s *Statter) cGroupv2CPUTotal() (total time.Duration, err error) {
142124
func (s *Statter) cGroupV1CPU() (used, total time.Duration, err error) {
143125
total, err = s.cGroupV1CPUTotal()
144126
if err != nil {
145-
return 0, 0, xerrors.Errorf("get cgroup v1 CPU total: %w", err)
127+
return 0, 0, xerrors.Errorf("get cpu total: %w", err)
146128
}
147129

148130
used, err = s.cgroupV1CPUUsed()
149131
if err != nil {
150-
return 0, 0, xerrors.Errorf("get cgruop v1 cpu used: %w", err)
132+
return 0, 0, xerrors.Errorf("get cpu used: %w", err)
151133
}
152134

153135
return used, total, nil
154136
}
155137

156138
func (s *Statter) cGroupV1CPUTotal() (time.Duration, error) {
157-
var data []byte
158-
var err error
159-
var quotaUs int64
160-
161-
data, err = afero.ReadFile(s.fs, cgroupV1CFSQuotaUs)
139+
quotaUs, err := readInt64(s.fs, cgroupV1CFSQuotaUs)
162140
if err != nil {
163-
return 0, xerrors.Errorf("read %s: %w", cgroupV1CFSQuotaUs, err)
164-
}
165-
166-
quotaUs, err = strconv.ParseInt(string(bytes.TrimSpace(data)), 10, 64)
167-
if err != nil {
168-
return 0, xerrors.Errorf("parse %s: %w", cgroupV1CFSQuotaUs, err)
141+
return 0, xerrors.Errorf("read cpu quota: %w", err)
169142
}
170143

171144
if quotaUs < 0 {
145+
// Fall back to the number of cores
172146
quotaUs = int64(s.nproc) * time.Second.Microseconds()
173147
}
174148

175149
return time.Duration(quotaUs) * time.Microsecond, nil
176150
}
177151

178152
func (s *Statter) cgroupV1CPUUsed() (time.Duration, error) {
179-
var data []byte
180-
var err error
181-
var usageUs int64
182-
183-
data, err = afero.ReadFile(s.fs, cgroupV1CPUAcctUsage)
153+
usageNs, err := readInt64(s.fs, cgroupV1CPUAcctUsage)
184154
if err != nil {
185155
// try alternate path
186-
data, err = afero.ReadFile(s.fs, cgroupV1CPUAcctUsageAlt)
156+
usageNs, err = readInt64(s.fs, cgroupV1CPUAcctUsageAlt)
187157
if err != nil {
188-
return 0, xerrors.Errorf("read %s or %s: %w", cgroupV1CPUAcctUsage, cgroupV1CPUAcctUsageAlt, err)
158+
return 0, xerrors.Errorf("read cpu used: %w", err)
189159
}
190160
}
191161

192-
usageUs, err = strconv.ParseInt(string(bytes.TrimSpace(data)), 10, 64)
193-
if err != nil {
194-
return 0, xerrors.Errorf("parse %s: %w", cgroupV1CPUAcctUsage, err)
195-
}
196-
197-
return time.Duration(usageUs) * time.Microsecond, nil
162+
return time.Duration(usageNs), nil
198163
}
199164

200165
// ContainerMemory returns the memory usage of the container cgroup.
201166
// If the system is not containerized, this always returns nil.
202167
func (s *Statter) ContainerMemory() (*Result, error) {
203168
if ok, err := IsContainerized(s.fs); err != nil || !ok {
204-
return nil, nil
169+
return nil, nil //nolint:nilnil
205170
}
206171

207172
if s.isCGroupV2() {
@@ -212,66 +177,115 @@ func (s *Statter) ContainerMemory() (*Result, error) {
212177
return s.cGroupv1Memory()
213178
}
214179

215-
func (*Statter) cGroupv2Memory() (*Result, error) {
216-
// TODO implement
217-
return nil, nil
180+
func (s *Statter) cGroupv2Memory() (*Result, error) {
181+
maxUsageBytes, err := readInt64(s.fs, cgroupV2MemoryMaxBytes)
182+
if err != nil {
183+
return nil, xerrors.Errorf("read memory total: %w", err)
184+
}
185+
186+
currUsageBytes, err := readInt64(s.fs, cgroupV2MemoryUsageBytes)
187+
if err != nil {
188+
return nil, xerrors.Errorf("read memory usage: %w", err)
189+
}
190+
191+
inactiveFileBytes, err := readInt64Prefix(s.fs, cgroupV2MemoryStat, "inactive_file")
192+
if err != nil {
193+
return nil, xerrors.Errorf("read memory stats: %w", err)
194+
}
195+
196+
return &Result{
197+
Total: ptr.To(float64(maxUsageBytes) / 1024 / 1024 / 1024),
198+
Used: float64(currUsageBytes-inactiveFileBytes) / 1024 / 1024 / 1024,
199+
Unit: "GB",
200+
}, nil
218201
}
219202

220203
func (s *Statter) cGroupv1Memory() (*Result, error) {
221-
var data []byte
222-
var err error
223-
var usageBytes int64
224-
var maxUsageBytes int64
225-
var totalInactiveFileBytes int64
226-
227-
// Read max memory usage
228-
data, err = afero.ReadFile(s.fs, cgroupV1MemoryMaxUsageBytes)
204+
maxUsageBytes, err := readInt64(s.fs, cgroupV1MemoryMaxUsageBytes)
205+
if err != nil {
206+
return nil, xerrors.Errorf("read memory total: %w", err)
207+
}
208+
209+
// need a space after total_rss so we don't hit something else
210+
usageBytes, err := readInt64(s.fs, cgroupV1MemoryUsageBytes)
211+
if err != nil {
212+
return nil, xerrors.Errorf("read memory usage: %w", err)
213+
}
214+
215+
totalInactiveFileBytes, err := readInt64Prefix(s.fs, cgroupV1MemoryStat, "total_inactive_file")
216+
if err != nil {
217+
return nil, xerrors.Errorf("read memory stats: %w", err)
218+
}
219+
220+
// Total memory used is usage - total_inactive_file
221+
return &Result{
222+
Total: ptr.To(float64(maxUsageBytes) / 1024 / 1024 / 1024),
223+
Used: float64(usageBytes-totalInactiveFileBytes) / 1024 / 1024 / 1024,
224+
Unit: "GB",
225+
}, nil
226+
}
227+
228+
// read an int64 value from path
229+
func readInt64(fs afero.Fs, path string) (int64, error) {
230+
data, err := afero.ReadFile(fs, path)
229231
if err != nil {
230-
return nil, xerrors.Errorf("read %s: %w", cgroupV1MemoryMaxUsageBytes, err)
232+
return 0, xerrors.Errorf("read %s: %w", path, err)
231233
}
232234

233-
maxUsageBytes, err = strconv.ParseInt(string(bytes.TrimSpace(data)), 10, 64)
235+
val, err := strconv.ParseInt(string(bytes.TrimSpace(data)), 10, 64)
234236
if err != nil {
235-
return nil, xerrors.Errorf("parse %s: %w", cgroupV1MemoryMaxUsageBytes, err)
237+
return 0, xerrors.Errorf("parse %s: %w", path, err)
236238
}
237239

238-
// Read current memory usage
239-
data, err = afero.ReadFile(s.fs, cgroupV1MemoryUsageBytes)
240+
return val, nil
241+
}
242+
243+
// read an int64 value from path at field idx separated by sep
244+
func readInt64SepIdx(fs afero.Fs, path, sep string, idx int) (int64, error) {
245+
data, err := afero.ReadFile(fs, path)
240246
if err != nil {
241-
return nil, xerrors.Errorf("read %s: %w", cgroupV1MemoryUsageBytes, err)
247+
return 0, xerrors.Errorf("read %s: %w", path, err)
248+
}
249+
250+
parts := strings.Split(string(data), sep)
251+
if len(parts) < idx {
252+
return 0, xerrors.Errorf("expected line %q to have at least %d parts", string(data), idx+1)
242253
}
243254

244-
usageBytes, err = strconv.ParseInt(string(bytes.TrimSpace(data)), 10, 64)
255+
val, err := strconv.ParseInt(parts[idx], 10, 64)
245256
if err != nil {
246-
return nil, xerrors.Errorf("parse %s: %w", cgroupV1MemoryUsageBytes, err)
257+
return 0, xerrors.Errorf("parse %s: %w", path, err)
247258
}
248259

249-
// Get total_inactive_file from memory.stat
250-
data, err = afero.ReadFile(s.fs, cgroupV1MemoryStat)
260+
return val, nil
261+
}
262+
263+
// read the first int64 value from path prefixed with prefix
264+
func readInt64Prefix(fs afero.Fs, path, prefix string) (int64, error) {
265+
data, err := afero.ReadFile(fs, path)
251266
if err != nil {
252-
return nil, xerrors.Errorf("read %s: %w", cgroupV1MemoryStat, err)
267+
return 0, xerrors.Errorf("read %s: %w", path, err)
253268
}
269+
254270
scn := bufio.NewScanner(bytes.NewReader(data))
255271
for scn.Scan() {
256-
line := scn.Bytes()
257-
if !bytes.HasPrefix(line, []byte("total_inactive_file")) {
272+
line := scn.Text()
273+
if !strings.HasPrefix(line, prefix) {
258274
continue
259275
}
260276

261-
parts := bytes.Split(line, []byte(" "))
277+
parts := strings.Fields(line)
262278
if len(parts) != 2 {
263-
return nil, xerrors.Errorf("unexpected value in %s: %s", cgroupV1MemoryUsageBytes, string(line))
279+
return 0, xerrors.Errorf("parse %s: expected two fields but got %s", path, line)
264280
}
265-
totalInactiveFileBytes, err = strconv.ParseInt(string(bytes.TrimSpace(parts[1])), 10, 64)
281+
282+
val, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64)
266283
if err != nil {
267-
return nil, xerrors.Errorf("parse %s: %w", cgroupV1MemoryUsageBytes, err)
284+
return 0, xerrors.Errorf("parse %s: %w", path, err)
268285
}
286+
287+
return val, nil
269288
}
270289

271-
// Total memory used is usage - total_inactive_file
272-
return &Result{
273-
Total: ptr.To(float64(maxUsageBytes) / 1024 / 1024 / 1024),
274-
Used: float64(usageBytes-totalInactiveFileBytes) / 1024 / 1024 / 1024,
275-
Unit: "GB",
276-
}, nil
290+
return 0, xerrors.Errorf("parse %s: did not find line with prefix %s", path, prefix)
277291
}

0 commit comments

Comments
 (0)