Skip to content

Commit e158483

Browse files
kailun-qinAkihiroSudaZheaoli
committed
libct/cg: add CFS bandwidth burst for CPU
Burstable CFS controller is introduced in Linux 5.14. This helps with parallel workloads that might be bursty. They can get throttled even when their average utilization is under quota. And they may be latency sensitive at the same time so that throttling them is undesired. This feature borrows time now against the future underrun, at the cost of increased interference against the other system users, by introducing cfs_burst_us into CFS bandwidth control to enact the cap on unused bandwidth accumulation, which will then used additionally for burst. The patch adds the support/control for CFS bandwidth burst. runtime-spec: opencontainers/runtime-spec#1120 Co-authored-by: Akihiro Suda <[email protected]> Co-authored-by: Nadeshiko Manju <[email protected]> Signed-off-by: Kailun Qin <[email protected]>
1 parent 319b2ba commit e158483

File tree

12 files changed

+108
-4
lines changed

12 files changed

+108
-4
lines changed

‎contrib/completions/bash/runc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -735,6 +735,7 @@ _runc_update() {
735735
--blkio-weight
736736
--cpu-period
737737
--cpu-quota
738+
--cpu-burst
738739
--cpu-rt-period
739740
--cpu-rt-runtime
740741
--cpu-share

‎docs/spec-conformance.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ Spec version | Feature | PR
1010
v1.0.0 | `SCMP_ARCH_PARISC` | Unplanned, due to lack of users
1111
v1.0.0 | `SCMP_ARCH_PARISC64` | Unplanned, due to lack of users
1212
v1.0.2 | `.linux.personality` | [#3126](https://github.com/opencontainers/runc/pull/3126)
13-
v1.1.0 | `.linux.resources.cpu.burst` | [#3749](https://github.com/opencontainers/runc/pull/3749)
1413
v1.1.0 | `SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` | [#3862](https://github.com/opencontainers/runc/pull/3862)
1514
v1.1.0 | time namespaces | [#3876](https://github.com/opencontainers/runc/pull/3876)
1615
v1.1.0 | rsvd hugetlb cgroup | TODO ([#3859](https://github.com/opencontainers/runc/issues/3859))

‎libcontainer/cgroups/fs/cpu.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,28 @@ func (s *CpuGroup) Set(path string, r *configs.Resources) error {
8484
period = ""
8585
}
8686
}
87+
88+
var burst string
89+
if r.CpuBurst != nil {
90+
burst = strconv.FormatUint(*r.CpuBurst, 10)
91+
if err := cgroups.WriteFile(path, "cpu.cfs_burst_us", burst); err != nil {
92+
// this is a special trick for burst feature, the current systemd and low version of kernel will not support it.
93+
// So, an `no such file or directory` error would be raised, and we can ignore it .
94+
if !errors.Is(err, unix.ENOENT) {
95+
// Sometimes when the burst to be set is larger
96+
// than the current one, it is rejected by the kernel
97+
// (EINVAL) as old_quota/new_burst exceeds the parent
98+
// cgroup quota limit. If this happens and the quota is
99+
// going to be set, ignore the error for now and retry
100+
// after setting the quota.
101+
if !errors.Is(err, unix.EINVAL) || r.CpuQuota == 0 {
102+
return err
103+
}
104+
}
105+
} else {
106+
burst = ""
107+
}
108+
}
87109
if r.CpuQuota != 0 {
88110
if err := cgroups.WriteFile(path, "cpu.cfs_quota_us", strconv.FormatInt(r.CpuQuota, 10)); err != nil {
89111
return err
@@ -93,6 +115,13 @@ func (s *CpuGroup) Set(path string, r *configs.Resources) error {
93115
return err
94116
}
95117
}
118+
if burst != "" {
119+
if err := cgroups.WriteFile(path, "cpu.cfs_burst_us", burst); err != nil {
120+
if !errors.Is(err, unix.ENOENT) {
121+
return err
122+
}
123+
}
124+
}
96125
}
97126

98127
if r.CPUIdle != nil {

‎libcontainer/cgroups/fs/cpu_test.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,23 +45,27 @@ func TestCpuSetBandWidth(t *testing.T) {
4545
const (
4646
quotaBefore = 8000
4747
quotaAfter = 5000
48+
burstBefore = 2000
4849
periodBefore = 10000
4950
periodAfter = 7000
5051
rtRuntimeBefore = 8000
5152
rtRuntimeAfter = 5000
5253
rtPeriodBefore = 10000
5354
rtPeriodAfter = 7000
5455
)
56+
burstAfter := uint64(1000)
5557

5658
writeFileContents(t, path, map[string]string{
5759
"cpu.cfs_quota_us": strconv.Itoa(quotaBefore),
60+
"cpu.cfs_burst_us": strconv.Itoa(burstBefore),
5861
"cpu.cfs_period_us": strconv.Itoa(periodBefore),
5962
"cpu.rt_runtime_us": strconv.Itoa(rtRuntimeBefore),
6063
"cpu.rt_period_us": strconv.Itoa(rtPeriodBefore),
6164
})
6265

6366
r := &configs.Resources{
6467
CpuQuota: quotaAfter,
68+
CpuBurst: &burstAfter,
6569
CpuPeriod: periodAfter,
6670
CpuRtRuntime: rtRuntimeAfter,
6771
CpuRtPeriod: rtPeriodAfter,
@@ -79,6 +83,14 @@ func TestCpuSetBandWidth(t *testing.T) {
7983
t.Fatal("Got the wrong value, set cpu.cfs_quota_us failed.")
8084
}
8185

86+
burst, err := fscommon.GetCgroupParamUint(path, "cpu.cfs_burst_us")
87+
if err != nil {
88+
t.Fatal(err)
89+
}
90+
if burst != burstAfter {
91+
t.Fatal("Got the wrong value, set cpu.cfs_burst_us failed.")
92+
}
93+
8294
period, err := fscommon.GetCgroupParamUint(path, "cpu.cfs_period_us")
8395
if err != nil {
8496
t.Fatal(err)

‎libcontainer/cgroups/fs/fs.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ func (m *Manager) Set(r *configs.Resources) error {
191191
if path == "" {
192192
// We never created a path for this cgroup, so we cannot set
193193
// limits for it (though we have already tried at this point).
194-
return fmt.Errorf("cannot set %s limit: container could not join or create cgroup", sys.Name())
194+
return fmt.Errorf("cannot set %s limit: container could not join or create cgroup, and the error is %w", sys.Name(), err)
195195
}
196196
return err
197197
}

‎libcontainer/cgroups/fs2/cpu.go

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,19 @@ package fs2
22

33
import (
44
"bufio"
5+
"errors"
56
"os"
67
"strconv"
78

9+
"golang.org/x/sys/unix"
10+
811
"github.com/opencontainers/runc/libcontainer/cgroups"
912
"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
1013
"github.com/opencontainers/runc/libcontainer/configs"
1114
)
1215

1316
func isCpuSet(r *configs.Resources) bool {
14-
return r.CpuWeight != 0 || r.CpuQuota != 0 || r.CpuPeriod != 0 || r.CPUIdle != nil
17+
return r.CpuWeight != 0 || r.CpuQuota != 0 || r.CpuPeriod != 0 || r.CPUIdle != nil || r.CpuBurst != nil
1518
}
1619

1720
func setCpu(dirPath string, r *configs.Resources) error {
@@ -32,6 +35,23 @@ func setCpu(dirPath string, r *configs.Resources) error {
3235
}
3336
}
3437

38+
var burst string
39+
if r.CpuBurst != nil {
40+
burst = strconv.FormatUint(*r.CpuBurst, 10)
41+
if err := cgroups.WriteFile(dirPath, "cpu.max.burst", burst); err != nil {
42+
// Sometimes when the burst to be set is larger
43+
// than the current one, it is rejected by the kernel
44+
// (EINVAL) as old_quota/new_burst exceeds the parent
45+
// cgroup quota limit. If this happens and the quota is
46+
// going to be set, ignore the error for now and retry
47+
// after setting the quota.
48+
if !errors.Is(err, unix.EINVAL) || r.CpuQuota == 0 {
49+
return err
50+
}
51+
} else {
52+
burst = ""
53+
}
54+
}
3555
if r.CpuQuota != 0 || r.CpuPeriod != 0 {
3656
str := "max"
3757
if r.CpuQuota > 0 {
@@ -47,6 +67,11 @@ func setCpu(dirPath string, r *configs.Resources) error {
4767
if err := cgroups.WriteFile(dirPath, "cpu.max", str); err != nil {
4868
return err
4969
}
70+
if burst != "" {
71+
if err := cgroups.WriteFile(dirPath, "cpu.max.burst", burst); err != nil {
72+
return err
73+
}
74+
}
5075
}
5176

5277
return nil

‎libcontainer/configs/cgroup_linux.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,9 @@ type Resources struct {
6969
// CPU hardcap limit (in usecs). Allowed cpu time in a given period.
7070
CpuQuota int64 `json:"cpu_quota"`
7171

72+
// CPU hardcap burst limit (in usecs). Allowed accumulated cpu time additionally for burst in a given period.
73+
CpuBurst *uint64 `json:"cpu_burst"` //nolint:revive
74+
7275
// CPU period to be used for hardcapping (in usecs). 0 to use system default.
7376
CpuPeriod uint64 `json:"cpu_period"`
7477

‎libcontainer/specconv/spec_linux.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,6 +755,9 @@ func CreateCgroupConfig(opts *CreateOpts, defaultDevs []*devices.Device) (*confi
755755
if r.CPU.Quota != nil {
756756
c.Resources.CpuQuota = *r.CPU.Quota
757757
}
758+
if r.CPU.Burst != nil {
759+
c.Resources.CpuBurst = r.CPU.Burst
760+
}
758761
if r.CPU.Period != nil {
759762
c.Resources.CpuPeriod = *r.CPU.Period
760763
}

‎man/runc-update.8.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ In case **-r** is used, the JSON format is like this:
2828
"cpu": {
2929
"shares": 0,
3030
"quota": 0,
31+
"burst": 0,
3132
"period": 0,
3233
"realtimeRuntime": 0,
3334
"realtimePeriod": 0,
@@ -53,6 +54,9 @@ stdin. If this option is used, all other options are ignored.
5354
**--cpu-quota** _num_
5455
: Set CPU usage limit within a given period (in microseconds).
5556

57+
**--cpu-burst** _num_
58+
: Set CPU burst limit within a given period (in microseconds).
59+
5660
**--cpu-rt-period** _num_
5761
: Set CPU realtime period to be used for hardcapping (in microseconds).
5862

‎tests/integration/helpers.bash

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,11 +260,16 @@ function get_cgroup_value() {
260260
# Helper to check a if value in a cgroup file matches the expected one.
261261
function check_cgroup_value() {
262262
local current
263+
local cgroup
264+
cgroup="$(get_cgroup_path "$1")"
265+
if [ ! -f "$cgroup/$1" ]; then
266+
skip "$cgroup/$1 does not exist"
267+
fi
263268
current="$(get_cgroup_value "$1")"
264269
local expected=$2
265270

266271
echo "current $current !? $expected"
267-
[ "$current" = "$expected" ]
272+
[ "$current" = "$expected" ] || [ "$current" = "$((expected / 1000))" ]
268273
}
269274

270275
# Helper to check a value in systemd.
@@ -310,6 +315,15 @@ function check_cpu_quota() {
310315
check_systemd_value "CPUQuotaPeriodUSec" $sd_period $sd_infinity
311316
}
312317

318+
function check_cpu_burst() {
319+
local burst=$1
320+
if [ -v CGROUP_V2 ]; then
321+
check_cgroup_value "cpu.max.burst" "$burst"
322+
else
323+
check_cgroup_value "cpu.cfs_burst_us" "$burst"
324+
fi
325+
}
326+
313327
# Works for cgroup v1 and v2, accepts v1 shares as an argument.
314328
function check_cpu_shares() {
315329
local shares=$1

0 commit comments

Comments
 (0)