Skip to content

Commit c70f88e

Browse files
kolyshkinclaude
andcommitted
libct: use CPUSetDynamic for affinity and mempolicy masks
ToCPUSet was limited to unix.CPUSet (a fixed-size mask capped at 1024 CPUs). Switch CPUAffinity, LinuxMemoryPolicy.Nodes, ToCPUSet, and SetMempolicy to unix.CPUSetDynamic, which can represent arbitrary CPU and NUMA node IDs. ToCPUSet now parses the input in a first pass to find the maximum value, then allocates a mask large enough to hold it (out-of-range Set calls on a dynamic mask are silently ignored). An arbitrary sanity cap is kept to avoid huge allocations on bogus input. This is backward compatible with reading older state.json: both the old *unix.CPUSet (a fixed array) and the new CPUSetDynamic (a slice) marshal to and from the same JSON array-of-numbers representation. Update TestToCPUSet accordingly, adding coverage for values beyond the old non-dynamic limit. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>
1 parent 05dc7f6 commit c70f88e

6 files changed

Lines changed: 107 additions & 58 deletions

File tree

CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88

9+
### libcontainer API ###
10+
- `configs.ToCPUSet` now returns a `unix.CPUSetDynamic` instead of a
11+
`*unix.CPUSet`, and the `Initial`/`Final` fields of `configs.CPUAffinity` and
12+
the `Nodes` field of `configs.LinuxMemoryPolicy` have changed type
13+
accordingly. This lifts the previous 1024 CPUs/nodes limit. (#5343)
14+
915
### Fixed ###
1016
- The poststart hooks are now executed after starting the user-specified
1117
process, fixing a runtime-spec conformance issue. (#4347, #5186)
1218

1319
### Changed ###
1420
- Updated builds to libseccomp v2.6.1. (#5376)
21+
- The `cpuAffinity` and NUMA `memoryPolicy` settings are no longer limited
22+
to 1024 CPUs/nodes, as runc now uses a dynamically-sized CPU mask. (#5343)
1523

1624
## [1.5.0] - 2026-06-19
1725

internal/linux/linux.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,10 @@ func Sendmsg(fd int, p, oob []byte, to unix.Sockaddr, flags int) error {
8181
return os.NewSyscallError("sendmsg", err)
8282
}
8383

84-
// SetMempolicy wraps set_mempolicy.
85-
func SetMempolicy(mode int, mask *unix.CPUSet) error {
84+
// SetMempolicy wraps [unix.SetMempolicyDynamic].
85+
func SetMempolicy(mode int, mask unix.CPUSetDynamic) error {
8686
err := retryOnEINTR(func() error {
87-
return unix.SetMemPolicy(mode, mask)
87+
return unix.SetMemPolicyDynamic(mode, mask)
8888
})
8989
return os.NewSyscallError("set_mempolicy", err)
9090
}

libcontainer/configs/config.go

Lines changed: 51 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"strconv"
1212
"strings"
1313
"time"
14-
"unsafe"
1514

1615
"github.com/sirupsen/logrus"
1716
"golang.org/x/sys/unix"
@@ -305,26 +304,53 @@ func ToSchedAttr(scheduler *Scheduler) (*unix.SchedAttr, error) {
305304
type IOPriority = specs.LinuxIOPriority
306305

307306
type CPUAffinity struct {
308-
Initial, Final *unix.CPUSet
307+
Initial, Final unix.CPUSetDynamic
309308
}
310309

311-
// ToCPUSet parses a string in list format into a unix.CPUSet, e.g. "0-3,5,7-9".
312-
func ToCPUSet(str string) (*unix.CPUSet, error) {
310+
// maxCPUSetCPU is the highest CPU/NUMA node ID that [ToCPUSet] accepts. It is
311+
// an arbitrary sanity limit, used to avoid allocating an unreasonably large
312+
// mask for a bogus input.
313+
const maxCPUSetCPU = 64 * 1024
314+
315+
// ToCPUSet parses a string in list format into a [unix.CPUSetDynamic],
316+
// e.g. "0-3,5,7-9".
317+
func ToCPUSet(str string) (unix.CPUSetDynamic, error) {
313318
if str == "" {
314319
return nil, nil
315320
}
316-
s := new(unix.CPUSet)
317321

318-
// Since (*CPUset).Set silently ignores too high CPU values,
319-
// find out what the maximum is, and return an error.
320-
maxCPU := uint64(unsafe.Sizeof(*s) * 8)
322+
maxCPU, ranges, err := cpuStrToRanges(str)
323+
if err != nil {
324+
return nil, err
325+
}
326+
if len(ranges) == 0 {
327+
return nil, fmt.Errorf("no members found in set %q", str)
328+
}
329+
330+
s := unix.NewCPUSet(maxCPU + 1)
331+
for _, cr := range ranges {
332+
for i := cr.start; i <= cr.end; i++ {
333+
s.Set(i)
334+
}
335+
}
336+
337+
return s, nil
338+
}
339+
340+
type cpuRange struct {
341+
start, end int
342+
}
343+
344+
// cpuStrToRanges parses string (e.g. "0-3,5,7-9") into a slice of cpuRange
345+
// values. It also returns the maximum value and an error, if any.
346+
func cpuStrToRanges(str string) (maxCPU int, ranges []cpuRange, _ error) {
321347
toInt := func(v string) (int, error) {
322348
ret, err := strconv.ParseUint(v, 10, 32)
323349
if err != nil {
324350
return 0, err
325351
}
326-
if ret >= maxCPU {
327-
return 0, fmt.Errorf("values larger than %d are not supported", maxCPU-1)
352+
if ret > maxCPUSetCPU {
353+
return 0, fmt.Errorf("values larger than %d are not supported", maxCPUSetCPU)
328354
}
329355
return int(ret), nil
330356
}
@@ -336,34 +362,27 @@ func ToCPUSet(str string) (*unix.CPUSet, error) {
336362
if r == "" {
337363
continue
338364
}
339-
if r0, r1, found := strings.Cut(r, "-"); found {
340-
start, err := toInt(r0)
341-
if err != nil {
342-
return nil, err
343-
}
344-
end, err := toInt(r1)
345-
if err != nil {
346-
return nil, err
347-
}
348-
if start > end {
349-
return nil, errors.New("invalid range: " + r)
365+
start, end, found := strings.Cut(r, "-")
366+
cr := cpuRange{}
367+
var err error
368+
if cr.start, err = toInt(start); err != nil {
369+
return 0, nil, err
370+
}
371+
if found {
372+
if cr.end, err = toInt(end); err != nil {
373+
return 0, nil, err
350374
}
351-
for i := start; i <= end; i++ {
352-
s.Set(i)
375+
if cr.start > cr.end {
376+
return 0, nil, errors.New("invalid range: " + r)
353377
}
354378
} else {
355-
val, err := toInt(r)
356-
if err != nil {
357-
return nil, err
358-
}
359-
s.Set(val)
379+
cr.end = cr.start
360380
}
361-
}
362-
if s.Count() == 0 {
363-
return nil, fmt.Errorf("no members found in set %q", str)
381+
maxCPU = max(maxCPU, cr.end)
382+
ranges = append(ranges, cr)
364383
}
365384

366-
return s, nil
385+
return maxCPU, ranges, nil
367386
}
368387

369388
// ConvertCPUAffinity converts [specs.CPUAffinity] to [CPUAffinity].

libcontainer/configs/memorypolicy.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ type LinuxMemoryPolicy struct {
1010
// Flags contains mode flags.
1111
Flags int `json:"flags,omitempty"`
1212
// Nodes contains NUMA nodes to which the mode applies.
13-
Nodes *unix.CPUSet `json:"nodes,omitempty"`
13+
Nodes unix.CPUSetDynamic `json:"nodes,omitempty"`
1414
}

libcontainer/configs/tocpuset_test.go

Lines changed: 42 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,67 @@
11
package configs
22

33
import (
4+
"slices"
45
"testing"
56

67
"golang.org/x/sys/unix"
78
)
89

910
func TestToCPUSet(t *testing.T) {
10-
set := func(cpus ...int) *unix.CPUSet {
11-
r := &unix.CPUSet{}
11+
set := func(cpus ...int) unix.CPUSetDynamic {
12+
maxCPU := 0
13+
for _, cpu := range cpus {
14+
maxCPU = max(maxCPU, cpu)
15+
}
16+
r := unix.NewCPUSet(maxCPU + 1)
1217
for _, cpu := range cpus {
1318
r.Set(cpu)
1419
}
1520
return r
1621
}
1722

23+
// trim removes trailing all-zero masks so that sets representing the
24+
// same CPUs compare equal regardless of how they were allocated.
25+
trim := func(s unix.CPUSetDynamic) unix.CPUSetDynamic {
26+
for len(s) > 0 && s[len(s)-1] == 0 {
27+
s = s[:len(s)-1]
28+
}
29+
return s
30+
}
31+
1832
testCases := []struct {
1933
in string
20-
out *unix.CPUSet
34+
out unix.CPUSetDynamic
2135
isErr bool
2236
}{
2337
{in: ""}, // Empty means unset.
2438

2539
// Valid cases.
26-
{in: "0", out: &unix.CPUSet{1}},
27-
{in: "1", out: &unix.CPUSet{2}},
28-
{in: "0-1", out: &unix.CPUSet{3}},
29-
{in: "0,1", out: &unix.CPUSet{3}},
30-
{in: ",0,1,", out: &unix.CPUSet{3}},
31-
{in: "0-3", out: &unix.CPUSet{0x0f}},
32-
{in: "0,1,2-3", out: &unix.CPUSet{0x0f}},
33-
{in: "4-7", out: &unix.CPUSet{0xf0}},
34-
{in: "0-7", out: &unix.CPUSet{0xff}},
35-
{in: "0-15", out: &unix.CPUSet{0xffff}},
36-
{in: "16", out: &unix.CPUSet{0x10000}},
40+
{in: "0", out: set(0)},
41+
{in: "1", out: set(1)},
42+
{in: "0-1", out: set(0, 1)},
43+
{in: "0,1", out: set(0, 1)},
44+
{in: ",0,1,", out: set(0, 1)},
45+
{in: "0-3", out: set(0, 1, 2, 3)},
46+
{in: "0,1,2-3", out: set(0, 1, 2, 3)},
47+
{in: "4-7", out: set(4, 5, 6, 7)},
48+
{in: "0-7", out: set(0, 1, 2, 3, 4, 5, 6, 7)},
49+
{in: "0-15", out: set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)},
50+
{in: "16", out: set(16)},
3751
// Extra whitespace in between ranges are OK.
38-
{in: "1, 2, 1-2", out: &unix.CPUSet{6}},
39-
{in: " , 1 , 3 , 5-7, ", out: &unix.CPUSet{0xea}},
40-
// Somewhat large values. The underlying type in unix.CPUSet
41-
// can either be uint32 or uint64, so we have to use a helper.
52+
{in: "1, 2, 1-2", out: set(1, 2)},
53+
{in: " , 1 , 3 , 5-7, ", out: set(1, 3, 5, 6, 7)},
54+
// Somewhat large values.
4255
{in: "0-3,32-33", out: set(0, 1, 2, 3, 32, 33)},
4356
{in: "127-129, 1", out: set(1, 127, 128, 129)},
4457
{in: "1023", out: set(1023)},
58+
// Values larger than what a (non-dynamic) unix.CPUSet can hold.
59+
{in: "1024", out: set(1024)},
60+
{in: "8191", out: set(8191)},
61+
{in: "4096-4098", out: set(4096, 4097, 4098)},
62+
{in: "0,65536", out: set(0, 65536)},
63+
// Maximum allowed value.
64+
{in: "65536", out: set(65536)},
4565

4666
// Error cases.
4767
{in: "-", isErr: true},
@@ -53,7 +73,9 @@ func TestToCPUSet(t *testing.T) {
5373
{in: "54-53", isErr: true},
5474
// Extra spaces inside a range is not OK.
5575
{in: "1 - 2", isErr: true},
56-
{in: "1024", isErr: true}, // Too big for unix.CPUSet.
76+
// Larger than the maximum supported value.
77+
{in: "65537", isErr: true},
78+
{in: "0-65537", isErr: true},
5779
}
5880

5981
for _, tc := range testCases {
@@ -80,7 +102,7 @@ func TestToCPUSet(t *testing.T) {
80102
if out == nil {
81103
t.Fatalf("want %v, got nil", tc.out)
82104
}
83-
if *out != *tc.out {
105+
if !slices.Equal(trim(out), trim(tc.out)) {
84106
t.Errorf("case %q: want %v, got %v", tc.in, tc.out, out)
85107
}
86108
})

libcontainer/process_linux.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ func (p *setnsProcess) startWithCPUAffinity() error {
224224
go func() {
225225
runtime.LockOSThread()
226226
// Command inherits the CPU affinity.
227-
if err := unix.SchedSetaffinity(unix.Gettid(), aff.Initial); err != nil {
227+
if err := linux.SchedSetaffinity(unix.Gettid(), aff.Initial); err != nil {
228228
errCh <- fmt.Errorf("error setting initial CPU affinity: %w", err)
229229
return
230230
}
@@ -250,7 +250,7 @@ func (p *setnsProcess) setFinalCPUAffinity() error {
250250
if aff.Final == nil {
251251
return nil
252252
}
253-
if err := unix.SchedSetaffinity(p.pid(), aff.Final); err != nil {
253+
if err := linux.SchedSetaffinity(p.pid(), aff.Final); err != nil {
254254
return fmt.Errorf("error setting final CPU affinity: %w", err)
255255
}
256256
return nil

0 commit comments

Comments
 (0)