Skip to content

Commit 2d8829d

Browse files
newborn22newborn22
committed
cmd/compile/internal: add intrinsic for sync atomic and/or 32/64 on riscv
1 parent 1644917 commit 2d8829d

8 files changed

Lines changed: 141 additions & 14 deletions

File tree

‎src/cmd/compile/internal/riscv64/ssa.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -750,6 +750,15 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
750750
p.To.Reg = v.Args[0].Reg()
751751
p.RegTo2 = riscv.REG_ZERO
752752

753+
case ssa.OpRISCV64LoweredAtomicAnd32value, ssa.OpRISCV64LoweredAtomicAnd64value,
754+
ssa.OpRISCV64LoweredAtomicOr32value, ssa.OpRISCV64LoweredAtomicOr64value:
755+
p := s.Prog(v.Op.Asm())
756+
p.From.Type = obj.TYPE_REG
757+
p.From.Reg = v.Args[1].Reg()
758+
p.To.Type = obj.TYPE_MEM
759+
p.To.Reg = v.Args[0].Reg()
760+
p.RegTo2 = v.Reg0()
761+
753762
case ssa.OpRISCV64LoweredZero:
754763
ptr := v.Args[0].Reg()
755764
sc := v.AuxValAndOff()

‎src/cmd/compile/internal/ssa/_gen/RISCV64.rules‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,6 +495,11 @@
495495

496496
(AtomicAnd32 ...) => (LoweredAtomicAnd32 ...)
497497

498+
(AtomicAnd32value ...) => (LoweredAtomicAnd32value ...)
499+
(AtomicAnd64value ...) => (LoweredAtomicAnd64value ...)
500+
(AtomicOr32value ...) => (LoweredAtomicOr32value ...)
501+
(AtomicOr64value ...) => (LoweredAtomicOr64value ...)
502+
498503
(AtomicCompareAndSwap32 ptr old new mem) => (LoweredAtomicCas32 ptr (SignExt32to64 old) new mem)
499504
(AtomicCompareAndSwap64 ...) => (LoweredAtomicCas64 ...)
500505

‎src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,12 @@ func init() {
420420
{name: "LoweredAtomicAnd32", argLength: 3, reg: gpatomic, asm: "AMOANDW", faultOnNilArg0: true, hasSideEffects: true},
421421
{name: "LoweredAtomicOr32", argLength: 3, reg: gpatomic, asm: "AMOORW", faultOnNilArg0: true, hasSideEffects: true},
422422

423+
// Atomic 32/64 bit AND/OR that return the old value.
424+
{name: "LoweredAtomicAnd32value", argLength: 3, reg: gpxchg, resultNotInArgs: true, asm: "AMOANDW", faultOnNilArg0: true, hasSideEffects: true},
425+
{name: "LoweredAtomicAnd64value", argLength: 3, reg: gpxchg, resultNotInArgs: true, asm: "AMOANDD", faultOnNilArg0: true, hasSideEffects: true},
426+
{name: "LoweredAtomicOr32value", argLength: 3, reg: gpxchg, resultNotInArgs: true, asm: "AMOORW", faultOnNilArg0: true, hasSideEffects: true},
427+
{name: "LoweredAtomicOr64value", argLength: 3, reg: gpxchg, resultNotInArgs: true, asm: "AMOORD", faultOnNilArg0: true, hasSideEffects: true},
428+
423429
// Lowering pass-throughs
424430
{name: "LoweredNilCheck", argLength: 2, faultOnNilArg0: true, nilCheck: true, reg: regInfo{inputs: []regMask{gpspMask}}}, // arg0=ptr,arg1=mem, returns void. Faults if ptr is nil.
425431
{name: "LoweredGetClosurePtr", reg: regInfo{outputs: []regMask{regCtxt}}}, // scheduler ensures only at beginning of entry block

‎src/cmd/compile/internal/ssa/opGen.go‎

Lines changed: 72 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/cmd/compile/internal/ssa/rewriteRISCV64.go‎

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/cmd/compile/internal/ssagen/intrinsics.go‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -662,31 +662,31 @@ func initIntrinsics(cfg *intrinsicBuildConfig) {
662662
s.vars[memVar] = p1
663663
return p0
664664
},
665-
sys.AMD64, sys.Loong64)
665+
sys.AMD64, sys.Loong64, sys.RISCV64)
666666
addF("internal/runtime/atomic", "And32",
667667
func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
668668
v := s.newValue3(ssa.OpAtomicAnd32value, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
669669
p0, p1 := s.split(v)
670670
s.vars[memVar] = p1
671671
return p0
672672
},
673-
sys.AMD64, sys.Loong64)
673+
sys.AMD64, sys.Loong64, sys.RISCV64)
674674
addF("internal/runtime/atomic", "Or64",
675675
func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
676676
v := s.newValue3(ssa.OpAtomicOr64value, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
677677
p0, p1 := s.split(v)
678678
s.vars[memVar] = p1
679679
return p0
680680
},
681-
sys.AMD64, sys.Loong64)
681+
sys.AMD64, sys.Loong64, sys.RISCV64)
682682
addF("internal/runtime/atomic", "Or32",
683683
func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
684684
v := s.newValue3(ssa.OpAtomicOr32value, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
685685
p0, p1 := s.split(v)
686686
s.vars[memVar] = p1
687687
return p0
688688
},
689-
sys.AMD64, sys.Loong64)
689+
sys.AMD64, sys.Loong64, sys.RISCV64)
690690

691691
// Aliases for atomic load operations
692692
alias("internal/runtime/atomic", "Loadint32", "internal/runtime/atomic", "Load", all...)
@@ -1333,16 +1333,16 @@ func initIntrinsics(cfg *intrinsicBuildConfig) {
13331333
alias("sync/atomic", "AddUintptr", "internal/runtime/atomic", "Xadd", p4...)
13341334
alias("sync/atomic", "AddUintptr", "internal/runtime/atomic", "Xadd64", p8...)
13351335

1336-
alias("sync/atomic", "AndInt32", "internal/runtime/atomic", "And32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1337-
alias("sync/atomic", "AndUint32", "internal/runtime/atomic", "And32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1338-
alias("sync/atomic", "AndInt64", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1339-
alias("sync/atomic", "AndUint64", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1340-
alias("sync/atomic", "AndUintptr", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1341-
alias("sync/atomic", "OrInt32", "internal/runtime/atomic", "Or32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1342-
alias("sync/atomic", "OrUint32", "internal/runtime/atomic", "Or32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1343-
alias("sync/atomic", "OrInt64", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1344-
alias("sync/atomic", "OrUint64", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1345-
alias("sync/atomic", "OrUintptr", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
1336+
alias("sync/atomic", "AndInt32", "internal/runtime/atomic", "And32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1337+
alias("sync/atomic", "AndUint32", "internal/runtime/atomic", "And32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1338+
alias("sync/atomic", "AndInt64", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1339+
alias("sync/atomic", "AndUint64", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1340+
alias("sync/atomic", "AndUintptr", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1341+
alias("sync/atomic", "OrInt32", "internal/runtime/atomic", "Or32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1342+
alias("sync/atomic", "OrUint32", "internal/runtime/atomic", "Or32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1343+
alias("sync/atomic", "OrInt64", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1344+
alias("sync/atomic", "OrUint64", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
1345+
alias("sync/atomic", "OrUintptr", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64, sys.ArchRISCV64)
13461346

13471347
/******** math/big ********/
13481348
alias("math/big", "mulWW", "math/bits", "Mul64", p8...)

‎src/cmd/compile/internal/ssagen/intrinsics_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1072,6 +1072,8 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{
10721072
{"ppc64le", "crypto/internal/constanttime", "Select"}: struct{}{},
10731073
{"ppc64le", "crypto/internal/constanttime", "boolToUint8"}: struct{}{},
10741074
{"riscv64", "internal/runtime/atomic", "And"}: struct{}{},
1075+
{"riscv64", "internal/runtime/atomic", "And32"}: struct{}{},
1076+
{"riscv64", "internal/runtime/atomic", "And64"}: struct{}{},
10751077
{"riscv64", "internal/runtime/atomic", "And8"}: struct{}{},
10761078
{"riscv64", "internal/runtime/atomic", "Cas"}: struct{}{},
10771079
{"riscv64", "internal/runtime/atomic", "Cas64"}: struct{}{},
@@ -1092,6 +1094,8 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{
10921094
{"riscv64", "internal/runtime/atomic", "Loaduint"}: struct{}{},
10931095
{"riscv64", "internal/runtime/atomic", "Loaduintptr"}: struct{}{},
10941096
{"riscv64", "internal/runtime/atomic", "Or"}: struct{}{},
1097+
{"riscv64", "internal/runtime/atomic", "Or32"}: struct{}{},
1098+
{"riscv64", "internal/runtime/atomic", "Or64"}: struct{}{},
10951099
{"riscv64", "internal/runtime/atomic", "Or8"}: struct{}{},
10961100
{"riscv64", "internal/runtime/atomic", "Store"}: struct{}{},
10971101
{"riscv64", "internal/runtime/atomic", "Store64"}: struct{}{},
@@ -1167,6 +1171,11 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{
11671171
{"riscv64", "sync/atomic", "AddUint32"}: struct{}{},
11681172
{"riscv64", "sync/atomic", "AddUint64"}: struct{}{},
11691173
{"riscv64", "sync/atomic", "AddUintptr"}: struct{}{},
1174+
{"riscv64", "sync/atomic", "AndInt32"}: struct{}{},
1175+
{"riscv64", "sync/atomic", "AndInt64"}: struct{}{},
1176+
{"riscv64", "sync/atomic", "AndUint32"}: struct{}{},
1177+
{"riscv64", "sync/atomic", "AndUint64"}: struct{}{},
1178+
{"riscv64", "sync/atomic", "AndUintptr"}: struct{}{},
11701179
{"riscv64", "sync/atomic", "CompareAndSwapInt32"}: struct{}{},
11711180
{"riscv64", "sync/atomic", "CompareAndSwapInt64"}: struct{}{},
11721181
{"riscv64", "sync/atomic", "CompareAndSwapUint32"}: struct{}{},
@@ -1178,6 +1187,11 @@ var wantIntrinsics = map[testIntrinsicKey]struct{}{
11781187
{"riscv64", "sync/atomic", "LoadUint32"}: struct{}{},
11791188
{"riscv64", "sync/atomic", "LoadUint64"}: struct{}{},
11801189
{"riscv64", "sync/atomic", "LoadUintptr"}: struct{}{},
1190+
{"riscv64", "sync/atomic", "OrInt32"}: struct{}{},
1191+
{"riscv64", "sync/atomic", "OrInt64"}: struct{}{},
1192+
{"riscv64", "sync/atomic", "OrUint32"}: struct{}{},
1193+
{"riscv64", "sync/atomic", "OrUint64"}: struct{}{},
1194+
{"riscv64", "sync/atomic", "OrUintptr"}: struct{}{},
11811195
{"riscv64", "sync/atomic", "StoreInt32"}: struct{}{},
11821196
{"riscv64", "sync/atomic", "StoreInt64"}: struct{}{},
11831197
{"riscv64", "sync/atomic", "StoreUint32"}: struct{}{},

‎test/codegen/atomics.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ func (c *Counter) Increment() {
2323
// arm64/v8.0:".*arm64HasATOMICS"
2424
// arm64/v8.1:-".*arm64HasATOMICS"
2525
// amd64:"LOCK" -"CMPXCHG"
26+
// riscv64:"AMOADDW" -"JAL"
2627
atomic.AddInt32(&c.count, 1)
2728
}
2829

@@ -35,12 +36,14 @@ func atomicLogical64(x *atomic.Uint64) uint64 {
3536
// arm64/v8.1:-".*arm64HasATOMICS"
3637
// On amd64, make sure we use LOCK+AND instead of CMPXCHG when we don't use the result.
3738
// amd64:"LOCK" -"CMPXCHGQ"
39+
// riscv64:"AMOANDD" -"JAL"
3840
x.And(11)
3941
// arm64/v8.0:"LDCLRALD"
4042
// arm64/v8.1:"LDCLRALD"
4143
// arm64/v8.0:".*arm64HasATOMICS"
4244
// arm64/v8.1:-".*arm64HasATOMICS"
4345
// amd64:"LOCK" "CMPXCHGQ"
46+
// riscv64:"AMOANDD" -"JAL"
4447
r += x.And(22)
4548

4649
// arm64/v8.0:"LDORALD"
@@ -49,12 +52,14 @@ func atomicLogical64(x *atomic.Uint64) uint64 {
4952
// arm64/v8.1:-".*arm64HasATOMICS"
5053
// On amd64, make sure we use LOCK+OR instead of CMPXCHG when we don't use the result.
5154
// amd64:"LOCK" -"CMPXCHGQ"
55+
// riscv64:"AMOORD" -"JAL"
5256
x.Or(33)
5357
// arm64/v8.0:"LDORALD"
5458
// arm64/v8.1:"LDORALD"
5559
// arm64/v8.0:".*arm64HasATOMICS"
5660
// arm64/v8.1:-".*arm64HasATOMICS"
5761
// amd64:"LOCK" "CMPXCHGQ"
62+
// riscv64:"AMOORD" -"JAL"
5863
r += x.Or(44)
5964

6065
return r
@@ -69,12 +74,14 @@ func atomicLogical32(x *atomic.Uint32) uint32 {
6974
// arm64/v8.1:-".*arm64HasATOMICS"
7075
// On amd64, make sure we use LOCK+AND instead of CMPXCHG when we don't use the result.
7176
// amd64:"LOCK" -"CMPXCHGL"
77+
// riscv64:"AMOANDW" -"JAL"
7278
x.And(11)
7379
// arm64/v8.0:"LDCLRALW"
7480
// arm64/v8.1:"LDCLRALW"
7581
// arm64/v8.0:".*arm64HasATOMICS"
7682
// arm64/v8.1:-".*arm64HasATOMICS"
7783
// amd64:"LOCK" "CMPXCHGL"
84+
// riscv64:"AMOANDW" -"JAL"
7885
r += x.And(22)
7986

8087
// arm64/v8.0:"LDORALW"
@@ -83,12 +90,14 @@ func atomicLogical32(x *atomic.Uint32) uint32 {
8390
// arm64/v8.1:-".*arm64HasATOMICS"
8491
// On amd64, make sure we use LOCK+OR instead of CMPXCHG when we don't use the result.
8592
// amd64:"LOCK" -"CMPXCHGL"
93+
// riscv64:"AMOORW" -"JAL"
8694
x.Or(33)
8795
// arm64/v8.0:"LDORALW"
8896
// arm64/v8.1:"LDORALW"
8997
// arm64/v8.0:".*arm64HasATOMICS"
9098
// arm64/v8.1:-".*arm64HasATOMICS"
9199
// amd64:"LOCK" "CMPXCHGL"
100+
// riscv64:"AMOORW" -"JAL"
92101
r += x.Or(44)
93102

94103
return r

0 commit comments

Comments
 (0)