Skip to content

Commit f613f86

Browse files
davecheneydavecheney
committed
interp: evaluate atomicrmw and cmpxchg during initialization
sync/atomic functions are compiled to ordinary bodies built on atomicrmw and cmpxchg, so interpreting those instructions covers Add, And, Or, Swap and CompareAndSwap for every type without matching on function names. Before other execution contexts exist an atomic operation on memory the interpreter owns is just a load and a store. Volatile operations and memory with external loads or stores are still left for runtime, as with the existing atomic load and store handling.
1 parent 7ef7f52 commit f613f86

8 files changed

Lines changed: 454 additions & 52 deletions

File tree

‎interp/compiler.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,27 @@ func (r *runner) compileFunction(llvmFn llvm.Value) *function {
249249
r.getValue(llvmInst.Operand(0)),
250250
r.getValue(llvmInst.Operand(1)),
251251
}
252+
case atomicRMW:
253+
// atomicrmw: [ptr, value, binop]
254+
inst.name = llvmInst.Name()
255+
if op, ok := atomicRMWBinOp(llvmInst); ok {
256+
inst.operands = []value{
257+
r.getValue(llvmInst.Operand(0)),
258+
r.getValue(llvmInst.Operand(1)),
259+
literalValue{uint8(op)},
260+
}
261+
}
262+
case atomicCmpXchg:
263+
// cmpxchg: [ptr, cmp, new, resultSize, successOffset]
264+
inst.name = llvmInst.Name()
265+
resultType := llvmInst.Type()
266+
inst.operands = []value{
267+
r.getValue(llvmInst.Operand(0)),
268+
r.getValue(llvmInst.Operand(1)),
269+
r.getValue(llvmInst.Operand(2)),
270+
literalValue{r.targetData.TypeAllocSize(resultType)},
271+
literalValue{r.targetData.ElementOffset(resultType, 1)},
272+
}
252273
case llvm.Alloca:
253274
// Alloca allocates stack space for local variables.
254275
numElements := r.getValue(inst.llvmInst.Operand(0)).(literalValue).value.(uint32)
@@ -409,6 +430,53 @@ func (r *runner) compileFunction(llvmFn llvm.Value) *function {
409430
return fn
410431
}
411432

433+
// Opcodes from llvm-c/Core.h (LLVMOpcode) that go-llvm doesn't export.
434+
const (
435+
atomicCmpXchg llvm.Opcode = 56
436+
atomicRMW llvm.Opcode = 57
437+
)
438+
439+
// atomicRMWBinOp returns the operation of an atomicrmw instruction.
440+
// go-llvm doesn't wrap LLVMGetAtomicRMWBinOp, so read it from the IR text.
441+
func atomicRMWBinOp(inst llvm.Value) (llvm.AtomicRMWBinOp, bool) {
442+
fields := strings.Fields(inst.String())
443+
for i, field := range fields {
444+
if field != "atomicrmw" {
445+
continue
446+
}
447+
for _, op := range fields[i+1:] {
448+
switch op {
449+
case "volatile":
450+
continue
451+
case "xchg":
452+
return llvm.AtomicRMWBinOpXchg, true
453+
case "add":
454+
return llvm.AtomicRMWBinOpAdd, true
455+
case "sub":
456+
return llvm.AtomicRMWBinOpSub, true
457+
case "and":
458+
return llvm.AtomicRMWBinOpAnd, true
459+
case "nand":
460+
return llvm.AtomicRMWBinOpNand, true
461+
case "or":
462+
return llvm.AtomicRMWBinOpOr, true
463+
case "xor":
464+
return llvm.AtomicRMWBinOpXor, true
465+
case "max":
466+
return llvm.AtomicRMWBinOpMax, true
467+
case "min":
468+
return llvm.AtomicRMWBinOpMin, true
469+
case "umax":
470+
return llvm.AtomicRMWBinOpUMax, true
471+
case "umin":
472+
return llvm.AtomicRMWBinOpUMin, true
473+
}
474+
return 0, false
475+
}
476+
}
477+
return 0, false
478+
}
479+
412480
// instructionNameMap maps from instruction opcodes to instruction names. This
413481
// can be useful for debug logging.
414482
var instructionNameMap = [...]string{
@@ -445,6 +513,8 @@ var instructionNameMap = [...]string{
445513
llvm.Load: "load",
446514
llvm.Store: "store",
447515
llvm.GetElementPtr: "getelementptr",
516+
atomicCmpXchg: "cmpxchg",
517+
atomicRMW: "atomicrmw",
448518

449519
// Cast Operators
450520
llvm.Trunc: "trunc",

‎interp/interpreter.go‎

Lines changed: 124 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -294,53 +294,6 @@ func (r *runner) run(fn *function, params []value, parentMem *memoryView, indent
294294
}
295295
callFn := r.getFunction(fnPtr.llvmValue(&mem))
296296
switch {
297-
case callFn.name == "sync/atomic.SwapInt32" || callFn.name == "sync/atomic.SwapInt64" ||
298-
callFn.name == "sync/atomic.SwapUint32" || callFn.name == "sync/atomic.SwapUint64" ||
299-
callFn.name == "sync/atomic.SwapUintptr" || callFn.name == "sync/atomic.SwapPointer" ||
300-
callFn.name == "sync/atomic.CompareAndSwapInt32" || callFn.name == "sync/atomic.CompareAndSwapInt64" ||
301-
callFn.name == "sync/atomic.CompareAndSwapUint32" || callFn.name == "sync/atomic.CompareAndSwapUint64" ||
302-
callFn.name == "sync/atomic.CompareAndSwapUintptr" || callFn.name == "sync/atomic.CompareAndSwapPointer":
303-
ptr, err := operands[1].asPointer(r)
304-
if err != nil {
305-
return nil, mem, r.errorAt(inst, err)
306-
}
307-
if mem.hasExternalLoadOrStore(ptr) {
308-
if err := r.runAtRuntime(fn, inst, locals, &mem, indent); err != nil {
309-
return nil, mem, err
310-
}
311-
continue
312-
}
313-
old := mem.load(ptr, operands[2].len(r))
314-
if old == nil {
315-
if err := r.runAtRuntime(fn, inst, locals, &mem, indent); err != nil {
316-
return nil, mem, err
317-
}
318-
continue
319-
}
320-
newValue := operands[2]
321-
compare := strings.HasPrefix(callFn.name, "sync/atomic.CompareAndSwap")
322-
if compare {
323-
equal, err := r.interpretICmp(old, operands[2], llvm.IntEQ)
324-
if err != nil {
325-
return nil, mem, r.errorAt(inst, err)
326-
}
327-
if !equal {
328-
locals[inst.localIndex] = literalValue{uint8(0)}
329-
continue
330-
}
331-
newValue = operands[3]
332-
}
333-
if !mem.store(newValue, ptr) {
334-
if err := r.runAtRuntime(fn, inst, locals, &mem, indent); err != nil {
335-
return nil, mem, err
336-
}
337-
continue
338-
}
339-
if compare {
340-
locals[inst.localIndex] = literalValue{uint8(1)}
341-
} else {
342-
locals[inst.localIndex] = old
343-
}
344297
case callFn.name == "runtime.trackPointer":
345298
// Allocas and such are created as globals, so don't need a
346299
// runtime.trackPointer.
@@ -762,6 +715,76 @@ func (r *runner) run(fn *function, params []value, parentMem *memoryView, indent
762715
return nil, mem, err
763716
}
764717
}
718+
case atomicRMW:
719+
if len(operands) != 3 {
720+
return nil, mem, r.errorAt(inst, errUnsupportedInst)
721+
}
722+
ptr, err := operands[0].asPointer(r)
723+
if err != nil {
724+
return nil, mem, r.errorAt(inst, err)
725+
}
726+
val := operands[1]
727+
size := val.len(r)
728+
var old value
729+
if !inst.llvmInst.IsVolatile() && !mem.hasExternalLoadOrStore(ptr) && (size == 1 || size == 2 || size == 4 || size == 8) {
730+
old = mem.load(ptr, size)
731+
}
732+
if old == nil {
733+
err := r.runAtRuntime(fn, inst, locals, &mem, indent)
734+
if err != nil {
735+
return nil, mem, err
736+
}
737+
continue
738+
}
739+
result, ok := r.interpretAtomicRMW(llvm.AtomicRMWBinOp(operands[2].(literalValue).value.(uint8)), old, val)
740+
if !ok || !mem.store(result, ptr) {
741+
err := r.runAtRuntime(fn, inst, locals, &mem, indent)
742+
if err != nil {
743+
return nil, mem, err
744+
}
745+
continue
746+
}
747+
if r.debug {
748+
fmt.Fprintln(os.Stderr, indent+"atomicrmw:", operands, "->", old)
749+
}
750+
locals[inst.localIndex] = old
751+
case atomicCmpXchg:
752+
ptr, err := operands[0].asPointer(r)
753+
if err != nil {
754+
return nil, mem, r.errorAt(inst, err)
755+
}
756+
var old value
757+
if !inst.llvmInst.IsVolatile() && !mem.hasExternalLoadOrStore(ptr) {
758+
old = mem.load(ptr, operands[1].len(r))
759+
}
760+
if old == nil {
761+
err := r.runAtRuntime(fn, inst, locals, &mem, indent)
762+
if err != nil {
763+
return nil, mem, err
764+
}
765+
continue
766+
}
767+
equal, err := r.interpretICmp(old, operands[1], llvm.IntEQ)
768+
if err != nil {
769+
return nil, mem, r.errorAt(inst, err)
770+
}
771+
if equal && !mem.store(operands[2], ptr) {
772+
err := r.runAtRuntime(fn, inst, locals, &mem, indent)
773+
if err != nil {
774+
return nil, mem, err
775+
}
776+
continue
777+
}
778+
// The result is {old, success}.
779+
result := newRawValue(uint32(operands[3].(literalValue).value.(uint64)))
780+
copy(result.buf, old.asRawValue(r).buf)
781+
if equal {
782+
result.buf[operands[4].(literalValue).value.(uint64)] = 1
783+
}
784+
if r.debug {
785+
fmt.Fprintln(os.Stderr, indent+"cmpxchg:", operands, "->", result)
786+
}
787+
locals[inst.localIndex] = result
765788
case llvm.Alloca:
766789
// Alloca normally allocates some stack memory. In the interpreter,
767790
// it allocates a global instead.
@@ -1157,6 +1180,45 @@ func (r *runner) interpretICmp(lhs, rhs value, predicate llvm.IntPredicate) (boo
11571180
}
11581181
}
11591182

1183+
// interpretAtomicRMW returns the value an atomicrmw stores. It returns false
1184+
// if the operation must be done at runtime.
1185+
func (r *runner) interpretAtomicRMW(op llvm.AtomicRMWBinOp, old, val value) (value, bool) {
1186+
if op == llvm.AtomicRMWBinOpXchg {
1187+
return val, true
1188+
}
1189+
oldUint, valUint, err := uintOperands(r, old, val)
1190+
if err != nil {
1191+
return nil, false
1192+
}
1193+
oldInt, valInt, _ := intOperands(r, old, val)
1194+
var result uint64
1195+
switch op {
1196+
case llvm.AtomicRMWBinOpAdd:
1197+
result = oldUint + valUint
1198+
case llvm.AtomicRMWBinOpSub:
1199+
result = oldUint - valUint
1200+
case llvm.AtomicRMWBinOpAnd:
1201+
result = oldUint & valUint
1202+
case llvm.AtomicRMWBinOpNand:
1203+
result = ^(oldUint & valUint)
1204+
case llvm.AtomicRMWBinOpOr:
1205+
result = oldUint | valUint
1206+
case llvm.AtomicRMWBinOpXor:
1207+
result = oldUint ^ valUint
1208+
case llvm.AtomicRMWBinOpMax:
1209+
result = uint64(max(oldInt, valInt))
1210+
case llvm.AtomicRMWBinOpMin:
1211+
result = uint64(min(oldInt, valInt))
1212+
case llvm.AtomicRMWBinOpUMax:
1213+
result = max(oldUint, valUint)
1214+
case llvm.AtomicRMWBinOpUMin:
1215+
result = min(oldUint, valUint)
1216+
default:
1217+
return nil, false
1218+
}
1219+
return makeLiteralInt(result, int(val.len(r)*8)), true
1220+
}
1221+
11601222
func uintOperands(r *runner, lhs, rhs value) (uint64, uint64, error) {
11611223
lhsUint, err := lhs.Uint(r)
11621224
if err != nil {
@@ -1231,6 +1293,21 @@ func (r *runner) runAtRuntime(fn *function, inst instruction, locals []value, me
12311293
if ordering := inst.llvmInst.Ordering(); ordering != llvm.AtomicOrderingNotAtomic {
12321294
result.SetOrdering(ordering)
12331295
}
1296+
case atomicRMW, atomicCmpXchg:
1297+
for _, op := range operands {
1298+
err := mem.markExternalStore(op)
1299+
if err != nil {
1300+
return r.errorAt(inst, err)
1301+
}
1302+
}
1303+
if inst.opcode == atomicRMW {
1304+
op := llvm.AtomicRMWBinOp(inst.operands[2].(literalValue).value.(uint8))
1305+
result = r.builder.CreateAtomicRMW(op, operands[0], operands[1], inst.llvmInst.Ordering(), inst.llvmInst.IsAtomicSingleThread())
1306+
} else {
1307+
result = r.builder.CreateAtomicCmpXchg(operands[0], operands[1], operands[2], inst.llvmInst.CmpXchgSuccessOrdering(), inst.llvmInst.CmpXchgFailureOrdering(), inst.llvmInst.IsAtomicSingleThread())
1308+
}
1309+
result.SetVolatile(inst.llvmInst.IsVolatile())
1310+
result.SetAlignment(inst.llvmInst.Alignment())
12341311
case llvm.BitCast:
12351312
result = r.builder.CreateBitCast(operands[0], inst.llvmInst.Type(), inst.name)
12361313
case llvm.PtrToInt:

0 commit comments

Comments
 (0)