New refcount representation

include/swift/ABI/System.h

@@ -56,16 +56,36 @@
 /// ``pointer & SWIFT_ABI_XXX_OBJC_RESERVED_BITS_MASK == 0 && 
 /// pointer & SWIFT_ABI_XXX_SWIFT_SPARE_BITS_MASK != 0``.
 
+// Weak references use a marker to tell when they are controlled by
+// the ObjC runtime and when they are controlled by the Swift runtime.
+// Non-ObjC platforms don't use this marker.
+#define SWIFT_ABI_DEFAULT_OBJC_WEAK_REFERENCE_MARKER_MASK  0
+#define SWIFT_ABI_DEFAULT_OBJC_WEAK_REFERENCE_MARKER_VALUE 0
+
 /*********************************** i386 *************************************/
 
 // Heap objects are pointer-aligned, so the low two bits are unused.
 #define SWIFT_ABI_I386_SWIFT_SPARE_BITS_MASK 0x00000003U
 
+// ObjC weak reference discriminator is the LSB.
+#define SWIFT_ABI_I386_OBJC_WEAK_REFERENCE_MARKER_MASK  \
+  (SWIFT_ABI_DEFAULT_OBJC_RESERVED_BITS_MASK |          \
+   1<<SWIFT_ABI_DEFAULT_OBJC_NUM_RESERVED_LOW_BITS)
+#define SWIFT_ABI_I386_OBJC_WEAK_REFERENCE_MARKER_VALUE \
+  (1<<SWIFT_ABI_DEFAULT_OBJC_NUM_RESERVED_LOW_BITS)
+
 /*********************************** arm **************************************/
 
 // Heap objects are pointer-aligned, so the low two bits are unused.
 #define SWIFT_ABI_ARM_SWIFT_SPARE_BITS_MASK 0x00000003U
 
+// ObjC weak reference discriminator is the LSB.
+#define SWIFT_ABI_ARM_OBJC_WEAK_REFERENCE_MARKER_MASK  \
+  (SWIFT_ABI_DEFAULT_OBJC_RESERVED_BITS_MASK |          \
+   1<<SWIFT_ABI_DEFAULT_OBJC_NUM_RESERVED_LOW_BITS)
+#define SWIFT_ABI_ARM_OBJC_WEAK_REFERENCE_MARKER_VALUE \
+  (1<<SWIFT_ABI_DEFAULT_OBJC_NUM_RESERVED_LOW_BITS)
+
 /*********************************** x86-64 ***********************************/
 
 /// Darwin reserves the low 4GB of address space.
@@ -79,6 +99,14 @@
 #define SWIFT_ABI_X86_64_OBJC_RESERVED_BITS_MASK 0x8000000000000001ULL
 #define SWIFT_ABI_X86_64_OBJC_NUM_RESERVED_LOW_BITS 1
 
+// ObjC weak reference discriminator is the two bits
+// reserved for ObjC tagged pointers plus one more low bit.
+#define SWIFT_ABI_X86_64_OBJC_WEAK_REFERENCE_MARKER_MASK  \
+  (SWIFT_ABI_X86_64_OBJC_RESERVED_BITS_MASK |          \
+   1<<SWIFT_ABI_X86_64_OBJC_NUM_RESERVED_LOW_BITS)
+#define SWIFT_ABI_X86_64_OBJC_WEAK_REFERENCE_MARKER_VALUE \
+  (1<<SWIFT_ABI_X86_64_OBJC_NUM_RESERVED_LOW_BITS)
+
 /*********************************** arm64 ************************************/
 
 /// Darwin reserves the low 4GB of address space.
@@ -92,6 +120,14 @@
 #define SWIFT_ABI_ARM64_OBJC_RESERVED_BITS_MASK 0x8000000000000000ULL
 #define SWIFT_ABI_ARM64_OBJC_NUM_RESERVED_LOW_BITS 0
 
+// ObjC weak reference discriminator is the high bit
+// reserved for ObjC tagged pointers plus the LSB.
+#define SWIFT_ABI_ARM64_OBJC_WEAK_REFERENCE_MARKER_MASK  \
+  (SWIFT_ABI_ARM64_OBJC_RESERVED_BITS_MASK |          \
+   1<<SWIFT_ABI_ARM64_OBJC_NUM_RESERVED_LOW_BITS)
+#define SWIFT_ABI_ARM64_OBJC_WEAK_REFERENCE_MARKER_VALUE \
+  (1<<SWIFT_ABI_ARM64_OBJC_NUM_RESERVED_LOW_BITS)
+
 /*********************************** powerpc64 ********************************/
 
 // Heap objects are pointer-aligned, so the low three bits are unused.

include/swift/Runtime/Debug.h

@@ -20,7 +20,6 @@
 #include <llvm/Support/Compiler.h>
 #include <stdint.h>
 #include "swift/Runtime/Config.h"
-#include "swift/Runtime/Metadata.h"
 #include "swift/Runtime/Unreachable.h"
 
 #ifdef SWIFT_HAVE_CRASHREPORTERCLIENT
@@ -63,6 +62,12 @@ static void CRSetCrashLogMessage(const char *) {}
 
 namespace swift {
 
+// Duplicated from Metadata.h. We want to use this header
+// in places that cannot themselves include Metadata.h.
+struct InProcess;
+template <typename Runtime> struct TargetMetadata;
+using Metadata = TargetMetadata<InProcess>;
+
 // swift::crash() halts with a crash log message, 
 // but otherwise tries not to disturb register state.
 
@@ -87,11 +92,6 @@ static inline void _failCorruptType(const Metadata *type) {
 LLVM_ATTRIBUTE_NORETURN
 extern void
 fatalError(uint32_t flags, const char *format, ...);
-
-struct InProcess;
-
-template <typename Runtime> struct TargetMetadata;
-using Metadata = TargetMetadata<InProcess>;
 
 // swift_dynamicCastFailure halts using fatalError()
 // with a description of a failed cast's types.
@@ -112,6 +112,14 @@ swift_dynamicCastFailure(const void *sourceType, const char *sourceName,
 SWIFT_RUNTIME_EXPORT
 void swift_reportError(uint32_t flags, const char *message);
 
+// Halt due to an overflow in swift_retain().
+LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE
+void swift_abortRetainOverflow();
+
+// Halt due to reading an unowned reference to a dead object.
+LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE
+void swift_abortRetainUnowned(const void *object);
+
 // namespace swift
 }
 

include/swift/Runtime/HeapObject.h

@@ -219,18 +219,6 @@ SWIFT_RUNTIME_EXPORT
 void (*SWIFT_CC(RegisterPreservingCC) _swift_nonatomic_retain_n)(HeapObject *object,
                                                        uint32_t n);
 
-static inline void _swift_retain_inlined(HeapObject *object) {
-  if (object) {
-    object->refCount.increment();
-  }
-}
-
-static inline void _swift_nonatomic_retain_inlined(HeapObject *object) {
-  if (object) {
-    object->refCount.incrementNonAtomic();
-  }
-}
-
 /// Atomically increments the reference count of an object, unless it has
 /// already been destroyed. Returns nil if the object is dead.
 SWIFT_RT_ENTRY_VISIBILITY
@@ -512,22 +500,22 @@ struct UnownedReference {
   HeapObject *Value;
 };
 
-/// Increment the weak/unowned retain count.
+/// Increment the unowned retain count.
 SWIFT_RT_ENTRY_VISIBILITY
 void swift_unownedRetain(HeapObject *value)
     SWIFT_CC(RegisterPreservingCC);
 
-/// Decrement the weak/unowned retain count.
+/// Decrement the unowned retain count.
 SWIFT_RT_ENTRY_VISIBILITY
 void swift_unownedRelease(HeapObject *value)
     SWIFT_CC(RegisterPreservingCC);
 
-/// Increment the weak/unowned retain count by n.
+/// Increment the unowned retain count by n.
 SWIFT_RT_ENTRY_VISIBILITY
 void swift_unownedRetain_n(HeapObject *value, int n)
     SWIFT_CC(RegisterPreservingCC);
 
-/// Decrement the weak/unowned retain count by n.
+/// Decrement the unowned retain count by n.
 SWIFT_RT_ENTRY_VISIBILITY
 void swift_unownedRelease_n(HeapObject *value, int n)
     SWIFT_CC(RegisterPreservingCC);
@@ -540,7 +528,7 @@ void swift_unownedRetainStrong(HeapObject *value)
 
 /// Increment the strong retain count of an object which may have been
 /// deallocated, aborting if it has been deallocated, and decrement its
-/// weak/unowned reference count.
+/// unowned reference count.
 SWIFT_RT_ENTRY_VISIBILITY
 void swift_unownedRetainStrongAndRelease(HeapObject *value)
     SWIFT_CC(RegisterPreservingCC);
@@ -614,16 +602,8 @@ static inline void swift_unownedTakeAssign(UnownedReference *dest,
 /****************************** WEAK REFERENCES ******************************/
 /*****************************************************************************/
 
-/// A weak reference value object.  This is ABI.
-struct WeakReference {
-  uintptr_t Value;
-};
-
-/// Return true if this is a native weak reference
-///
-/// \param ref - never null
-/// \return true if ref is a native weak reference
-bool isNativeSwiftWeakReference(WeakReference *ref);
+// Defined in Runtime/WeakReference.h
+class WeakReference;
 
 /// Initialize a weak reference.
 ///

include/swift/Runtime/Metadata.h

@@ -136,7 +136,7 @@ using TargetFarRelativeIndirectablePointer
   = typename Runtime::template FarRelativeIndirectablePointer<Pointee,Nullable>;
 
 struct HeapObject;
-struct WeakReference;
+class WeakReference;
 
 template <typename Runtime> struct TargetMetadata;
 using Metadata = TargetMetadata<InProcess>;
@@ -1832,17 +1832,6 @@ struct TargetObjCClassWrapperMetadata : public TargetMetadata<Runtime> {
 using ObjCClassWrapperMetadata
   = TargetObjCClassWrapperMetadata<InProcess>;
 
-// FIXME: Workaround for rdar://problem/18889711. 'Consume' does not require
-// a barrier on ARM64, but LLVM doesn't know that. Although 'relaxed'
-// is formally UB by C++11 language rules, we should be OK because neither
-// the processor model nor the optimizer can realistically reorder our uses
-// of 'consume'.
-#if __arm64__ || __arm__
-#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_relaxed)
-#else
-#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_consume)
-#endif
-
 /// The structure of metadata for foreign types where the source
 /// language doesn't provide any sort of more interesting metadata for
 /// us to use.

stdlib/public/SwiftShims/HeapObject.h

@@ -31,8 +31,7 @@ typedef struct HeapMetadata HeapMetadata;
 // The members of the HeapObject header that are not shared by a
 // standard Objective-C instance
 #define SWIFT_HEAPOBJECT_NON_OBJC_MEMBERS       \
-  StrongRefCount refCount;                      \
-  WeakRefCount weakRefCount
+  InlineRefCounts refCounts;
 
 /// The Swift heap-object header.
 struct HeapObject {
@@ -48,8 +47,7 @@ struct HeapObject {
   // Initialize a HeapObject header as appropriate for a newly-allocated object.
   constexpr HeapObject(HeapMetadata const *newMetadata) 
     : metadata(newMetadata)
-    , refCount(StrongRefCount::Initialized)
-    , weakRefCount(WeakRefCount::Initialized)
+    , refCounts(InlineRefCounts::Initialized)
   { }
 #endif
 };
@@ -59,6 +57,11 @@ static_assert(swift::IsTriviallyConstructible<HeapObject>::value,
               "HeapObject must be trivially initializable");
 static_assert(std::is_trivially_destructible<HeapObject>::value,
               "HeapObject must be trivially destructible");
+// FIXME: small header for 32-bit
+//static_assert(sizeof(HeapObject) == 2*sizeof(void*),
+//              "HeapObject must be two pointers long");
+static_assert(alignof(HeapObject) == alignof(void*),
+              "HeapObject must be pointer-aligned");
 
 } // end namespace swift
 #endif