[vm] Handle sentinel in the IL serializer and Array-backed hash maps.

Fixes crashes in vm/dart when use_nnbd = true.

Change-Id: Ieeb45227208cc6219f536130df38249d902dfe7f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/137462
Reviewed-by: Siva Annamalai <[email protected]>
Commit-Queue: Ryan Macnak <[email protected]>

Author: rmacnak-google

runtime/vm/compiler/backend/il_deserializer.cc

@@ -1432,6 +1432,10 @@ bool FlowGraphDeserializer::ParseDartValue(SExpression* sexp, Object* out) {
     // We'll use the null value in *out as a marker later, so go ahead and exit
     // early if we parse one.
     if (sym->Equals("null")) return true;
+    if (sym->Equals("sentinel")) {
+      *out = Object::sentinel().raw();
+      return true;
+    }
 
     // The only other symbols that should appear in Dart value position are
     // names of constant definitions.
@@ -1662,6 +1666,8 @@ bool FlowGraphDeserializer::ParseInstance(SExpList* list, Object* out) {
     return false;
   }
 
+  ASSERT(cid_sexp->value() != kNullCid);  // Must use canonical instances.
+  ASSERT(cid_sexp->value() != kBoolCid);  // Must use canonical instances.
   instance_class_ = table->At(cid_sexp->value());
   *out = Instance::New(instance_class_, Heap::kOld);
   auto& instance = Instance::Cast(*out);

runtime/vm/compiler/backend/il_serializer.cc

@@ -670,6 +670,9 @@ SExpression* FlowGraphSerializer::ObjectToSExp(const Object& dartval) {
   if (dartval.IsNull()) {
     return new (zone()) SExpSymbol("null");
   }
+  if (dartval.raw() == Object::sentinel().raw()) {
+    return new (zone()) SExpSymbol("sentinel");
+  }
   if (dartval.IsString()) {
     return new (zone()) SExpString(dartval.ToCString());
   }

runtime/vm/hash_table.h

@@ -62,7 +62,7 @@ namespace dart {
 // Each entry contains a key, followed by zero or more payload components,
 // and has 3 possible states: unused, occupied, or deleted.
 // The header tracks the number of entries in each state.
-// Any object except Object::sentinel() and Object::transition_sentinel()
+// Any object except the backing storage array and Object::transition_sentinel()
 // may be stored as a key. Any object may be stored in a payload.
 //
 // Parameters
@@ -138,7 +138,7 @@ class HashTable : public ValueObject {
 #endif  // !defined(PRODUCT)
 
     for (intptr_t i = kHeaderSize; i < data_->Length(); ++i) {
-      data_->SetAt(i, Object::sentinel());
+      data_->SetAt(i, *data_);
     }
   }
 
@@ -225,6 +225,9 @@ class HashTable : public ValueObject {
   // Sets the key of a previously unoccupied entry. This must not be the last
   // unoccupied entry.
   void InsertKey(intptr_t entry, const Object& key) const {
+    ASSERT(key.raw() != data_->raw());
+    ASSERT(key.raw() != Object::transition_sentinel().raw());
+
     ASSERT(!IsOccupied(entry));
     AdjustSmiValueAt(kOccupiedEntriesIndex, 1);
     if (IsDeleted(entry)) {
@@ -238,7 +241,7 @@ class HashTable : public ValueObject {
   }
 
   bool IsUnused(intptr_t entry) const {
-    return InternalGetKey(entry) == Object::sentinel().raw();
+    return InternalGetKey(entry) == data_->raw();
   }
   bool IsOccupied(intptr_t entry) const {
     return !IsUnused(entry) && !IsDeleted(entry);

runtime/vm/object.cc

@@ -17245,8 +17245,8 @@ bool Instance::CanonicalizeEquals(const Instance& other) const {
 }
 
 uint32_t Instance::CanonicalizeHash() const {
-  if (IsNull()) {
-    return 2011;
+  if (GetClassId() == kNullCid) {
+    return 2011;  // Matches null_patch.dart.
   }
   Thread* thread = Thread::Current();
   uint32_t hash = thread->heap()->GetCanonicalHash(raw());