[vm] Do not inline force-optimized functions

Change-Id: I11b5e799f46be7631b148c479ccc19fc507cf56a
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/120660
Reviewed-by: Samir Jindel <[email protected]>
Reviewed-by: Martin Kustermann <[email protected]>

Author: dcharkes

runtime/vm/object.cc

@@ -6623,6 +6623,15 @@ void Function::SetIsOptimizable(bool value) const {
 }
 
 bool Function::CanBeInlined() const {
+  // Our force-optimized functions cannot deoptimize to an unoptimized frame.
+  // If the instructions of the force-optimized function body get moved via
+  // code motion, we might attempt do deoptimize a frame where the force-
+  // optimized function has only partially finished. Since force-optimized
+  // functions cannot deoptimize to unoptimized frames we prevent them from
+  // being inlined (for now).
+  if (ForceOptimize()) {
+    return false;
+  }
 #if defined(PRODUCT)
   return is_inlinable() && !is_external() && !is_generated_body();
 #else

runtime/vm/object.h

@@ -5292,9 +5292,6 @@ class Code : public Object {
     return ForceOptimizedBit::decode(raw_ptr()->state_bits_);
   }
   void set_is_force_optimized(bool value) const;
-  static bool IsForceOptimized(RawCode* code) {
-    return Code::ForceOptimizedBit::decode(code->ptr()->state_bits_);
-  }
 
   bool is_alive() const { return AliveBit::decode(raw_ptr()->state_bits_); }
   void set_is_alive(bool value) const;