Merge pull request #30298 from gottesmm/pr-4e1212f1e4e3640e5037f11164d07e20fcea7324

[inliner] Add a new Inliner that only inlines AlwaysInline functions (but do not put it in the pass pipeline).

Author: gottesmm

include/swift/SILOptimizer/PassManager/Passes.def

@@ -243,6 +243,8 @@ PASS(NonTransparentFunctionOwnershipModelEliminator,
      "Eliminate Ownership Annotations from non-transparent SIL Functions")
 PASS(RCIdentityDumper, "rc-id-dumper",
      "Print Reference Count Identities")
+PASS(AlwaysInlineInliner, "always-inline",
+     "Inline always inline functions")
 PASS(PerfInliner, "inline",
      "Performance Function Inlining")
 PASS(PerformanceConstantPropagation, "performance-constant-propagation",

include/swift/SILOptimizer/Utils/PerformanceInlinerUtils.h

@@ -35,7 +35,8 @@ class SideEffectAnalysis;
 enum class InlineSelection {
   Everything,
   NoGlobalInit, // and no availability semantics calls
-  NoSemanticsAndGlobalInit
+  NoSemanticsAndGlobalInit,
+  OnlyInlineAlways,
 };
 
 // Returns the callee of an apply_inst if it is basically inlinable.

lib/SILOptimizer/Transforms/PerformanceInliner.cpp

@@ -44,6 +44,11 @@ llvm::cl::opt<bool>
     EnableSILAggressiveInlining("sil-aggressive-inline", llvm::cl::init(false),
                                llvm::cl::desc("Enable aggressive inlining"));
 
+llvm::cl::opt<bool> EnableVerifyAfterInlining(
+    "sil-inline-verify-after-inline", llvm::cl::init(false),
+    llvm::cl::desc("Run sil verification after inlining all found callee apply "
+                   "sites into a caller."));
+
 //===----------------------------------------------------------------------===//
 //                           Performance Inliner
 //===----------------------------------------------------------------------===//
@@ -919,11 +924,9 @@ bool SILPerformanceInliner::inlineCallsIntoFunction(SILFunction *Caller) {
     }
 
     // If we have a callee that doesn't have ownership, but the caller does have
-    // ownership... do not inline. The two modes are incompatible. Today this
-    // should only happen with transparent functions.
+    // ownership... do not inline. The two modes are incompatible, so skip this
+    // apply site for now.
     if (!Callee->hasOwnership() && Caller->hasOwnership()) {
-      assert(Caller->isTransparent() &&
-             "Should only happen with transparent functions");
       continue;
     }
 
@@ -953,6 +956,13 @@ bool SILPerformanceInliner::inlineCallsIntoFunction(SILFunction *Caller) {
     StackNesting().correctStackNesting(Caller);
   }
 
+  // If we were asked to verify our caller after inlining all callees we could
+  // find into it, do so now. This makes it easier to catch verification bugs in
+  // the inliner without running the entire inliner.
+  if (EnableVerifyAfterInlining) {
+    Caller->verify();
+  }
+
   return true;
 }
 
@@ -1027,6 +1037,11 @@ class SILPerformanceInlinerPass : public SILFunctionTransform {
 };
 } // end anonymous namespace
 
+SILTransform *swift::createAlwaysInlineInliner() {
+  return new SILPerformanceInlinerPass(InlineSelection::OnlyInlineAlways,
+                                       "InlineAlways");
+}
+
 /// Create an inliner pass that does not inline functions that are marked with
 /// the @_semantics, @_effects or global_init attributes.
 SILTransform *swift::createEarlyInliner() {

lib/SILOptimizer/Utils/PerformanceInlinerUtils.cpp

@@ -637,14 +637,21 @@ SILFunction *swift::getEligibleFunction(FullApplySite AI,
   if (!SILInliner::canInlineApplySite(AI))
     return nullptr;
 
+  // If our inline selection is only always inline, do a quick check if we have
+  // an always inline function and bail otherwise.
+  if (WhatToInline == InlineSelection::OnlyInlineAlways &&
+      Callee->getInlineStrategy() != AlwaysInline) {
+    return nullptr;
+  }
+
   ModuleDecl *SwiftModule = Callee->getModule().getSwiftModule();
   bool IsInStdlib = (SwiftModule->isStdlibModule() ||
                      SwiftModule->isOnoneSupportModule());
 
   // Don't inline functions that are marked with the @_semantics or @_effects
   // attribute if the inliner is asked not to inline them.
   if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) {
-    if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) {
+    if (WhatToInline >= InlineSelection::NoSemanticsAndGlobalInit) {
       if (shouldSkipApplyDuringEarlyInlining(AI))
         return nullptr;
       if (Callee->hasSemanticsAttr("inline_late"))
@@ -692,13 +699,14 @@ SILFunction *swift::getEligibleFunction(FullApplySite AI,
     // Check if passed Self is the same as the Self of the caller.
     // In this case, it is safe to inline because both functions
     // use the same Self.
-    if (AI.hasSelfArgument() && Caller->hasSelfMetadataParam()) {
-      auto CalleeSelf = stripCasts(AI.getSelfArgument());
-      auto CallerSelf = Caller->getSelfMetadataArgument();
-      if (CalleeSelf != SILValue(CallerSelf))
-        return nullptr;
-    } else
+    if (!AI.hasSelfArgument() || !Caller->hasSelfMetadataParam()) {
       return nullptr;
+    }
+    auto CalleeSelf = stripCasts(AI.getSelfArgument());
+    auto CallerSelf = Caller->getSelfMetadataArgument();
+    if (CalleeSelf != SILValue(CallerSelf)) {
+      return nullptr;
+    }
   }
 
   // Detect self-recursive calls.

test/SILOptimizer/inlinealways_inliner.sil

@@ -0,0 +1,44 @@
+// RUN: %target-sil-opt -enable-sil-verify-all %s -always-inline | %FileCheck %s
+
+sil @doSomething1 : $@convention(thin) () -> ()
+sil @doSomething2 : $@convention(thin) () -> ()
+sil @doSomething3 : $@convention(thin) () -> ()
+
+sil [ossa] [always_inline] @do_inline_this : $@convention(thin) () -> () {
+bb0:
+  %d1 = function_ref @doSomething1 : $@convention(thin) () -> ()
+  apply %d1() : $@convention(thin) () -> ()
+  %9999 = tuple()
+  return %9999 : $()
+}
+
+sil [ossa] @donot_inline_this : $@convention(thin) () -> () {
+bb0:
+  %d1 = function_ref @doSomething2 : $@convention(thin) () -> ()
+  apply %d1() : $@convention(thin) () -> ()
+  %9999 = tuple()
+  return %9999 : $()
+}
+
+sil [ossa] @empty_function : $@convention(thin) () -> () {
+bb0:
+  %9999 = tuple()
+  return %9999 : $()
+}
+
+// CHECK-LABEL: sil [ossa] @caller : $@convention(thin) () -> () {
+// CHECK-NOT: function_ref @do_inline_this : $@convention(thin) () -> ()
+// CHECK: function_ref @donot_inline_this : $@convention(thin) () -> ()
+// CHECK: function_ref @empty_function : $@convention(thin) () -> ()
+// CHECK: } // end sil function 'caller'
+sil [ossa] @caller : $@convention(thin) () -> () {
+bb0:
+  %c1 = function_ref @do_inline_this : $@convention(thin) () -> ()
+  apply %c1() : $@convention(thin) () -> ()
+  %c2 = function_ref @donot_inline_this : $@convention(thin) () -> ()
+  apply %c2() : $@convention(thin) () -> ()
+  %c3 = function_ref @empty_function : $@convention(thin) () -> ()
+  apply %c3() : $@convention(thin) () -> ()
+  %9999 = tuple()
+  return %9999 : $()
+}