[inliner] Add a new Inliner that only inlines AlwaysInline functions (but do not put it in the pass pipeline).

We need this anyways for -Onone and I want to do some experiments with running
this very early so I can expose more of the stdlib (modulo inlining) to the new
ownership optimizing passes.

I also changed how the inliner handles inlining around OSSA by changing it to
check early that if the caller is in ossa, then we only inline if all of the
callees that the caller calls are in ossa. The intention is to hopefully avoid
weird swings in code-size/perf due to the inliner heuristic's calculation being
artificially manipulated due to some callees not being available to inline (due
to this difference) when others are already available.

Author: gottesmm

include/swift/SILOptimizer/PassManager/Passes.def

@@ -243,6 +243,8 @@ PASS(NonTransparentFunctionOwnershipModelEliminator,
      "Eliminate Ownership Annotations from non-transparent SIL Functions")
 PASS(RCIdentityDumper, "rc-id-dumper",
      "Print Reference Count Identities")
+PASS(AlwaysInlineInliner, "always-inline",
+     "Inline always inline functions")
 PASS(PerfInliner, "inline",
      "Performance Function Inlining")
 PASS(PerformanceConstantPropagation, "performance-constant-propagation",

include/swift/SILOptimizer/Utils/PerformanceInlinerUtils.h

@@ -35,7 +35,8 @@ class SideEffectAnalysis;
 enum class InlineSelection {
   Everything,
   NoGlobalInit, // and no availability semantics calls
-  NoSemanticsAndGlobalInit
+  NoSemanticsAndGlobalInit,
+  OnlyInlineAlways,
 };
 
 // Returns the callee of an apply_inst if it is basically inlinable.

lib/SILOptimizer/Transforms/PerformanceInliner.cpp

@@ -44,6 +44,11 @@ llvm::cl::opt<bool>
     EnableSILAggressiveInlining("sil-aggressive-inline", llvm::cl::init(false),
                                llvm::cl::desc("Enable aggressive inlining"));
 
+llvm::cl::opt<bool> EnableVerifyAfterInlining(
+    "sil-inline-verify-after-inline", llvm::cl::init(false),
+    llvm::cl::desc("Run sil verification after inlining all found callee apply "
+                   "sites into a caller."));
+
 //===----------------------------------------------------------------------===//
 //                           Performance Inliner
 //===----------------------------------------------------------------------===//
@@ -919,11 +924,9 @@ bool SILPerformanceInliner::inlineCallsIntoFunction(SILFunction *Caller) {
     }
 
     // If we have a callee that doesn't have ownership, but the caller does have
-    // ownership... do not inline. The two modes are incompatible. Today this
-    // should only happen with transparent functions.
+    // ownership... do not inline. The two modes are incompatible, so skip this
+    // apply site for now.
     if (!Callee->hasOwnership() && Caller->hasOwnership()) {
-      assert(Caller->isTransparent() &&
-             "Should only happen with transparent functions");
       continue;
     }
 
@@ -953,6 +956,13 @@ bool SILPerformanceInliner::inlineCallsIntoFunction(SILFunction *Caller) {
     StackNesting().correctStackNesting(Caller);
   }
 
+  // If we were asked to verify our caller after inlining all callees we could
+  // find into it, do so now. This makes it easier to catch verification bugs in
+  // the inliner without running the entire inliner.
+  if (EnableVerifyAfterInlining) {
+    Caller->verify();
+  }
+
   return true;
 }
 
@@ -1027,6 +1037,11 @@ class SILPerformanceInlinerPass : public SILFunctionTransform {
 };
 } // end anonymous namespace
 
+SILTransform *swift::createAlwaysInlineInliner() {
+  return new SILPerformanceInlinerPass(InlineSelection::OnlyInlineAlways,
+                                       "InlineAlways");
+}
+
 /// Create an inliner pass that does not inline functions that are marked with
 /// the @_semantics, @_effects or global_init attributes.
 SILTransform *swift::createEarlyInliner() {

lib/SILOptimizer/Utils/PerformanceInlinerUtils.cpp

@@ -637,14 +637,21 @@ SILFunction *swift::getEligibleFunction(FullApplySite AI,
   if (!SILInliner::canInlineApplySite(AI))
     return nullptr;
 
+  // If our inline selection is only always inline, do a quick check if we have
+  // an always inline function and bail otherwise.
+  if (WhatToInline == InlineSelection::OnlyInlineAlways &&
+      Callee->getInlineStrategy() != AlwaysInline) {
+    return nullptr;
+  }
+
   ModuleDecl *SwiftModule = Callee->getModule().getSwiftModule();
   bool IsInStdlib = (SwiftModule->isStdlibModule() ||
                      SwiftModule->isOnoneSupportModule());
 
   // Don't inline functions that are marked with the @_semantics or @_effects
   // attribute if the inliner is asked not to inline them.
   if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) {
-    if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) {
+    if (WhatToInline >= InlineSelection::NoSemanticsAndGlobalInit) {
       if (shouldSkipApplyDuringEarlyInlining(AI))
         return nullptr;
       if (Callee->hasSemanticsAttr("inline_late"))
@@ -692,13 +699,14 @@ SILFunction *swift::getEligibleFunction(FullApplySite AI,
     // Check if passed Self is the same as the Self of the caller.
     // In this case, it is safe to inline because both functions
     // use the same Self.
-    if (AI.hasSelfArgument() && Caller->hasSelfMetadataParam()) {
-      auto CalleeSelf = stripCasts(AI.getSelfArgument());
-      auto CallerSelf = Caller->getSelfMetadataArgument();
-      if (CalleeSelf != SILValue(CallerSelf))
-        return nullptr;
-    } else
+    if (!AI.hasSelfArgument() || !Caller->hasSelfMetadataParam()) {
       return nullptr;
+    }
+    auto CalleeSelf = stripCasts(AI.getSelfArgument());
+    auto CallerSelf = Caller->getSelfMetadataArgument();
+    if (CalleeSelf != SILValue(CallerSelf)) {
+      return nullptr;
+    }
   }
 
   // Detect self-recursive calls.

test/SILOptimizer/inlinealways_inliner.sil

@@ -0,0 +1,44 @@
+// RUN: %target-sil-opt -enable-sil-verify-all %s -always-inline | %FileCheck %s
+
+sil @doSomething1 : $@convention(thin) () -> ()
+sil @doSomething2 : $@convention(thin) () -> ()
+sil @doSomething3 : $@convention(thin) () -> ()
+
+sil [ossa] [always_inline] @do_inline_this : $@convention(thin) () -> () {
+bb0:
+  %d1 = function_ref @doSomething1 : $@convention(thin) () -> ()
+  apply %d1() : $@convention(thin) () -> ()
+  %9999 = tuple()
+  return %9999 : $()
+}
+
+sil [ossa] @donot_inline_this : $@convention(thin) () -> () {
+bb0:
+  %d1 = function_ref @doSomething2 : $@convention(thin) () -> ()
+  apply %d1() : $@convention(thin) () -> ()
+  %9999 = tuple()
+  return %9999 : $()
+}
+
+sil [ossa] @empty_function : $@convention(thin) () -> () {
+bb0:
+  %9999 = tuple()
+  return %9999 : $()
+}
+
+// CHECK-LABEL: sil [ossa] @caller : $@convention(thin) () -> () {
+// CHECK-NOT: function_ref @do_inline_this : $@convention(thin) () -> ()
+// CHECK: function_ref @donot_inline_this : $@convention(thin) () -> ()
+// CHECK: function_ref @empty_function : $@convention(thin) () -> ()
+// CHECK: } // end sil function 'caller'
+sil [ossa] @caller : $@convention(thin) () -> () {
+bb0:
+  %c1 = function_ref @do_inline_this : $@convention(thin) () -> ()
+  apply %c1() : $@convention(thin) () -> ()
+  %c2 = function_ref @donot_inline_this : $@convention(thin) () -> ()
+  apply %c2() : $@convention(thin) () -> ()
+  %c3 = function_ref @empty_function : $@convention(thin) () -> ()
+  apply %c3() : $@convention(thin) () -> ()
+  %9999 = tuple()
+  return %9999 : $()
+}