[next] Cherry-pick commits on stable/20240723 but not next

clang/include/clang/AST/ASTConsumer.h

@@ -27,6 +27,7 @@ namespace clang {
   class VarDecl;
   class FunctionDecl;
   class ImportDecl;
+  class TargetInfo;
 
 /// ASTConsumer - This is an abstract interface that should be implemented by
 /// clients that read ASTs.  This abstraction layer allows the client to be
@@ -47,6 +48,14 @@ class ASTConsumer {
   /// ASTContext.
   virtual void Initialize(ASTContext &Context) {}
 
+  /// Initialize - This is called to initialize the consumer, providing the
+  /// ASTContext. 'CodeGenTargetInfo' specifies the code-generation configuration
+  /// for this compilation instance, which may differ from the one carried
+  /// by the Context itself only in the OS Version number -
+  /// for example when type-checking must be performed against an epoch OS version
+  /// while code-generation must run according to the user-specified OS version.
+  virtual void Initialize(ASTContext &Context, const TargetInfo &CodeGenTargetInfo) {}
+
   /// HandleTopLevelDecl - Handle the specified top-level declaration.  This is
   /// called by the parser to process every top-level Decl*.
   ///

clang/lib/CodeGen/CodeGenModule.cpp

@@ -335,12 +335,14 @@ CodeGenModule::CodeGenModule(ASTContext &C,
                              IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
                              const HeaderSearchOptions &HSO,
                              const PreprocessorOptions &PPO,
-                             const CodeGenOptions &CGO, llvm::Module &M,
+                             const CodeGenOptions &CGO,
+                             const TargetInfo &CGTI,
+                             llvm::Module &M,
                              DiagnosticsEngine &diags,
                              CoverageSourceInfo *CoverageInfo)
     : Context(C), LangOpts(C.getLangOpts()), FS(FS), HeaderSearchOpts(HSO),
       PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags),
-      Target(C.getTargetInfo()), ABI(createCXXABI(*this)),
+      Target(CGTI), ABI(createCXXABI(*this)),
       VMContext(M.getContext()), VTables(*this), StackHandler(diags),
       SanitizerMD(new SanitizerMetadata(*this)),
       AtomicOpts(Target.getAtomicOpts()) {
@@ -357,19 +359,19 @@ CodeGenModule::CodeGenModule(ASTContext &C,
   BFloatTy = llvm::Type::getBFloatTy(LLVMContext);
   FloatTy = llvm::Type::getFloatTy(LLVMContext);
   DoubleTy = llvm::Type::getDoubleTy(LLVMContext);
-  PointerWidthInBits = C.getTargetInfo().getPointerWidth(LangAS::Default);
+  PointerWidthInBits = Target.getPointerWidth(LangAS::Default);
   PointerAlignInBytes =
-      C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(LangAS::Default))
+      C.toCharUnitsFromBits(Target.getPointerAlign(LangAS::Default))
           .getQuantity();
   SizeSizeInBytes =
-    C.toCharUnitsFromBits(C.getTargetInfo().getMaxPointerWidth()).getQuantity();
+    C.toCharUnitsFromBits(Target.getMaxPointerWidth()).getQuantity();
   IntAlignInBytes =
-    C.toCharUnitsFromBits(C.getTargetInfo().getIntAlign()).getQuantity();
+    C.toCharUnitsFromBits(Target.getIntAlign()).getQuantity();
   CharTy =
-    llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getCharWidth());
-  IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth());
+    llvm::IntegerType::get(LLVMContext, Target.getCharWidth());
+  IntTy = llvm::IntegerType::get(LLVMContext, Target.getIntWidth());
   IntPtrTy = llvm::IntegerType::get(LLVMContext,
-    C.getTargetInfo().getMaxPointerWidth());
+    Target.getMaxPointerWidth());
   Int8PtrTy = llvm::PointerType::get(LLVMContext,
                                      C.getTargetAddressSpace(LangAS::Default));
   const llvm::DataLayout &DL = M.getDataLayout();

clang/lib/CodeGen/CodeGenModule.h

@@ -690,7 +690,9 @@ class CodeGenModule : public CodeGenTypeCache {
   CodeGenModule(ASTContext &C, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
                 const HeaderSearchOptions &headersearchopts,
                 const PreprocessorOptions &ppopts,
-                const CodeGenOptions &CodeGenOpts, llvm::Module &M,
+                const CodeGenOptions &CodeGenOpts,
+                const TargetInfo &CodeGenTargetInfo,
+                llvm::Module &M,
                 DiagnosticsEngine &Diags,
                 CoverageSourceInfo *CoverageInfo = nullptr);
 

clang/lib/CodeGen/ModuleBuilder.cpp

@@ -149,21 +149,26 @@ namespace {
     }
 
     void Initialize(ASTContext &Context) override {
+      Initialize(Context, Context.getTargetInfo());
+    }
+
+    void Initialize(ASTContext &Context, const TargetInfo &CodeGenTargetInfo) override {
       Ctx = &Context;
 
-      M->setTargetTriple(Ctx->getTargetInfo().getTriple());
-      M->setDataLayout(Ctx->getTargetInfo().getDataLayoutString());
-      const auto &SDKVersion = Ctx->getTargetInfo().getSDKVersion();
+      M->setTargetTriple(CodeGenTargetInfo.getTriple());
+      M->setDataLayout(CodeGenTargetInfo.getDataLayoutString());
+      const auto &SDKVersion = CodeGenTargetInfo.getSDKVersion();
       if (!SDKVersion.empty())
         M->setSDKVersion(SDKVersion);
-      if (const auto *TVT = Ctx->getTargetInfo().getDarwinTargetVariantTriple())
+      if (const auto *TVT = CodeGenTargetInfo.getDarwinTargetVariantTriple())
         M->setDarwinTargetVariantTriple(TVT->getTriple());
       if (auto TVSDKVersion =
-              Ctx->getTargetInfo().getDarwinTargetVariantSDKVersion())
+          CodeGenTargetInfo.getDarwinTargetVariantSDKVersion())
         M->setDarwinTargetVariantSDKVersion(*TVSDKVersion);
       Builder.reset(new CodeGen::CodeGenModule(Context, FS, HeaderSearchOpts,
                                                PreprocessorOpts, CodeGenOpts,
-                                               *M, Diags, CoverageInfo));
+                                               CodeGenTargetInfo, *M,
+                                               Diags, CoverageInfo));
 
       for (auto &&Lib : CodeGenOpts.DependentLibraries)
         Builder->AddDependentLib(Lib);

clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp

@@ -174,14 +174,19 @@ class PCHContainerGenerator : public ASTConsumer {
   ~PCHContainerGenerator() override = default;
 
   void Initialize(ASTContext &Context) override {
+    Initialize(Context, Context.getTargetInfo());
+  }
+
+  void Initialize(ASTContext &Context, const TargetInfo &CodeGenTargetInfo) override {
     assert(!Ctx && "initialized multiple times");
 
     Ctx = &Context;
     VMContext.reset(new llvm::LLVMContext());
     M.reset(new llvm::Module(MainFileName, *VMContext));
     M->setDataLayout(Ctx->getTargetInfo().getDataLayoutString());
     Builder.reset(new CodeGen::CodeGenModule(
-        *Ctx, FS, HeaderSearchOpts, PreprocessorOpts, CodeGenOpts, *M, Diags));
+        *Ctx, FS, HeaderSearchOpts, PreprocessorOpts, CodeGenOpts,
+        CodeGenTargetInfo, *M, Diags));
 
     // Prepare CGDebugInfo to emit debug info for a clang module.
     auto *DI = Builder->getModuleDebugInfo();

llvm/include/llvm/AsmParser/LLToken.h

@@ -162,6 +162,7 @@ enum Kind {
   kw_anyregcc,
   kw_swiftcc,
   kw_swifttailcc,
+  kw_swiftcorocc,
   kw_preserve_mostcc,
   kw_preserve_allcc,
   kw_preserve_nonecc,

llvm/include/llvm/Bitcode/LLVMBitCodes.h

@@ -791,6 +791,7 @@ enum AttributeKindCodes {
   ATTR_KIND_NO_DIVERGENCE_SOURCE = 100,
   ATTR_KIND_SANITIZE_TYPE = 101,
   ATTR_KIND_CAPTURES = 102,
+  ATTR_KIND_SWIFT_CORO = 103,
 };
 
 enum ComdatSelectionKindCodes {

llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h

@@ -538,6 +538,15 @@ class CallLowering {
     return false;
   }
 
+  /// This hook must be implemented to lower @llvm.ret.popless intrinsics,
+  /// which are required to be musttail, and are effectively annotating a
+  /// return instruction to mark it "popless", i.e., not restoring SP.
+  /// This "adjustment" step runs after lowerReturn, and is only meant to make
+  /// it a little less painful to maintain before we add this to the main hook.
+  virtual bool adjustReturnToPopless(MachineIRBuilder &MIRBuilder) const {
+    report_fatal_error("Popless returns not implemented for this target");
+  }
+
   virtual bool fallBackToDAGISel(const MachineFunction &MF) const {
     return false;
   }

llvm/include/llvm/CodeGen/MachineFrameInfo.h

@@ -319,6 +319,10 @@ class MachineFrameInfo {
   /// instructions which manipulate the stack pointer.
   bool HasCopyImplyingStackAdjustment = false;
 
+  /// True if the function contains a call using a calling convention that
+  /// allows it to be "popless", i.e., to not restore SP when the call returns.
+  bool HasPoplessCall = false;
+
   /// True if the function contains a call to the llvm.vastart intrinsic.
   bool HasVAStart = false;
 
@@ -634,6 +638,9 @@ class MachineFrameInfo {
     HasCopyImplyingStackAdjustment = B;
   }
 
+  bool hasPoplessCall() const { return HasPoplessCall; }
+  void setHasPoplessCall(bool B = true) { HasPoplessCall = B; }
+
   /// Returns true if the function calls the llvm.va_start intrinsic.
   bool hasVAStart() const { return HasVAStart; }
   void setHasVAStart(bool B) { HasVAStart = B; }

llvm/include/llvm/CodeGen/TargetCallingConv.h

@@ -41,6 +41,7 @@ namespace ISD {
     unsigned IsSplitEnd : 1;   ///< Last part of a split
     unsigned IsSwiftSelf : 1;  ///< Swift self parameter
     unsigned IsSwiftAsync : 1;  ///< Swift async context parameter
+    unsigned IsSwiftCoro : 1;  ///< Swift coro parameter
     unsigned IsSwiftError : 1; ///< Swift error parameter
     unsigned IsCFGuardTarget : 1; ///< Control Flow Guard target
     unsigned IsHva : 1;        ///< HVA field for
@@ -64,6 +65,7 @@ namespace ISD {
         : IsZExt(0), IsSExt(0), IsNoExt(0), IsInReg(0), IsSRet(0), IsByVal(0),
           IsByRef(0), IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0),
           IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), IsSwiftAsync(0),
+          IsSwiftCoro(0),
           IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0),
           IsSecArgPass(0), MemAlign(0), OrigAlign(0),
           IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
@@ -104,6 +106,9 @@ namespace ISD {
     bool isSwiftAsync() const { return IsSwiftAsync; }
     void setSwiftAsync() { IsSwiftAsync = 1; }
 
+    bool isSwiftCoro() const { return IsSwiftCoro; }
+    void setSwiftCoro() { IsSwiftCoro = 1; }
+
     bool isSwiftError() const { return IsSwiftError; }
     void setSwiftError() { IsSwiftError = 1; }
 

llvm/include/llvm/CodeGen/TargetLowering.h

@@ -315,6 +315,7 @@ class TargetLoweringBase {
     bool IsSwiftSelf : 1;
     bool IsSwiftAsync : 1;
     bool IsSwiftError : 1;
+    bool IsSwiftCoro : 1;
     bool IsCFGuardTarget : 1;
     MaybeAlign Alignment = std::nullopt;
     Type *IndirectType = nullptr;
@@ -324,7 +325,7 @@ class TargetLoweringBase {
           IsSRet(false), IsNest(false), IsByVal(false), IsByRef(false),
           IsInAlloca(false), IsPreallocated(false), IsReturned(false),
           IsSwiftSelf(false), IsSwiftAsync(false), IsSwiftError(false),
-          IsCFGuardTarget(false) {}
+          IsSwiftCoro(false), IsCFGuardTarget(false) {}
 
     void setAttributes(const CallBase *Call, unsigned ArgIdx);
   };
@@ -4889,6 +4890,10 @@ class TargetLowering : public TargetLoweringBase {
     llvm_unreachable("Not Implemented");
   }
 
+  virtual SDValue adjustReturnPopless(SDValue Chain, SelectionDAG &DAG) const {
+    report_fatal_error("Popless returns not implemented for this target");
+  }
+
   /// Return true if result of the specified node is used by a return node
   /// only. It also compute and return the input chain for the tail call.
   ///

llvm/include/llvm/IR/Attributes.td

@@ -358,6 +358,9 @@ def SwiftSelf : EnumAttr<"swiftself", IntersectPreserve, [ParamAttr]>;
 /// Argument is swift async context.
 def SwiftAsync : EnumAttr<"swiftasync", IntersectPreserve, [ParamAttr]>;
 
+/// Argument is swift coro allocator.
+def SwiftCoro : EnumAttr<"swiftcoro", IntersectPreserve, [ParamAttr]>;
+
 /// Function must be in a unwind table.
 def UWTable : IntAttr<"uwtable", IntersectPreserve, [FnAttr]>;
 

llvm/include/llvm/IR/CallingConv.h

@@ -284,6 +284,10 @@ namespace CallingConv {
     RISCV_VLSCall_32768 = 122,
     RISCV_VLSCall_65536 = 123,
 
+    /// This follows the Swift calling convention in how arguments are passed
+    /// but doesn't clean up the stack on a return.
+    SwiftCoro = 124,
+
     /// The highest possible ID. Must be some 2^k - 1.
     MaxID = 1023
   };

llvm/include/llvm/IR/Intrinsics.td

@@ -865,6 +865,20 @@ def int_localrecover : DefaultAttrsIntrinsic<[llvm_ptr_ty],
                                  [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty],
                                  [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 
+// Marks the following ret instruction as a "popless" return, one that does not
+// not restore SP to its function-entry value (i.e., does not deallocate the
+// stack frame), allowing allocations made in the function to be accessible
+// by the caller.
+//
+// The function must be annotated with an appropriate target-specific calling
+// convention, so the caller can generate stack accesses accordingly, generally
+// by treating the call as a variably-sized alloca, so using FP-based addressing
+// for its own frame rather than relying on statically known SP offsets.
+//
+// Calls to this intrinsic need to be musttail, but don't follow the other ABI
+// requirements for musttail calls, since this is really annotating the ret.
+def int_ret_popless : DefaultAttrsIntrinsic<[], [], [IntrInaccessibleMemOnly]>;
+
 // Given the frame pointer passed into an SEH filter function, returns a
 // pointer to the local variable area suitable for use with llvm.localrecover.
 def int_eh_recoverfp : DefaultAttrsIntrinsic<[llvm_ptr_ty],
@@ -1742,6 +1756,10 @@ def int_coro_id_retcon_once : Intrinsic<[llvm_token_ty],
     [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty,
      llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, llvm_vararg_ty],
     []>;
+def int_coro_id_retcon_once_dynamic : Intrinsic<[llvm_token_ty],
+    [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty,
+     llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
+    []>;
 def int_coro_alloc : Intrinsic<[llvm_i1_ty], [llvm_token_ty], []>;
 def int_coro_id_async : Intrinsic<[llvm_token_ty],
   [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty],

llvm/include/llvm/Target/TargetCallingConv.td

@@ -61,6 +61,11 @@ class CCIfSwiftAsync<CCAction A> : CCIf<"ArgFlags.isSwiftAsync()", A> {
 class CCIfSwiftError<CCAction A> : CCIf<"ArgFlags.isSwiftError()", A> {
 }
 
+/// CCIfSwiftCoro - If the current argument has swiftcoro parameter attribute,
+/// apply Action A.
+class CCIfSwiftCoro<CCAction A> : CCIf<"ArgFlags.isSwiftCoro()", A> {
+}
+
 /// CCIfCFGuardTarget - If the current argument has cfguardtarget parameter
 /// attribute, apply Action A.
 class CCIfCFGuardTarget<CCAction A> : CCIf<"ArgFlags.isCFGuardTarget()", A> {

llvm/include/llvm/Transforms/Coroutines/CoroInstr.h

@@ -135,6 +135,7 @@ class AnyCoroIdInst : public IntrinsicInst {
     auto ID = I->getIntrinsicID();
     return ID == Intrinsic::coro_id || ID == Intrinsic::coro_id_retcon ||
            ID == Intrinsic::coro_id_retcon_once ||
+           ID == Intrinsic::coro_id_retcon_once_dynamic ||
            ID == Intrinsic::coro_id_async;
   }
 
@@ -314,6 +315,72 @@ class CoroIdRetconOnceInst : public AnyCoroIdRetconInst {
   }
 };
 
+/// This represents the llvm.coro.id.retcon.once.dynamic instruction.
+class LLVM_LIBRARY_VISIBILITY CoroIdRetconOnceDynamicInst
+    : public AnyCoroIdInst {
+  enum {
+    SizeArg,
+    AlignArg,
+    CoroFuncPtrArg,
+    AllocatorArg,
+    StorageArg,
+    PrototypeArg,
+    AllocArg,
+    DeallocArg
+  };
+
+public:
+  void checkWellFormed() const;
+
+  uint64_t getStorageSize() const {
+    return cast<ConstantInt>(getArgOperand(SizeArg))->getZExtValue();
+  }
+
+  Align getStorageAlignment() const {
+    return cast<ConstantInt>(getArgOperand(AlignArg))->getAlignValue();
+  }
+
+  Value *getStorage() const { return getArgOperand(StorageArg); }
+
+  /// Return the coro function pointer address. This should be the address of
+  /// a coro function pointer struct for the current coro function.
+  /// struct coro_function_pointer {
+  ///   uint32_t frame size;
+  ///   uint32_t relative_pointer(coro_function);
+  ///  };
+  GlobalVariable *getCoroFunctionPointer() const {
+    return cast<GlobalVariable>(
+        getArgOperand(CoroFuncPtrArg)->stripPointerCasts());
+  }
+
+  /// Return the prototype for the continuation function.  The type,
+  /// attributes, and calling convention of the continuation function(s)
+  /// are taken from this declaration.
+  Function *getPrototype() const {
+    return cast<Function>(getArgOperand(PrototypeArg)->stripPointerCasts());
+  }
+
+  /// Return the function to use for allocating memory.
+  Function *getAllocFunction() const {
+    return cast<Function>(getArgOperand(AllocArg)->stripPointerCasts());
+  }
+
+  /// Return the function to use for deallocating memory.
+  Function *getDeallocFunction() const {
+    return cast<Function>(getArgOperand(DeallocArg)->stripPointerCasts());
+  }
+
+  Value *getAllocator() const { return getArgOperand(AllocatorArg); }
+
+  // Methods to support type inquiry through isa, cast, and dyn_cast:
+  static bool classof(const IntrinsicInst *I) {
+    return I->getIntrinsicID() == Intrinsic::coro_id_retcon_once_dynamic;
+  }
+  static bool classof(const Value *V) {
+    return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
+  }
+};
+
 /// This represents the llvm.coro.id.async instruction.
 class CoroIdAsyncInst : public AnyCoroIdInst {
   enum { SizeArg, AlignArg, StorageArg, AsyncFuncPtrArg };