[SYCL][OpenMP] Update compilation behaviors with fat archives

Allow for compilation using multiple objects and also enable compilation
from source.  Initial support for compilation from fat archives only allowed
the compile to be from a single object and the fat archive.  These changes
expand on that to allow compilation direct from source files and multiple
objects.

Signed-off-by: Michael D Toguchi <[email protected]>

Author: mdtoguchi

clang/include/clang/Driver/Action.h

@@ -598,7 +598,7 @@ class OffloadUnbundlingJobAction final : public JobAction {
 
 public:
   // Offloading unbundling doesn't change the type of output.
-  OffloadUnbundlingJobAction(Action *Input);
+  OffloadUnbundlingJobAction(ActionList &Inputs);
 
   /// Register information about a dependent action.
   void registerDependentActionInfo(const ToolChain *TC, StringRef BoundArch,

clang/include/clang/Driver/Types.def

@@ -104,4 +104,5 @@ TYPE("spirv",                    SPIRV,        INVALID,         "spv",   "")
 TYPE("sycl-header",              SYCL_Header,  INVALID,         "h",     "")
 TYPE("sycl-fatbin",              SYCL_FATBIN,  INVALID,         nullptr, "")
 TYPE("hip-fatbin",               HIP_FATBIN,   INVALID,         "hipfb", "A")
+TYPE("tempfilelist",             Tempfilelist, INVALID,         nullptr, "")
 TYPE("none",                     Nothing,      INVALID,         nullptr, "u")

clang/lib/Driver/Action.cpp

@@ -42,6 +42,8 @@ const char *Action::getClassName(ActionClass AC) {
     return "clang-offload-unbundler";
   case OffloadWrappingJobClass:
     return "clang-offload-wrapper";
+  case SPIRVTranslatorJobClass:
+    return "llvm-spirv";
   }
 
   llvm_unreachable("invalid class");
@@ -407,8 +409,8 @@ OffloadBundlingJobAction::OffloadBundlingJobAction(ActionList &Inputs)
 
 void OffloadUnbundlingJobAction::anchor() {}
 
-OffloadUnbundlingJobAction::OffloadUnbundlingJobAction(Action *Input)
-    : JobAction(OffloadUnbundlingJobClass, Input, Input->getType()) {}
+OffloadUnbundlingJobAction::OffloadUnbundlingJobAction(ActionList &Inputs)
+    : JobAction(OffloadUnbundlingJobClass, Inputs, Inputs.back()->getType()) {}
 
 void OffloadWrappingJobAction::anchor() {}
 

clang/lib/Driver/Driver.cpp

@@ -2461,17 +2461,18 @@ class OffloadingActionBuilder final {
           return ABRT_Inactive;
 
         CudaDeviceActions.clear();
-        auto *IA = cast<InputAction>(UA->getInputs().back());
-        std::string FileName = IA->getInputArg().getAsString(Args);
-        // Check if the type of the file is the same as the action. Do not
-        // unbundle it if it is not. Do not unbundle .so files, for example,
-        // which are not object files.
-        if (IA->getType() == types::TY_Object &&
-            (!llvm::sys::path::has_extension(FileName) ||
-             types::lookupTypeForExtension(
-                 llvm::sys::path::extension(FileName).drop_front()) !=
-                 types::TY_Object))
-          return ABRT_Inactive;
+        if (auto *IA = dyn_cast<InputAction>(UA->getInputs().back())) {
+          std::string FileName = IA->getInputArg().getAsString(Args);
+          // Check if the type of the file is the same as the action. Do not
+          // unbundle it if it is not. Do not unbundle .so files, for example,
+          // which are not object files.
+          if (IA->getType() == types::TY_Object &&
+              (!llvm::sys::path::has_extension(FileName) ||
+               types::lookupTypeForExtension(
+                   llvm::sys::path::extension(FileName).drop_front()) !=
+                   types::TY_Object))
+            return ABRT_Inactive;
+        }
 
         for (auto Arch : GpuArchList) {
           CudaDeviceActions.push_back(UA);
@@ -2893,17 +2894,18 @@ class OffloadingActionBuilder final {
       // If this is an unbundling action use it as is for each OpenMP toolchain.
       if (auto *UA = dyn_cast<OffloadUnbundlingJobAction>(HostAction)) {
         OpenMPDeviceActions.clear();
-        auto *IA = cast<InputAction>(UA->getInputs().back());
-        std::string FileName = IA->getInputArg().getAsString(Args);
-        // Check if the type of the file is the same as the action. Do not
-        // unbundle it if it is not. Do not unbundle .so files, for example,
-        // which are not object files.
-        if (IA->getType() == types::TY_Object &&
-            (!llvm::sys::path::has_extension(FileName) ||
-             types::lookupTypeForExtension(
-                 llvm::sys::path::extension(FileName).drop_front()) !=
-                 types::TY_Object))
-          return ABRT_Inactive;
+        if (auto *IA = dyn_cast<InputAction>(UA->getInputs().back())) {
+          std::string FileName = IA->getInputArg().getAsString(Args);
+          // Check if the type of the file is the same as the action. Do not
+          // unbundle it if it is not. Do not unbundle .so files, for example,
+          // which are not object files.
+          if (IA->getType() == types::TY_Object &&
+              (!llvm::sys::path::has_extension(FileName) ||
+               types::lookupTypeForExtension(
+                   llvm::sys::path::extension(FileName).drop_front()) !=
+                   types::TY_Object))
+            return ABRT_Inactive;
+        }
         for (unsigned I = 0; I < ToolChains.size(); ++I) {
           OpenMPDeviceActions.push_back(UA);
           UA->registerDependentActionInfo(
@@ -3102,21 +3104,22 @@ class OffloadingActionBuilder final {
       // If this is an unbundling action use it as is for each SYCL toolchain.
       if (auto *UA = dyn_cast<OffloadUnbundlingJobAction>(HostAction)) {
         SYCLDeviceActions.clear();
-        auto *IA = cast<InputAction>(UA->getInputs().back());
-        std::string FileName = IA->getInputArg().getAsString(Args);
-        // Check if the type of the file is the same as the action. Do not
-        // unbundle it if it is not. Do not unbundle .so files, for example,
-        // which are not object files.
-        if (IA->getType() == types::TY_Object &&
-            (!llvm::sys::path::has_extension(FileName) ||
-             types::lookupTypeForExtension(
+        if (auto *IA = dyn_cast<InputAction>(UA->getInputs().back())) {
+          std::string FileName = IA->getInputArg().getAsString(Args);
+          // Check if the type of the file is the same as the action. Do not
+          // unbundle it if it is not. Do not unbundle .so files, for example,
+          // which are not object files.
+          if (IA->getType() == types::TY_Object &&
+              (!llvm::sys::path::has_extension(FileName) ||
+               types::lookupTypeForExtension(
                  llvm::sys::path::extension(FileName).drop_front()) !=
                  types::TY_Object))
-          return ABRT_Inactive;
+            return ABRT_Inactive;
+        }
         for (unsigned I = 0; I < ToolChains.size(); ++I) {
           SYCLDeviceActions.push_back(UA);
           UA->registerDependentActionInfo(
-              ToolChains[I], /*BoundArch=*/StringRef(), Action::OFK_SYCL);
+            ToolChains[I], /*BoundArch=*/StringRef(), Action::OFK_SYCL);
         }
         return ABRT_Success;
       }
@@ -3373,7 +3376,8 @@ class OffloadingActionBuilder final {
   /// results will be kept in this action builder. Return true if an error was
   /// found.
   bool addHostDependenceToDeviceActions(Action *&HostAction,
-                                        const Arg *InputArg) {
+                                        const Arg *InputArg,
+                                        DerivedArgList &Args) {
     if (!IsValid)
       return true;
 
@@ -3386,12 +3390,27 @@ class OffloadingActionBuilder final {
     if (CanUseBundler && isa<InputAction>(HostAction) &&
         InputArg->getOption().getKind() == llvm::opt::Option::InputClass &&
         !types::isSrcFile(HostAction->getType())) {
-      auto UnbundlingHostAction =
-          C.MakeAction<OffloadUnbundlingJobAction>(HostAction);
-      UnbundlingHostAction->registerDependentActionInfo(
-          C.getSingleOffloadToolChain<Action::OFK_Host>(),
-          /*BoundArch=*/StringRef(), Action::OFK_Host);
-      HostAction = UnbundlingHostAction;
+      const char * InputName = InputArg->getValue();
+      // Do not create an unbundling action for an object when we know a fat
+      // static library is being used.  A separate unbundling action is created
+      // for all objects and the fat static library.
+      if (!(HostAction->getType() == types::TY_Object &&
+            llvm::sys::path::has_extension(InputName) &&
+            types::lookupTypeForExtension(
+              llvm::sys::path::extension(InputName).drop_front()) ==
+              types::TY_Object &&
+            Args.hasArg(options::OPT_foffload_static_lib_EQ))) {
+        ActionList HostActionList;
+        HostActionList.push_back(HostAction);
+        if (!HostActionList.empty()) {
+          auto UnbundlingHostAction =
+            C.MakeAction<OffloadUnbundlingJobAction>(HostActionList);
+          UnbundlingHostAction->registerDependentActionInfo(
+            C.getSingleOffloadToolChain<Action::OFK_Host>(),
+            /*BoundArch=*/StringRef(), Action::OFK_Host);
+          HostAction = UnbundlingHostAction;
+        }
+      }
     }
 
     assert(HostAction && "Invalid host action!");
@@ -3423,6 +3442,43 @@ class OffloadingActionBuilder final {
     return false;
   }
 
+  /// Generate an action that adds a host dependence to an unbundling action.
+  /// The results will be kept in this action builder. Return true if an error
+  /// was found.
+  bool addHostDependenceToUnbundlingAction(Action *&HostAction,
+                                           ActionList &InputActionList,
+                                           const Arg *InputArg) {
+    if (!IsValid || InputActionList.empty())
+      return true;
+
+    auto *DeviceUnbundlingAction =
+              C.MakeAction<OffloadUnbundlingJobAction>(InputActionList);
+    DeviceUnbundlingAction->registerDependentActionInfo(
+          C.getSingleOffloadToolChain<Action::OFK_Host>(),
+          /*BoundArch=*/StringRef(), Action::OFK_Host);
+    HostAction = DeviceUnbundlingAction;
+
+    // Register the offload kinds that are used.
+    auto &OffloadKind = InputArgToOffloadKindMap[InputArg];
+    for (auto *SB : SpecializedBuilders) {
+      if (!SB->isValid())
+        continue;
+
+      auto RetCode = SB->addDeviceDepences(HostAction);
+
+      // Host dependences for device actions are not compatible with that same
+      // action being ignored.
+      assert(RetCode != DeviceActionBuilder::ABRT_Ignore_Host &&
+             "Host dependence not expected to be ignored.!");
+
+      // Unless the builder was inactive for this action, we have to record the
+      // offload kind because the host will have to use it.
+      if (RetCode != DeviceActionBuilder::ABRT_Inactive)
+        OffloadKind |= SB->getAssociatedOffloadKind();
+    }
+    return false;
+  }
+
   /// Add the offloading top level actions that are specific for unique
   /// linking situations where objects are used at only the device link
   /// with no intermedate steps.
@@ -3674,7 +3730,8 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
 
     // Use the current host action in any of the offloading actions, if
     // required.
-    if (OffloadBuilder.addHostDependenceToDeviceActions(Current, InputArg))
+    if (OffloadBuilder.addHostDependenceToDeviceActions(Current, InputArg,
+                                                        Args))
       break;
 
     for (SmallVectorImpl<phases::ID>::iterator i = PL.begin(), e = PL.end();
@@ -3688,6 +3745,7 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
       // Add any offload action the host action depends on.
       Current = OffloadBuilder.addDeviceDependencesToHostAction(
           Current, InputArg, Phase, FinalPhase, PL);
+
       if (!Current)
         break;
 
@@ -3726,7 +3784,8 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
 
       // Use the current host action in any of the offloading actions, if
       // required.
-      if (OffloadBuilder.addHostDependenceToDeviceActions(Current, InputArg))
+      if (OffloadBuilder.addHostDependenceToDeviceActions(Current, InputArg,
+                                                          Args))
         break;
 
       if (Current->getType() == types::TY_Nothing)
@@ -3743,6 +3802,43 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
 
   OffloadBuilder.appendTopLevelLinkAction(Actions);
 
+  // When a static fat archive is provided, create a new unbundling step
+  // for all of the objects.
+  if (Args.hasArg(options::OPT_foffload_static_lib_EQ) &&
+      !LinkerInputs.empty()) {
+    ActionList UnbundlerInputs;
+    ActionList TempLinkerInputs;
+    for (const auto &LI : LinkerInputs) {
+      // Unbundler only handles objects.
+      if (auto *IA = dyn_cast<InputAction>(LI)) {
+        std::string FileName = IA->getInputArg().getAsString(Args);
+        if (IA->getType() == types::TY_Object &&
+            (!llvm::sys::path::has_extension(FileName) ||
+             types::lookupTypeForExtension(
+               llvm::sys::path::extension(FileName).drop_front()) !=
+               types::TY_Object))
+          // Pass the Input along to linker.
+          TempLinkerInputs.push_back(LI);
+        else
+          // Add to unbundler.
+          UnbundlerInputs.push_back(LI);
+      } else
+        UnbundlerInputs.push_back(LI);
+    }
+    LinkerInputs.clear();
+    if (!UnbundlerInputs.empty()) {
+      Action *Current;
+      const Arg *LastArg = Args.getLastArg(options::OPT_foffload_static_lib_EQ);
+      OffloadBuilder.addHostDependenceToUnbundlingAction(Current,
+                                                    UnbundlerInputs, LastArg);
+      Current = OffloadBuilder.addDeviceDependencesToHostAction(Current,
+                                       LastArg, phases::Link, FinalPhase, PL);
+      LinkerInputs.push_back(Current);
+    }
+    for (const auto &TLI : TempLinkerInputs)
+      LinkerInputs.push_back(TLI);
+  }
+
   // Add a link action if necessary.
   if (!LinkerInputs.empty()) {
     Action *LA = C.MakeAction<LinkJobAction>(LinkerInputs, types::TY_Image);
@@ -4495,18 +4591,30 @@ InputInfo Driver::BuildJobsForActionNoCache(
       // offloading prefix, we also do that for the host file because the
       // unbundling action does not change the type of the output which can
       // cause a overwrite.
-      std::string OffloadingPrefix = Action::GetOffloadingFileNamePrefix(
+      InputInfo CurI;
+      if (C.getInputArgs().hasArg(options::OPT_foffload_static_lib_EQ) &&
+          UI.DependentOffloadKind != Action::OFK_Host &&
+          JA->getType() == types::TY_Object) {
+        std::string TmpFileName =
+           C.getDriver().GetTemporaryPath(llvm::sys::path::stem(BaseInput),
+                                          "txt");
+        const char *TmpFile =
+                        C.addTempFile(C.getArgs().MakeArgString(TmpFileName));
+        CurI = InputInfo(types::TY_Tempfilelist, TmpFile, TmpFile);
+      } else {
+        std::string OffloadingPrefix = Action::GetOffloadingFileNamePrefix(
           UI.DependentOffloadKind,
           UI.DependentToolChain->getTriple().normalize(),
           /*CreatePrefixForHost=*/true);
-      auto CurI = InputInfo(
+        CurI = InputInfo(
           UA,
           GetNamedOutputPath(C, *UA, BaseInput, UI.DependentBoundArch,
                              /*AtTopLevel=*/false,
                              MultipleArchs ||
                                  UI.DependentOffloadKind == Action::OFK_HIP,
                              OffloadingPrefix),
           BaseInput);
+      }
       // Save the unbundling result.
       UnbundlingResults.push_back(CurI);
 

clang/lib/Driver/ToolChains/Clang.cpp

@@ -3508,6 +3508,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   bool IsWindowsMSVC = RawTriple.isWindowsMSVCEnvironment();
   bool IsIAMCU = RawTriple.isOSIAMCU();
   bool IsSYCLDevice = (RawTriple.getEnvironment() == llvm::Triple::SYCLDevice);
+  // Using just the sycldevice environment is not enough to determine usage
+  // of the device triple when considering fat static archives.  The
+  // compilation path requires the host object to be fed into the partial link
+  // step, and being part of the SYCL tool chain causes the incorrect target.
+  // FIXME - Is it possible to retain host environment when on a target
+  // device toolchain.
+  bool UseSYCLTriple = IsSYCLDevice && (!IsSYCL || IsSYCLOffloadDevice);
 
   // Adjust IsWindowsXYZ for CUDA/HIP compilations.  Even when compiling in
   // device mode (i.e., getToolchain().getTriple() is NVPTX/AMDGCN, not
@@ -3529,7 +3536,16 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Add the "effective" target triple.
   CmdArgs.push_back("-triple");
-  CmdArgs.push_back(Args.MakeArgString(TripleStr));
+  if (!UseSYCLTriple && IsSYCLDevice) {
+    // Do not use device triple when we know the device is not SYCL
+    // FIXME: We override the toolchain triple in this instance to address a
+    // disconnect with fat static archives.  We should have a cleaner way of
+    // using the Host environment when on a device toolchain.
+    std::string NormalizedTriple =
+        llvm::Triple(llvm::sys::getProcessTriple()).normalize();
+    CmdArgs.push_back(Args.MakeArgString(NormalizedTriple));
+  } else
+    CmdArgs.push_back(Args.MakeArgString(TripleStr));
 
   if (const Arg *MJ = Args.getLastArg(options::OPT_MJ)) {
     DumpCompilationDatabase(C, MJ->getValue(), TripleStr, Output, Input, Args);
@@ -3568,7 +3584,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(NormalizedTriple));
   }
 
-  if (IsSYCLDevice) {
+  if (UseSYCLTriple) {
     // We want to compile sycl kernels.
     if (types::isCXX(Input.getType()))
       CmdArgs.push_back("-std=c++11");
@@ -6365,8 +6381,6 @@ void OffloadBundler::ConstructJobMultipleOutputs(
   //   -unbundle
 
   ArgStringList CmdArgs;
-
-  assert(Inputs.size() == 1 && "Expecting to unbundle a single file!");
   InputInfo Input = Inputs.front();
   const char *TypeArg = types::getTypeTempSuffix(Input.getType());
   const char *InputFileName = Input.getFilename();
@@ -6386,9 +6400,10 @@ void OffloadBundler::ConstructJobMultipleOutputs(
           llvm::sys::path::stem(Input.getFilename()).str() + "-prelink", "o");
     InputFileName = C.addTempFile(C.getArgs().MakeArgString(TmpName));
     LinkArgs.push_back(InputFileName);
-    // Input files consist of fat libraries and the object to be unbundled.
-    LinkArgs.push_back(Input.getFilename());
-    for (const auto& A :
+    // Input files consist of fat libraries and the object(s) to be unbundled.
+    for (const auto &I : Inputs)
+      LinkArgs.push_back(I.getFilename());
+    for (const auto &A :
             TCArgs.getAllArgValues(options::OPT_foffload_static_lib_EQ))
       LinkArgs.push_back(TCArgs.MakeArgString(A));
     const char *Exec = TCArgs.MakeArgString(getToolChain().GetLinkerPath());