[SYCL][SPIR-V Backend][clang-sycl-linker] Add SPIR-V backend support inside clang-sycl-linker (#133967)

This PR does the following:

1. Use SPIR-V backend to do LLVM to SPIR-V translation inside
clang-sycl-linker
2. Remove llvm-spirv translator from clang-sycl-linker Currently, no
SPIR-V extensions are enabled for SYCL compilation flow. This will be
updated in subsequent commits.

Thanks

Note: This is one of the many PRs being introduced to add SYCL
programming model support to LLVM
([RFC](https://discourse.llvm.org/t/rfc-add-sycl-programming-model-support/50812)).

---------

Signed-off-by: Arvind Sudarsanam <[email protected]>

Author: asudarsa

clang/test/Driver/Inputs/SYCL/bar.ll

@@ -1,3 +1,4 @@
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G1"
 target triple = "spirv64"
 
 define spir_func i32 @bar_func1(i32 %a, i32 %b) {

clang/test/Driver/Inputs/SYCL/baz.ll

@@ -1,3 +1,4 @@
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G1"
 target triple = "spirv64"
 
 define spir_func i32 @bar_func1(i32 %a, i32 %b) {

clang/test/Driver/Inputs/SYCL/foo.ll

@@ -1,3 +1,4 @@
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G1"
 target triple = "spirv64"
 
 define spir_func i32 @foo_func1(i32 %a, i32 %b) {

clang/test/Driver/Inputs/SYCL/libsycl.ll

@@ -1,3 +1,4 @@
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G1"
 target triple = "spirv64"
 
 define spir_func i32 @addFive(i32 %a) {

clang/test/Driver/clang-sycl-linker-test.cpp

@@ -6,15 +6,15 @@
 // RUN: clang-sycl-linker --dry-run -v -triple=spirv64 %t_1.bc %t_2.bc -o a.spv 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=SIMPLE-FO
 // SIMPLE-FO: sycl-device-link: inputs: {{.*}}.bc, {{.*}}.bc  libfiles:  output: [[LLVMLINKOUT:.*]].bc
-// SIMPLE-FO-NEXT: "{{.*}}llvm-spirv{{.*}}" {{.*}}-o a.spv [[LLVMLINKOUT]].bc
+// SIMPLE-FO-NEXT: SPIR-V Backend: input: [[LLVMLINKOUT]].bc, output: a.spv
 //
 // Test the dry run of a simple case with device library files specified.
 // RUN: touch %T/lib1.bc
 // RUN: touch %T/lib2.bc
 // RUN: clang-sycl-linker --dry-run -v -triple=spirv64 %t_1.bc %t_2.bc --library-path=%T --device-libs=lib1.bc,lib2.bc -o a.spv 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=DEVLIBS
 // DEVLIBS: sycl-device-link: inputs: {{.*}}.bc  libfiles: {{.*}}lib1.bc, {{.*}}lib2.bc  output: [[LLVMLINKOUT:.*]].bc
-// DEVLIBS-NEXT: "{{.*}}llvm-spirv{{.*}}" {{.*}}-o a.spv [[LLVMLINKOUT]].bc
+// DEVLIBS-NEXT: SPIR-V Backend: input: [[LLVMLINKOUT]].bc, output: a.spv
 //
 // Test a simple case with a random file (not bitcode) as input.
 // RUN: touch %t.o
@@ -29,13 +29,3 @@
 // RUN: not clang-sycl-linker --dry-run -triple=spirv64 %t_1.bc %t_2.bc --library-path=%T --device-libs=lib1.bc,lib2.bc,lib3.bc -o a.spv 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=DEVLIBSERR2
 // DEVLIBSERR2: '{{.*}}lib3.bc' SYCL device library file is not found
-//
-// Test if correct set of llvm-spirv options are emitted for windows environment.
-// RUN: clang-sycl-linker --dry-run -v -triple=spirv64 --is-windows-msvc-env %t_1.bc %t_2.bc -o a.spv 2>&1 \
-// RUN:   | FileCheck %s --check-prefix=LLVMOPTSWIN
-// LLVMOPTSWIN: -spirv-debug-info-version=ocl-100 -spirv-allow-extra-diexpressions -spirv-allow-unknown-intrinsics=llvm.genx. -spirv-ext=
-//
-// Test if correct set of llvm-spirv options are emitted for linux environment.
-// RUN: clang-sycl-linker --dry-run -v -triple=spirv64  %t_1.bc %t_2.bc -o a.spv 2>&1 \
-// RUN:   | FileCheck %s --check-prefix=LLVMOPTSLIN
-// LLVMOPTSLIN: -spirv-debug-info-version=nonsemantic-shader-200 -spirv-allow-unknown-intrinsics=llvm.genx. -spirv-ext=

clang/test/Driver/sycl-link-spirv-target.cpp

@@ -3,7 +3,7 @@
 //
 // Test that -Xlinker options are being passed to clang-sycl-linker.
 // RUN: touch %t.bc
-// RUN: %clangxx -### --target=spirv64 --sycl-link -Xlinker --llvm-spirv-path=/tmp \
-// RUN:   -Xlinker -triple=spirv64 -Xlinker --library-path=/tmp -Xlinker --device-libs=lib1.bc,lib2.bc %t.bc 2>&1 \
+// RUN: %clangxx -### --target=spirv64 --sycl-link -Xlinker -triple=spirv64 -Xlinker --library-path=/tmp \
+// RUN:   -Xlinker --device-libs=lib1.bc,lib2.bc %t.bc 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=XLINKEROPTS
-// XLINKEROPTS: "{{.*}}clang-sycl-linker{{.*}}" "--llvm-spirv-path=/tmp" "-triple=spirv64" "--library-path=/tmp" "--device-libs=lib1.bc,lib2.bc" "{{.*}}.bc" "-o" "a.out"
+// XLINKEROPTS: "{{.*}}clang-sycl-linker{{.*}}" "-triple=spirv64" "--library-path=/tmp" "--device-libs=lib1.bc,lib2.bc" "{{.*}}.bc" "-o" "a.out"

clang/tools/clang-sycl-linker/CMakeLists.txt

@@ -1,14 +1,17 @@
 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
+  Analysis
   BinaryFormat
   BitWriter
   Core
   IRReader
   Linker
+  MC
   Option
   Object
-  TargetParser
   Support
+  Target
+  TargetParser
   )
 
 set(LLVM_TARGET_DEFINITIONS SYCLLinkOpts.td)

clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp

@@ -25,6 +25,7 @@
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Linker/Linker.h"
+#include "llvm/MC/TargetRegistry.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/ArchiveWriter.h"
 #include "llvm/Object/Binary.h"
@@ -48,6 +49,7 @@
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/WithColor.h"
+#include "llvm/Target/TargetMachine.h"
 
 using namespace llvm;
 using namespace llvm::opt;
@@ -141,40 +143,6 @@ Expected<StringRef> createTempFile(const ArgList &Args, const Twine &Prefix,
   return TempFiles.back();
 }
 
-Expected<std::string> findProgram(const ArgList &Args, StringRef Name,
-                                  ArrayRef<StringRef> Paths) {
-  if (Args.hasArg(OPT_dry_run))
-    return Name.str();
-  ErrorOr<std::string> Path = sys::findProgramByName(Name, Paths);
-  if (!Path)
-    Path = sys::findProgramByName(Name);
-  if (!Path)
-    return createStringError(Path.getError(),
-                             "Unable to find '" + Name + "' in path");
-  return *Path;
-}
-
-void printCommands(ArrayRef<StringRef> CmdArgs) {
-  if (CmdArgs.empty())
-    return;
-
-  llvm::errs() << " \"" << CmdArgs.front() << "\" ";
-  llvm::errs() << llvm::join(std::next(CmdArgs.begin()), CmdArgs.end(), " ")
-               << "\n";
-}
-
-/// Execute the command \p ExecutablePath with the arguments \p Args.
-Error executeCommands(StringRef ExecutablePath, ArrayRef<StringRef> Args) {
-  if (Verbose || DryRun)
-    printCommands(Args);
-
-  if (!DryRun)
-    if (sys::ExecuteAndWait(ExecutablePath, Args))
-      return createStringError(
-          "'%s' failed", sys::path::filename(ExecutablePath).str().c_str());
-  return Error::success();
-}
-
 Expected<SmallVector<std::string>> getInput(const ArgList &Args) {
   // Collect all input bitcode files to be passed to the device linking stage.
   SmallVector<std::string> BitcodeFiles;
@@ -243,12 +211,11 @@ Expected<SmallVector<std::string>> getSYCLDeviceLibs(const ArgList &Args) {
 /// 3. Link all the images gathered in Step 2 with the output of Step 1 using
 /// linkInModule API. LinkOnlyNeeded flag is used.
 Expected<StringRef> linkDeviceCode(ArrayRef<std::string> InputFiles,
-                                   const ArgList &Args) {
+                                   const ArgList &Args, LLVMContext &C) {
   llvm::TimeTraceScope TimeScope("SYCL link device code");
 
   assert(InputFiles.size() && "No inputs to link");
 
-  LLVMContext C;
   auto LinkerOutput = std::make_unique<Module>("sycl-device-link", C);
   Linker L(*LinkerOutput);
   // Link SYCL device input files.
@@ -307,120 +274,61 @@ Expected<StringRef> linkDeviceCode(ArrayRef<std::string> InputFiles,
   return *BitcodeOutput;
 }
 
-/// Add any llvm-spirv option that relies on a specific Triple in addition
-/// to user supplied options.
-static void getSPIRVTransOpts(const ArgList &Args,
-                              SmallVector<StringRef, 8> &TranslatorArgs,
-                              const llvm::Triple Triple) {
-  // Enable NonSemanticShaderDebugInfo.200 for non-Windows
-  const bool IsWindowsMSVC =
-      Triple.isWindowsMSVCEnvironment() || Args.hasArg(OPT_is_windows_msvc_env);
-  const bool EnableNonSemanticDebug = !IsWindowsMSVC;
-  if (EnableNonSemanticDebug) {
-    TranslatorArgs.push_back(
-        "-spirv-debug-info-version=nonsemantic-shader-200");
-  } else {
-    TranslatorArgs.push_back("-spirv-debug-info-version=ocl-100");
-    // Prevent crash in the translator if input IR contains DIExpression
-    // operations which don't have mapping to OpenCL.DebugInfo.100 spec.
-    TranslatorArgs.push_back("-spirv-allow-extra-diexpressions");
-  }
-  std::string UnknownIntrinsics("-spirv-allow-unknown-intrinsics=llvm.genx.");
-
-  TranslatorArgs.push_back(Args.MakeArgString(UnknownIntrinsics));
-
-  // Disable all the extensions by default
-  std::string ExtArg("-spirv-ext=-all");
-  std::string DefaultExtArg =
-      ",+SPV_EXT_shader_atomic_float_add,+SPV_EXT_shader_atomic_float_min_max"
-      ",+SPV_KHR_no_integer_wrap_decoration,+SPV_KHR_float_controls"
-      ",+SPV_KHR_expect_assume,+SPV_KHR_linkonce_odr";
-  std::string INTELExtArg =
-      ",+SPV_INTEL_subgroups,+SPV_INTEL_media_block_io"
-      ",+SPV_INTEL_device_side_avc_motion_estimation"
-      ",+SPV_INTEL_fpga_loop_controls,+SPV_INTEL_unstructured_loop_controls"
-      ",+SPV_INTEL_fpga_reg,+SPV_INTEL_blocking_pipes"
-      ",+SPV_INTEL_function_pointers,+SPV_INTEL_kernel_attributes"
-      ",+SPV_INTEL_io_pipes,+SPV_INTEL_inline_assembly"
-      ",+SPV_INTEL_arbitrary_precision_integers"
-      ",+SPV_INTEL_float_controls2,+SPV_INTEL_vector_compute"
-      ",+SPV_INTEL_fast_composite"
-      ",+SPV_INTEL_arbitrary_precision_fixed_point"
-      ",+SPV_INTEL_arbitrary_precision_floating_point"
-      ",+SPV_INTEL_variable_length_array,+SPV_INTEL_fp_fast_math_mode"
-      ",+SPV_INTEL_long_composites"
-      ",+SPV_INTEL_arithmetic_fence"
-      ",+SPV_INTEL_global_variable_decorations"
-      ",+SPV_INTEL_cache_controls"
-      ",+SPV_INTEL_fpga_buffer_location"
-      ",+SPV_INTEL_fpga_argument_interfaces"
-      ",+SPV_INTEL_fpga_invocation_pipelining_attributes"
-      ",+SPV_INTEL_fpga_latency_control"
-      ",+SPV_INTEL_task_sequence"
-      ",+SPV_KHR_shader_clock"
-      ",+SPV_INTEL_bindless_images";
-  ExtArg = ExtArg + DefaultExtArg + INTELExtArg;
-  ExtArg += ",+SPV_INTEL_token_type"
-            ",+SPV_INTEL_bfloat16_conversion"
-            ",+SPV_INTEL_joint_matrix"
-            ",+SPV_INTEL_hw_thread_queries"
-            ",+SPV_KHR_uniform_group_instructions"
-            ",+SPV_INTEL_masked_gather_scatter"
-            ",+SPV_INTEL_tensor_float32_conversion"
-            ",+SPV_INTEL_optnone"
-            ",+SPV_KHR_non_semantic_info"
-            ",+SPV_KHR_cooperative_matrix";
-  TranslatorArgs.push_back(Args.MakeArgString(ExtArg));
-}
-
 /// Run LLVM to SPIR-V translation.
-/// Converts 'File' from LLVM bitcode to SPIR-V format using llvm-spirv tool.
+/// Converts 'File' from LLVM bitcode to SPIR-V format using SPIR-V backend.
 /// 'Args' encompasses all arguments required for linking device code and will
-/// be parsed to generate options required to be passed into llvm-spirv tool.
-static Expected<StringRef> runLLVMToSPIRVTranslation(StringRef File,
-                                                     const ArgList &Args) {
-  llvm::TimeTraceScope TimeScope("LLVMToSPIRVTranslation");
-  StringRef LLVMSPIRVPath = Args.getLastArgValue(OPT_llvm_spirv_path_EQ);
-  Expected<std::string> LLVMToSPIRVProg =
-      findProgram(Args, "llvm-spirv", {LLVMSPIRVPath});
-  if (!LLVMToSPIRVProg)
-    return LLVMToSPIRVProg.takeError();
-
-  SmallVector<StringRef, 8> CmdArgs;
-  CmdArgs.push_back(*LLVMToSPIRVProg);
-  const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
-  getSPIRVTransOpts(Args, CmdArgs, Triple);
-  StringRef LLVMToSPIRVOptions;
-  if (Arg *A = Args.getLastArg(OPT_llvm_spirv_options_EQ))
-    LLVMToSPIRVOptions = A->getValue();
-  LLVMToSPIRVOptions.split(CmdArgs, " ", /* MaxSplit = */ -1,
-                           /* KeepEmpty = */ false);
-  CmdArgs.append({"-o", OutputFile});
-  CmdArgs.push_back(File);
-  if (Error Err = executeCommands(*LLVMToSPIRVProg, CmdArgs))
-    return std::move(Err);
-
-  if (!SPIRVDumpDir.empty()) {
-    std::error_code EC =
-        llvm::sys::fs::create_directory(SPIRVDumpDir, /*IgnoreExisting*/ true);
-    if (EC)
-      return createStringError(
-          EC,
-          formatv("failed to create dump directory. path: {0}, error_code: {1}",
-                  SPIRVDumpDir, EC.value()));
-
-    StringRef Path = OutputFile;
-    StringRef Filename = llvm::sys::path::filename(Path);
-    SmallString<128> CopyPath = SPIRVDumpDir;
-    CopyPath.append(Filename);
-    EC = llvm::sys::fs::copy_file(Path, CopyPath);
-    if (EC)
-      return createStringError(
-          EC,
-          formatv(
-              "failed to copy file. original: {0}, copy: {1}, error_code: {2}",
-              Path, CopyPath, EC.value()));
-  }
+/// be parsed to generate options required to be passed into the backend.
+static Expected<StringRef> runSPIRVCodeGen(StringRef File, const ArgList &Args,
+                                           LLVMContext &C) {
+  llvm::TimeTraceScope TimeScope("SPIR-V code generation");
+
+  // Parse input module.
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseIRFile(File, Err, C);
+  if (!M)
+    return createStringError(Err.getMessage());
+
+  Triple TargetTriple(Args.getLastArgValue(OPT_triple_EQ));
+  M->setTargetTriple(TargetTriple);
+
+  // Get a handle to SPIR-V target backend.
+  std::string Msg;
+  const Target *T = TargetRegistry::lookupTarget(M->getTargetTriple(), Msg);
+  if (!T)
+    return createStringError(Msg + ": " + M->getTargetTriple().str());
+
+  // Allocate SPIR-V target machine.
+  TargetOptions Options;
+  std::optional<Reloc::Model> RM;
+  std::optional<CodeModel::Model> CM;
+  std::unique_ptr<TargetMachine> TM(
+      T->createTargetMachine(M->getTargetTriple(), /* CPU */ "",
+                             /* Features */ "", Options, RM, CM));
+  if (!TM)
+    return createStringError("Could not allocate target machine!");
+
+  // Set data layout if needed.
+  if (M->getDataLayout().isDefault())
+    M->setDataLayout(TM->createDataLayout());
+
+  // Open output file for writing.
+  int FD = -1;
+  if (std::error_code EC = sys::fs::openFileForWrite(OutputFile, FD))
+    return errorCodeToError(EC);
+  auto OS = std::make_unique<llvm::raw_fd_ostream>(FD, true);
+
+  // Run SPIR-V codegen passes to generate SPIR-V file.
+  legacy::PassManager CodeGenPasses;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
+  CodeGenPasses.add(new TargetLibraryInfoWrapperPass(TLII));
+  if (TM->addPassesToEmitFile(CodeGenPasses, *OS, nullptr,
+                              CodeGenFileType::ObjectFile))
+    return createStringError("Failed to execute SPIR-V Backend");
+  CodeGenPasses.run(*M);
+
+  if (Verbose)
+    errs() << formatv("SPIR-V Backend: input: {0}, output: {1}\n", File,
+                      OutputFile);
 
   return OutputFile;
 }
@@ -429,15 +337,17 @@ static Expected<StringRef> runLLVMToSPIRVTranslation(StringRef File,
 /// 1. Link input device code (user code and SYCL device library code).
 /// 2. Run SPIR-V code generation.
 Error runSYCLLink(ArrayRef<std::string> Files, const ArgList &Args) {
-  llvm::TimeTraceScope TimeScope("SYCLDeviceLink");
+  llvm::TimeTraceScope TimeScope("SYCL device linking");
+
+  LLVMContext C;
 
   // Link all input bitcode files and SYCL device library files, if any.
-  auto LinkedFile = linkDeviceCode(Files, Args);
+  auto LinkedFile = linkDeviceCode(Files, Args, C);
   if (!LinkedFile)
     reportError(LinkedFile.takeError());
 
-  // LLVM to SPIR-V translation step
-  auto SPVFile = runLLVMToSPIRVTranslation(*LinkedFile, Args);
+  // SPIR-V code generation step.
+  auto SPVFile = runSPIRVCodeGen(*LinkedFile, Args, C);
   if (!SPVFile)
     return SPVFile.takeError();
   return Error::success();
@@ -447,6 +357,11 @@ Error runSYCLLink(ArrayRef<std::string> Files, const ArgList &Args) {
 
 int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
+  InitializeAllTargetInfos();
+  InitializeAllTargets();
+  InitializeAllTargetMCs();
+  InitializeAllAsmParsers();
+  InitializeAllAsmPrinters();
 
   Executable = argv[0];
   sys::PrintStackTraceOnErrorSignal(argv[0]);

clang/tools/clang-sycl-linker/SYCLLinkOpts.td

@@ -27,7 +27,7 @@ def device_libs_EQ : CommaJoined<["--", "-"], "device-libs=">,
 def arch_EQ : Joined<["--", "-"], "arch=">,
               Flags<[LinkerOnlyOption]>,
               MetaVarName<"<arch>">,
-              HelpText<"The device subarchitecture">;
+              HelpText<"The device architecture">;
 def triple_EQ : Joined<["--", "-"], "triple=">,
                 Flags<[LinkerOnlyOption]>,
                 MetaVarName<"<triple>">,
@@ -43,18 +43,6 @@ def spirv_dump_device_code_EQ : Joined<["--", "-"], "spirv-dump-device-code=">,
   Flags<[LinkerOnlyOption]>,
   HelpText<"Path to the folder where the tool dumps SPIR-V device code. Other formats aren't dumped.">;
 
-def is_windows_msvc_env : Flag<["--", "-"], "is-windows-msvc-env">,
-  Flags<[LinkerOnlyOption, HelpHidden]>;
-
-def llvm_spirv_path_EQ : Joined<["--"], "llvm-spirv-path=">,
-  Flags<[LinkerOnlyOption]>, MetaVarName<"<dir>">,
-  HelpText<"Set the system llvm-spirv path">;
-
-// Options to pass to llvm-spirv tool
-def llvm_spirv_options_EQ : Joined<["--", "-"], "llvm-spirv-options=">,
-  Flags<[LinkerOnlyOption]>,
-  HelpText<"Options that will control llvm-spirv step">;
-
 def print_linked_module : Flag<["--"], "print-linked-module">,
                           Flags<[LinkerOnlyOption]>,
                           HelpText<"Print the linked module's IR for testing">;